From 0260ae92f621270fe32d840427443739abb71638 Mon Sep 17 00:00:00 2001 From: mesh-tools Date: Fri, 16 Oct 2026 09:42:45 +0000 Subject: [PATCH] Feature 57 (config and provisioning): release the catalogue, the reviewed release branch merged with main, on top of main --- .gitignore | 3 + RELEASE-57.md | 6 + checks/config/go.mod | 3 + checks/config/main.go | 201 ++++ checks/config/main_test.go | 131 +++ checks/config/unmoved | 5 + checks/predefined/go.mod | 3 + checks/predefined/main.go | 153 +++ checks/predefined/main_test.go | 100 ++ checks/secrets/main.go | 12 +- checks/secrets/main_test.go | 29 + checks/templates/go.mod | 3 + checks/templates/inline | 9 + checks/templates/main.go | 178 +++ checks/templates/main_test.go | 102 ++ health-undeclared | 2 +- merge-check.sh | 49 +- modules/adwaita/README.md | 8 +- .../adwaita/cmd/adwaita-tools/adwaita_test.go | 64 +- .../{config => files}/cursor-index.theme | 0 modules/adwaita/files/env-gtk-theme | 1 + modules/adwaita/files/env-gtk2-rc-files | 1 + .../adwaita/files/env-qt-qpa-platformtheme | 1 + modules/adwaita/files/env-qt-select | 1 + modules/adwaita/files/env-qt-style-override | 1 + modules/adwaita/files/env-xcursor-size | 1 + modules/adwaita/files/env-xcursor-theme | 1 + .../{config => files}/gtk-settings.ini | 0 .../adwaita/{config => files}/portals.conf | 0 modules/adwaita/{config => files}/qt6ct.conf | 0 modules/adwaita/files/xinitrc-normal | 10 + modules/adwaita/files/xresources-normal | 3 + modules/adwaita/module.json | 104 +- .../artifact-store-tools/templates_test.go | 64 ++ .../artifact-store-tools/files/store-names | 4 + modules/artifact-store-tools/module.json | 22 +- .../cmd/zephyrus/keys_test.go | 8 +- .../cmd/zephyrus/manifest_test.go | 42 +- .../cmd/zephyrus/templates_test.go | 83 ++ modules/asus-zephyrus-g14/module.json | 66 +- .../UPower.conf.d/50-asus-zephyrus-g14.conf | 11 + .../30-asus-zephyrus-g14-touchpad.conf | 12 + .../asus-zephyrus-g14/templates/after-wake | 7 + .../templates/display-session | 30 + .../templates/hotkeys-triggers | 10 + .../modprobe.d/g14-nvidia-power.conf | 16 + .../modprobe.d/video-brightness-switch.conf | 6 + .../asus-zephyrus-g14.conf | 7 + .../asus-zephyrus-g14-nvidia.conf | 7 + .../asus-zephyrus-g14-nvidia.conf | 7 + .../asus-zephyrus-g14-nvidia.conf | 7 + .../templates/udev/rules.d/90-backlight.rules | 5 + modules/baserow/files/config.json | 4 + modules/baserow/files/server.env | 7 + modules/baserow/module.json | 30 +- modules/betterbird/README.md | 2 +- .../cmd/betterbird-tools/configuration.go | 3 +- .../cmd/betterbird-tools/manifest_test.go | 40 +- .../cmd/betterbird-tools/templates_test.go | 64 ++ modules/betterbird/files/accounts.json | 3 + modules/betterbird/files/betterbird | 10 + modules/betterbird/files/configuration.json | 6 + .../files/eu.betterbird.Betterbird.desktop | 25 + modules/betterbird/files/prefs.json | 3 + modules/betterbird/files/tags.json | 3 + modules/betterbird/module.json | 71 +- modules/build-agent/files/build-agent.env | 8 + modules/build-agent/manifest_test.go | 17 +- modules/build-agent/module.json | 7 +- modules/build-agent/templates_test.go | 64 ++ modules/ca-trust/files/anchor | 58 + modules/ca-trust/files/mesh-ca-trust.service | 16 + modules/ca-trust/module.json | 15 +- modules/claude-code/README.md | 13 +- modules/claude-code/cmd/claude-code/forge.go | 142 +++ .../claude-code/cmd/claude-code/forge_test.go | 262 +++++ modules/claude-code/cmd/claude-code/main.go | 18 + modules/claude-code/cmd/claude-code/node.go | 3 + modules/claude-code/cmd/claude-code/render.go | 4 + .../claude-code/cmd/claude-code/shell_test.go | 27 +- .../cmd/claude-code/templates_test.go | 64 ++ modules/claude-code/files/agent-account | 1 + modules/claude-code/files/claude-agent | 13 + modules/claude-code/files/facts.json | 4 + modules/claude-code/files/settings.json | 10 + modules/claude-code/files/zsh-normal | 12 + modules/claude-code/module.json | 82 +- .../claude-licence-manager/templates_test.go | 64 ++ .../claude-licence-manager/files/database.url | 1 + .../files/settings.json | 7 + modules/claude-licence-manager/module.json | 47 +- .../clipmenu-tools/manifest_helpers_test.go | 151 ++- .../cmd/clipmenu-tools/manifest_test.go | 19 +- modules/clipmenu/files/bin/clipmenud-session | 41 + modules/clipmenu/files/bin/xsel | 20 + modules/clipmenu/files/env-cm-histlength | 1 + modules/clipmenu/files/env-cm-max-clips | 1 + modules/clipmenu/files/env-cm-selections | 1 + .../files/systemd/clipmenud-mesh.conf | 14 + modules/clipmenu/module.json | 60 +- modules/confluence/files/config.json | 1 + modules/confluence/module.json | 33 +- modules/de-spiegel/module.json | 70 -- .../cmd/desk-channel/templates_test.go | 64 ++ modules/desk-channel/files/settings.json | 3 + modules/desk-channel/module.json | 16 +- .../cmd/dnsmasq-tools/templates_test.go | 64 ++ modules/dnsmasq/files/dnsmasq.conf | 87 ++ modules/dnsmasq/files/mesh.conf | 14 + modules/dnsmasq/module.json | 9 +- .../cmd/docker-tools/logrotation_test.go | 31 +- .../docker/cmd/docker-tools/templates_test.go | 64 ++ modules/docker/files/daemon.json | 1 + modules/docker/files/docker-prune.service | 15 + modules/docker/files/docker-prune.timer | 11 + modules/docker/files/log-rotation.conf | 5 + modules/docker/module.json | 40 +- .../cmd/dunst-tools/manifest_helpers_test.go | 152 ++- .../dunst/cmd/dunst-tools/manifest_test.go | 62 +- modules/dunst/files/dunstrc | 4 +- modules/dunst/module.json | 43 +- .../fail2ban/cmd/fail2ban-tools/own_test.go | 11 +- .../cmd/fail2ban-tools/templates_test.go | 64 ++ modules/fail2ban/files/fail2ban | 7 + .../files/iptables-allports-dualchain.conf | 40 + modules/fail2ban/files/jail.local | 48 + .../fail2ban/files/mesh-own-addresses.json | 3 + modules/fail2ban/files/own-addresses | 0 modules/fail2ban/files/recidive.conf | 9 + modules/fail2ban/files/sshd.conf | 8 + modules/fail2ban/module.json | 31 +- .../cmd/feh-tools/manifest_helpers_test.go | 151 ++- modules/feh/cmd/feh-tools/manifest_test.go | 13 +- modules/feh/files/xinitrc-normal | 2 + modules/feh/module.json | 30 +- modules/fonts/cmd/fonts-tools/fonts_test.go | 2 +- .../cmd/fonts-tools/manifest_kit_test.go | 6 +- .../fonts/cmd/fonts-tools/templates_test.go | 64 ++ modules/fonts/files/50-mesh-fonts.conf | 40 + modules/fonts/module.json | 7 +- modules/gitea/README.md | 36 + modules/gitea/cmd/agent-teams/main.go | 42 + modules/gitea/cmd/agent-teams/teams.go | 339 ++++++ modules/gitea/cmd/agent-teams/teams_test.go | 333 ++++++ modules/gitea/cmd/agent-token/main.go | 72 ++ modules/gitea/cmd/agent-token/token.go | 348 ++++++ modules/gitea/cmd/agent-token/token_test.go | 376 ++++++ modules/gitea/cmd/forge/templates_test.go | 64 ++ modules/gitea/files/agent-teams.json | 1 + modules/gitea/files/config.json | 1 + modules/gitea/files/origins | 1 + modules/gitea/files/ready-authors | 3 + modules/gitea/files/server.env | 7 + modules/gitea/go.mod | 9 +- modules/gitea/go.sum | 8 +- modules/gitea/module.json | 110 +- modules/gitea/test/tools-account.test.ts | 2 +- modules/gitlab/files/config.json | 1 + modules/gitlab/module.json | 23 +- .../manifest_helpers_test.go | 108 +- .../cmd/gnome-keyring-tools/manifest_test.go | 15 +- modules/gnome-keyring/files/xinitrc-first | 5 + modules/gnome-keyring/module.json | 18 +- modules/grafana/files/admin.secret | 1 + modules/grafana/files/config.json | 4 + modules/grafana/files/influxdb-api | 1 + modules/grafana/files/mesh-influxdb.yaml | 19 + modules/grafana/files/oidc-client.secret | 1 + modules/grafana/files/oidc.env | 13 + modules/grafana/module.json | 32 +- modules/grafana/test/manifest.test.ts | 19 +- modules/hello-web/files/index.html | 1 + modules/hello-web/module.json | 7 +- modules/home-assistant/files/config.json | 1 + modules/home-assistant/files/provisions.env | 4 + modules/home-assistant/module.json | 20 +- .../cmd/hostname-tools/hostsfile_test.go | 2 +- .../cmd/hostname-tools/templates_test.go | 64 ++ modules/hostname/files/hostname | 1 + modules/hostname/files/hosts | 7 + modules/hostname/module.json | 17 +- modules/i3/README.md | 2 +- modules/i3/cmd/i3-tools/manifest_test.go | 65 +- modules/i3/{config => files}/config | 0 modules/i3/files/env-xdg-current-desktop | 1 + modules/i3/files/env-xdg-session-desktop | 1 + modules/i3/files/wms/i3 | 9 + modules/i3/files/xinitrc-last | 2 + modules/i3/module.json | 42 +- modules/i3status-rust/README.md | 25 +- .../cmd/i3status-rust-tools/blocks_test.go | 97 +- .../manifest_helpers_test.go | 151 ++- .../cmd/i3status-rust-tools/manifest_test.go | 20 +- modules/i3status-rust/files/block.tmpl | 26 - modules/i3status-rust/files/bottom-bar.toml | 4 +- modules/i3status-rust/files/top-bar.toml | 2 +- modules/i3status-rust/files/xinitrc-normal | 3 + modules/i3status-rust/module.json | 45 +- modules/icecast/files/config.json | 1 + modules/icecast/files/icecast.xml | 56 + modules/icecast/module.json | 26 +- modules/influxdb/files/config.json | 1 + modules/influxdb/files/server.env | 6 + modules/influxdb/module.json | 41 +- modules/invoicing/files/api.env | 9 + modules/invoicing/module.json | 13 +- modules/jira/files/config.json | 1 + modules/jira/module.json | 33 +- .../kernel/cmd/kernel-tools/manifest_test.go | 12 +- modules/kernel/cmd/kernel-tools/shape_test.go | 13 +- modules/kernel/files/huge-pages.sh | 84 ++ .../files/mesh-kernel-huge-pages.service | 25 + modules/kernel/module.json | 18 +- .../keycloak/cmd/keycloak-provider/main.go | 62 +- .../keycloak/cmd/keycloak-provider/moved.go | 274 +++++ .../cmd/keycloak-provider/moved_test.go | 394 +++++++ .../cmd/keycloak-provider/templates_test.go | 64 ++ modules/keycloak/files/admin.env | 2 + modules/keycloak/files/config.json | 1 + modules/keycloak/files/consumers.json | 1 + modules/keycloak/files/database.env | 3 + modules/keycloak/files/hostname.env | 1 + modules/keycloak/go.mod | 2 +- modules/keycloak/go.sum | 4 +- modules/keycloak/module.json | 62 +- modules/lab/files/lab.env | 3 + modules/lab/module.json | 15 +- modules/lab/test/runs.test.ts | 18 + modules/lemurs/README.md | 2 +- .../lemurs/cmd/lemurs-tools/lemurs_test.go | 2 +- .../lemurs/cmd/lemurs-tools/manifest_test.go | 37 +- modules/lemurs/{config => files}/config.toml | 0 modules/lemurs/files/xserver | 8 + modules/lemurs/module.json | 9 +- modules/letta/files/config.json | 3 + modules/letta/files/pgpass | 1 + modules/letta/files/server.env | 6 + modules/letta/files/start.sh | 23 + modules/letta/module.json | 22 +- modules/local-model-consumer/files/openai.env | 3 + modules/local-model-consumer/module.json | 13 +- .../cmd/localization-tools/manifest_test.go | 6 +- .../cmd/localization-tools/templates_test.go | 64 ++ modules/localization/files/locale.conf | 3 + modules/localization/files/vconsole.conf | 3 + modules/localization/module.json | 9 +- .../cmd/logrotate-tools/logrotate_test.go | 2 +- .../cmd/logrotate-tools/manifest_test.go | 8 +- .../cmd/logrotate-tools/templates_test.go | 64 ++ modules/logrotate/files/logrotate.conf | 32 + modules/logrotate/module.json | 7 +- modules/mailu/files/admin.env | 2 + modules/mailu/files/config.json | 1 + modules/mailu/files/database.env | 5 + modules/mailu/files/mailu.env | 78 ++ modules/mailu/files/secret.env | 1 + modules/mailu/module.json | 56 +- modules/matrix/files/conduit.toml | 15 + modules/matrix/files/element.json | 53 + modules/matrix/module.json | 123 +- .../cmd/memory-pressure/templates_test.go | 64 ++ modules/memory-pressure/files/oomd.conf | 8 + modules/memory-pressure/files/root-slice.conf | 5 + modules/memory-pressure/files/sysctl.conf | 9 + .../memory-pressure/files/user-manager.conf | 7 + .../memory-pressure/files/zram-generator.conf | 11 + modules/memory-pressure/module.json | 15 +- modules/mesh-catalog/files/database.url | 1 + modules/mesh-catalog/module.json | 7 +- .../mesh-cli/cmd/mesh-cli/manifest_test.go | 18 +- modules/mesh-cli/files/zsh-normal | 3 + modules/mesh-cli/module.json | 13 +- modules/mesh-config/README.md | 52 + modules/mesh-config/cmd/mesh-config/config.go | 438 +++++++ .../cmd/mesh-config/config_test.go | 553 +++++++++ .../mesh-config/cmd/mesh-config/declared.go | 232 ++++ modules/mesh-config/cmd/mesh-config/list.go | 144 +++ modules/mesh-config/cmd/mesh-config/main.go | 283 +++++ .../mesh-config/cmd/mesh-config/proposals.go | 928 +++++++++++++++ .../cmd/mesh-config/proposals_test.go | 568 +++++++++ .../cmd/mesh-config/pulled_in_test.go | 24 + .../mesh-config/cmd/mesh-config/resolve.go | 229 ++++ .../mesh-config/cmd/mesh-config/sources.go | 252 ++++ modules/mesh-config/go.mod | 5 + modules/mesh-config/go.sum | 2 + modules/mesh-config/module.json | 73 ++ .../cmd/mesh-issues/manifest_test.go | 11 +- .../cmd/mesh-issues/templates_test.go | 64 ++ modules/mesh-issues/files/settings.json | 145 +++ modules/mesh-issues/module.json | 209 +++- .../cmd/mesh-knowledge/manifest_test.go | 5 +- .../cmd/mesh-knowledge/templates_test.go | 64 ++ modules/mesh-knowledge/files/database.url | 1 + modules/mesh-knowledge/files/settings.json | 5 + modules/mesh-knowledge/module.json | 28 +- .../mesh-mail/cmd/mesh-mail/manifest_test.go | 19 +- modules/mesh-mail/files/accounts.json | 3 + modules/mesh-mail/files/database-url | 1 + modules/mesh-mail/files/grants.json | 3 + modules/mesh-mail/files/store.json | 9 + modules/mesh-mail/module.json | 35 +- modules/mesh-provisioner-client/README.md | 88 ++ .../cmd/mesh-provisioner-client/asks.go | 119 ++ .../cmd/mesh-provisioner-client/asks_test.go | 154 +++ .../bars_golden_test.go | 89 ++ .../bound_gitea_test.go | 146 +++ .../cmd/mesh-provisioner-client/client.go | 1021 +++++++++++++++++ .../mesh-provisioner-client/client_test.go | 649 +++++++++++ .../cmd/mesh-provisioner-client/collect.go | 256 +++++ .../collect_select_test.go | 131 +++ .../mesh-provisioner-client/collect_test.go | 179 +++ .../cmd/mesh-provisioner-client/consumers.go | 257 +++++ .../mesh-provisioner-client/consumers_test.go | 262 +++++ .../mesh-provisioner-client/environment.go | 234 ++++ .../environment_test.go | 192 ++++ .../cmd/mesh-provisioner-client/fill.go | 104 ++ .../mesh-provisioner-client/i3_golden_test.go | 90 ++ .../cmd/mesh-provisioner-client/main.go | 170 +++ .../moved_collectors_test.go | 105 ++ .../moved_collectors_want_test.go | 10 + .../moved_forge_test.go | 106 ++ .../cmd/mesh-provisioner-client/plan.go | 335 ++++++ .../mesh-provisioner-client/providers_test.go | 104 ++ .../mesh-provisioner-client/pulled_in_test.go | 78 ++ .../cmd/mesh-provisioner-client/secrets.go | 254 ++++ .../mesh-provisioner-client/secrets_test.go | 511 +++++++++ .../cmd/mesh-provisioner-client/template.go | 194 ++++ .../testdata/accounts.conf | 25 + .../testdata/bars/battery.bottom-bar.toml | 119 ++ .../testdata/bars/battery.top-bar.toml | 36 + .../testdata/bars/both.bottom-bar.toml | 126 ++ .../testdata/bars/both.top-bar.toml | 36 + .../testdata/bars/none.bottom-bar.toml | 109 ++ .../testdata/bars/none.top-bar.toml | 36 + .../testdata/bars/power-meter.bottom-bar.toml | 116 ++ .../testdata/bars/power-meter.top-bar.toml | 36 + .../testdata/environment/all.posix | 26 + .../testdata/environment/all.systemd | 23 + .../testdata/environment/zsh.posix | 9 + .../testdata/environment/zsh.systemd | 6 + .../testdata/i3/all.config | 310 +++++ .../testdata/i3/i3-only.config | 201 ++++ .../testdata/xorg/all.xinitrc | 86 ++ .../testdata/xorg/all.xresources | 48 + .../testdata/xorg/i3-xterm.xinitrc | 53 + .../testdata/xorg/i3-xterm.xresources | 44 + .../xorg_golden_test.go | 98 ++ modules/mesh-provisioner-client/go.mod | 22 + modules/mesh-provisioner-client/go.sum | 27 + modules/mesh-provisioner-client/module.json | 59 + modules/mesh-provisioner/README.md | 33 + .../cmd/mesh-provisioner/main.go | 95 ++ .../cmd/mesh-provisioner/provisioner_test.go | 306 +++++ .../cmd/mesh-provisioner/provisioning.go | 216 ++++ .../cmd/mesh-provisioner/registry.go | 156 +++ modules/mesh-provisioner/go.mod | 5 + modules/mesh-provisioner/go.sum | 2 + modules/mesh-provisioner/module.json | 55 + modules/mesh-vault/README.md | 279 +++++ modules/mesh-vault/client.ts | 172 --- modules/mesh-vault/cmd/mesh-vault/askers.go | 175 +++ modules/mesh-vault/cmd/mesh-vault/core.go | 142 +++ .../mesh-vault/cmd/mesh-vault/core_test.go | 278 +++++ .../cmd/mesh-vault/followups_test.go | 162 +++ .../mesh-vault/cmd/mesh-vault/form_test.go | 221 ++++ modules/mesh-vault/cmd/mesh-vault/intake.go | 760 ++++++++++++ .../mesh-vault/cmd/mesh-vault/intake_test.go | 552 +++++++++ .../cmd/mesh-vault/issued_account_test.go | 99 ++ modules/mesh-vault/cmd/mesh-vault/keys.go | 168 +++ modules/mesh-vault/cmd/mesh-vault/ledger.go | 251 ++++ .../mesh-vault/cmd/mesh-vault/ledger_test.go | 129 +++ modules/mesh-vault/cmd/mesh-vault/main.go | 118 ++ modules/mesh-vault/cmd/mesh-vault/move.go | 369 ++++++ .../mesh-vault/cmd/mesh-vault/move_test.go | 321 ++++++ .../cmd/mesh-vault/named_pair_test.go | 44 + .../mesh-vault/cmd/mesh-vault/nodekey_test.go | 191 +++ .../cmd/mesh-vault/owncalls_test.go | 230 ++++ modules/mesh-vault/cmd/mesh-vault/pair.go | 814 +++++++++++++ .../mesh-vault/cmd/mesh-vault/pair_test.go | 487 ++++++++ .../mesh-vault/cmd/mesh-vault/perholder.go | 201 ++++ .../cmd/mesh-vault/perholder_bus_test.go | 284 +++++ .../cmd/mesh-vault/perholder_test.go | 291 +++++ .../cmd/mesh-vault/providers_test.go | 21 + .../cmd/mesh-vault/pulled_in_test.go | 45 + .../cmd/mesh-vault/reseal_health.go | 44 + .../cmd/mesh-vault/reseal_health_test.go | 82 ++ .../mesh-vault/cmd/mesh-vault/retirement.go | 466 ++++++++ .../cmd/mesh-vault/retirement_test.go | 537 +++++++++ .../cmd/mesh-vault/rotate_one_test.go | 119 ++ modules/mesh-vault/cmd/mesh-vault/store.go | 443 +++++++ .../mesh-vault/testdata/retirement-state.json | 56 + .../mesh-vault/testdata/retirement-word.json | 23 + modules/mesh-vault/cmd/mesh-vault/tools.go | 663 +++++++++++ .../mesh-vault/cmd/mesh-vault/tools_test.go | 152 +++ modules/mesh-vault/cmd/mesh-vault/vault.go | 361 ++++++ .../mesh-vault/cmd/mesh-vault/vault_test.go | 864 ++++++++++++++ modules/mesh-vault/go.mod | 22 + modules/mesh-vault/go.sum | 27 + modules/mesh-vault/index.ts | 31 - modules/mesh-vault/module.json | 97 +- modules/mesh-vault/package.json | 17 - modules/mesh-vault/provisioner/index.ts | 48 - modules/mesh-vault/test/ledger.test.ts | 80 -- modules/mesh-vault/tools/index.ts | 131 --- modules/mesh-vault/tsconfig.json | 12 - .../cmd/mesh-watcher/templates_test.go | 64 ++ modules/mesh-watcher/files/settings.json | 3 + modules/mesh-watcher/module.json | 16 +- .../messenger/cmd/messenger/templates_test.go | 64 ++ modules/messenger/files/settings.json | 4 + modules/messenger/module.json | 28 +- modules/minio/files/root.env | 2 + modules/minio/module.json | 7 +- modules/model-usage/files/database.url | 1 + modules/model-usage/module.json | 7 +- modules/mongodb/files/server-root.secret | 1 + modules/mongodb/module.json | 7 +- modules/mosquitto/files/bootstrap.env | 7 + modules/mosquitto/files/mosquitto.conf | 20 + modules/mosquitto/module.json | 9 +- modules/mounts/cmd/mounts/manifest_test.go | 25 +- modules/mounts/cmd/mounts/templates_test.go | 64 ++ modules/mounts/files/mounts.conf | 11 + modules/mounts/module.json | 31 +- modules/mssql/files/backup.sql | 14 + modules/mssql/files/sa.env | 2 + modules/mssql/module.json | 9 +- modules/n8n/files/n8n-database.secret | 1 + modules/n8n/files/server.env | 12 + modules/n8n/module.json | 9 +- modules/nats/files/nats.conf | 46 + modules/nats/module.json | 7 +- modules/nats/snapshot/templates_test.go | 64 ++ .../cmd/uplink-tools/templates_test.go | 64 ++ modules/networkmanager/files/50-mesh.conf | 63 + modules/networkmanager/module.json | 7 +- modules/nextcloud/files/config.json | 1 + modules/nextcloud/files/server.env | 14 + modules/nextcloud/module.json | 30 +- .../cmd/nfs-server/manifest_test.go | 34 +- .../cmd/nfs-server/templates_test.go | 64 ++ modules/nfs-server/files/50-mesh.conf | 12 + modules/nfs-server/files/shares.conf | 10 + modules/nfs-server/module.json | 30 +- modules/nftables/files/mesh-filter.service | 14 + modules/nftables/files/mesh.conf | 5 + modules/nftables/module.json | 9 +- modules/nftables/test/remove.test.ts | 10 +- modules/node-env/files/50-mesh.conf | 4 + modules/node-env/files/environment.sh | 5 + modules/node-env/module.json | 16 +- modules/nodered/files/config.json | 3 + modules/nodered/files/mqtt.env | 4 + modules/nodered/files/settings.js | 48 + modules/nodered/files/settings.json | 14 + modules/nodered/module.json | 116 +- modules/only-office/files/server.env | 5 + modules/only-office/module.json | 13 +- .../pacman/cmd/pacman-tools/manifest_test.go | 4 +- .../pacman/cmd/pacman-tools/templates_test.go | 64 ++ modules/pacman/files/pacman.conf | 29 + modules/pacman/files/reflector.conf | 8 + modules/pacman/module.json | 9 +- .../cmd/picom-tools/manifest_helpers_test.go | 98 +- .../picom/cmd/picom-tools/manifest_test.go | 28 +- modules/picom/files/picom.conf | 4 +- modules/picom/module.json | 37 +- .../cmd/polychromatic-tools/manifest_test.go | 36 +- modules/polychromatic/files/display-session | 6 + modules/polychromatic/module.json | 14 +- .../postgres/cmd/postgres-provider/main.go | 61 +- .../postgres/cmd/postgres-provider/moved.go | 253 ++++ .../cmd/postgres-provider/moved_test.go | 358 ++++++ modules/postgres/files/consumers.json | 1 + modules/postgres/go.mod | 2 +- modules/postgres/go.sum | 4 +- modules/postgres/module.json | 24 +- modules/power/README.md | 5 +- modules/power/cmd/power/templates_test.go | 129 +++ modules/power/module.json | 154 ++- modules/power/templates/bar-battery | 8 + modules/power/templates/bar-power-draw | 5 + .../templates/mesh-power/moments/after-boot | 4 + .../templates/mesh-power/moments/after-wake | 4 + .../mesh-power/moments/before-shutdown | 4 + .../templates/mesh-power/moments/before-sleep | 4 + .../templates/mesh-power/moments/on-battery | 4 + .../templates/mesh-power/moments/on-mains | 4 + .../systemd/logind.conf.d/power.conf | 8 + .../system/hibernate.target.d/mesh-power.conf | 3 + .../hybrid-sleep.target.d/mesh-power.conf | 3 + .../system/mesh-power-after-boot.service | 13 + .../system/mesh-power-after-wake.service | 11 + .../system/mesh-power-before-shutdown.service | 18 + .../system/mesh-power-before-sleep.service | 11 + .../systemd/system/mesh-power-supply.service | 10 + .../system/sleep.target.d/mesh-power.conf | 3 + .../mesh-power.conf | 3 + .../system/suspend.target.d/mesh-power.conf | 3 + .../udev/rules.d/90-mesh-power.rules | 5 + modules/powerlevel10k/files/zsh-normal | 3 + modules/powerlevel10k/module.json | 19 +- modules/powerlevel10k/test/module.test.ts | 20 +- modules/records/cmd/records/templates_test.go | 64 ++ modules/records/files/config.json | 1 + modules/records/files/origin | 1 + modules/records/module.json | 19 +- modules/redis/files/redis.conf | 3 + modules/redis/module.json | 7 +- .../cmd/restic-backups/retention_test.go | 29 +- .../cmd/restic-backups/templates_test.go | 64 ++ modules/restic/files/backups.conf | 2 + modules/restic/files/data.conf | 2 + modules/restic/files/retention.conf | 6 + modules/restic/module.json | 66 +- modules/rofi/cmd/rofi-tools/main.go | 4 +- .../cmd/rofi-tools/manifest_helpers_test.go | 110 +- modules/rofi/cmd/rofi-tools/manifest_test.go | 15 +- modules/rofi/cmd/rofi-tools/secret.go | 25 +- modules/rofi/cmd/rofi-tools/secret_test.go | 32 +- modules/rofi/go.mod | 2 +- modules/rofi/go.sum | 4 +- modules/rofi/module.json | 26 +- modules/route-adapter/README.md | 8 + .../route-adapter/cmd/route-provider/main.go | 37 + .../cmd/route-provider/provider.go | 36 + .../cmd/route-provider/provider_test.go | 72 ++ modules/route-adapter/files/config.json | 6 + modules/route-adapter/go.mod | 5 + modules/route-adapter/go.sum | 2 + modules/route-adapter/module.json | 55 +- modules/route-proxy/README.md | 7 + .../route-proxy/cmd/route-provider/main.go | 37 + .../cmd/route-provider/provider.go | 36 + .../cmd/route-provider/provider_test.go | 108 ++ modules/route-proxy/files/acme.env | 3 + modules/route-proxy/files/internal-acme.env | 3 + modules/route-proxy/go.mod | 5 + modules/route-proxy/go.sum | 2 + modules/route-proxy/module.json | 29 +- .../cmd/screen-lock-tools/lock_test.go | 2 +- .../manifest_helpers_test.go | 108 +- .../cmd/screen-lock-tools/manifest_test.go | 15 +- modules/screen-lock/files/xinitrc-normal | 8 + modules/screen-lock/module.json | 16 +- modules/searxng/files/config.json | 1 + modules/searxng/files/settings.yml | 16 + modules/searxng/module.json | 84 +- modules/showcase/module.json | 8 +- modules/showcase/provisioner/index.ts | 21 +- modules/showcase/templates/showcase.env | 5 + modules/showcase/tools/index.ts | 46 +- .../slack/cmd/slack-tools/manifest_test.go | 42 +- modules/slack/cmd/slack-tools/slack_test.go | 2 +- modules/slack/files/display-session | 11 + modules/slack/module.json | 14 +- .../cmd/ssh-client-tools/ssh_test.go | 19 +- .../cmd/ssh-client-tools/templates_test.go | 64 ++ modules/ssh-client/files/config | 6 + modules/ssh-client/module.json | 7 +- modules/sshd/files/10-mesh.conf | 10 + modules/sshd/module.json | 13 +- modules/step-ca/files/config.json | 1 + modules/step-ca/files/init.env | 2 + modules/step-ca/module.json | 16 +- modules/sudo/cmd/sudo-tools/manifest_test.go | 2 +- modules/sudo/cmd/sudo-tools/templates_test.go | 64 ++ modules/sudo/files/10-mesh-operator | 9 + modules/sudo/module.json | 7 +- modules/supabase/files/analytics-env.env | 14 + .../supabase/files/analytics-reconcile.exs | 86 ++ modules/supabase/files/analytics-start.sh | 41 + modules/supabase/files/db-init/jwt.sql | 2 + modules/supabase/files/db-init/logs.sql | 6 + modules/supabase/files/db-init/pooler.sql | 6 + modules/supabase/files/db-init/realtime.sql | 4 + modules/supabase/files/db-init/roles.sql | 7 + modules/supabase/files/db-init/supabase.sql | 3 + modules/supabase/files/db-init/webhooks.sql | 208 ++++ modules/supabase/files/functions-env.env | 6 + .../supabase/files/functions-main/index.ts | 94 ++ modules/supabase/files/gotrue.env | 28 + modules/supabase/files/kong.yml | 242 ++++ modules/supabase/files/meta-env.env | 6 + modules/supabase/files/pooler-env.env | 14 + modules/supabase/files/pooler.exs | 43 + modules/supabase/files/postgrest.conf | 7 + modules/supabase/files/realtime-env.env | 16 + modules/supabase/files/settings.json | 7 + modules/supabase/files/storage-env.env | 13 + modules/supabase/files/studio-env.env | 18 + modules/supabase/files/vector.yml | 258 +++++ modules/supabase/module.json | 70 +- .../cmd/uplink-tools/templates_test.go | 64 ++ .../systemd-networkd/files/00-mesh0.network | 35 + modules/systemd-networkd/module.json | 7 +- .../cmd/time-sync-tools/manifest_test.go | 2 +- .../cmd/time-sync-tools/templates_test.go | 64 ++ modules/time-sync/files/50-mesh.conf | 6 + modules/time-sync/module.json | 7 +- .../cmd/triggerhappy-tools/templates_test.go | 64 ++ modules/triggerhappy/files/service.conf | 8 + modules/triggerhappy/files/triggers.conf | 4 + modules/triggerhappy/module.json | 10 +- modules/umami/files/server.env | 3 + modules/umami/module.json | 7 +- modules/unifi/files/config.json | 4 + modules/unifi/module.json | 28 +- modules/xdg/README.md | 8 +- modules/xdg/cmd/xdg-tools/machine_test.go | 8 +- modules/xdg/cmd/xdg-tools/manifest_test.go | 43 +- modules/xdg/{config => files}/mimeapps.list | 0 modules/xdg/{config => files}/user-dirs.conf | 0 modules/xdg/{config => files}/user-dirs.dirs | 6 +- .../xdg/{config => files}/user-dirs.locale | 0 modules/xdg/module.json | 33 +- modules/xorg/README.md | 1 - modules/xorg/cmd/xorg/manifest_test.go | 9 +- modules/xorg/cmd/xorg/templates_test.go | 64 ++ modules/xorg/files/xinitrc | 51 + modules/xorg/files/xresources | 13 + modules/xorg/module.json | 10 +- modules/xterm/cmd/xterm-tools/xterm_test.go | 22 + modules/xterm/files/env-terminal | 1 + modules/xterm/files/xresources-normal | 31 + modules/xterm/module.json | 28 +- modules/zfs/cmd/zfs-tools/manifest_test.go | 30 +- modules/zfs/cmd/zfs-tools/templates_test.go | 64 ++ modules/zfs/files/scrub.conf | 6 + modules/zfs/module.json | 66 +- modules/zsh-autosuggestions/files/zsh-normal | 2 + modules/zsh-autosuggestions/module.json | 25 +- .../zsh-syntax-highlighting/files/zsh-last | 3 + modules/zsh-syntax-highlighting/module.json | 25 +- modules/zsh/files/env-editor | 1 + modules/zsh/files/env-path-local-bin | 1 + modules/zsh/files/env-path-scripts | 1 + modules/zsh/files/env-path-scripts-bin | 1 + modules/zsh/files/env-visual | 1 + modules/zsh/files/env-xdg-config-home | 1 + modules/zsh/files/tools.conf | 3 + modules/zsh/files/zshenv | 9 + modules/zsh/files/zshrc | 60 + modules/zsh/module.json | 109 +- modules/zsh/test/manifest.test.ts | 50 +- predefined.json | 13 + 646 files changed, 37611 insertions(+), 1861 deletions(-) create mode 100644 RELEASE-57.md create mode 100644 checks/config/go.mod create mode 100644 checks/config/main.go create mode 100644 checks/config/main_test.go create mode 100644 checks/config/unmoved create mode 100644 checks/predefined/go.mod create mode 100644 checks/predefined/main.go create mode 100644 checks/predefined/main_test.go create mode 100644 checks/templates/go.mod create mode 100644 checks/templates/inline create mode 100644 checks/templates/main.go create mode 100644 checks/templates/main_test.go rename modules/adwaita/{config => files}/cursor-index.theme (100%) create mode 100644 modules/adwaita/files/env-gtk-theme create mode 100644 modules/adwaita/files/env-gtk2-rc-files create mode 100644 modules/adwaita/files/env-qt-qpa-platformtheme create mode 100644 modules/adwaita/files/env-qt-select create mode 100644 modules/adwaita/files/env-qt-style-override create mode 100644 modules/adwaita/files/env-xcursor-size create mode 100644 modules/adwaita/files/env-xcursor-theme rename modules/adwaita/{config => files}/gtk-settings.ini (100%) rename modules/adwaita/{config => files}/portals.conf (100%) rename modules/adwaita/{config => files}/qt6ct.conf (100%) create mode 100644 modules/adwaita/files/xinitrc-normal create mode 100644 modules/adwaita/files/xresources-normal create mode 100644 modules/artifact-store-tools/cmd/artifact-store-tools/templates_test.go create mode 100644 modules/artifact-store-tools/files/store-names create mode 100644 modules/asus-zephyrus-g14/cmd/zephyrus/templates_test.go create mode 100644 modules/asus-zephyrus-g14/templates/UPower/UPower.conf.d/50-asus-zephyrus-g14.conf create mode 100644 modules/asus-zephyrus-g14/templates/X11/xorg.conf.d/30-asus-zephyrus-g14-touchpad.conf create mode 100644 modules/asus-zephyrus-g14/templates/after-wake create mode 100644 modules/asus-zephyrus-g14/templates/display-session create mode 100644 modules/asus-zephyrus-g14/templates/hotkeys-triggers create mode 100644 modules/asus-zephyrus-g14/templates/modprobe.d/g14-nvidia-power.conf create mode 100644 modules/asus-zephyrus-g14/templates/modprobe.d/video-brightness-switch.conf create mode 100644 modules/asus-zephyrus-g14/templates/systemd/system/nvidia-powerd.service.d/asus-zephyrus-g14.conf create mode 100644 modules/asus-zephyrus-g14/templates/systemd/system/systemd-hibernate.service.d/asus-zephyrus-g14-nvidia.conf create mode 100644 modules/asus-zephyrus-g14/templates/systemd/system/systemd-suspend-then-hibernate.service.d/asus-zephyrus-g14-nvidia.conf create mode 100644 modules/asus-zephyrus-g14/templates/systemd/system/systemd-suspend.service.d/asus-zephyrus-g14-nvidia.conf create mode 100644 modules/asus-zephyrus-g14/templates/udev/rules.d/90-backlight.rules create mode 100644 modules/baserow/files/config.json create mode 100644 modules/baserow/files/server.env create mode 100644 modules/betterbird/cmd/betterbird-tools/templates_test.go create mode 100644 modules/betterbird/files/accounts.json create mode 100755 modules/betterbird/files/betterbird create mode 100644 modules/betterbird/files/configuration.json create mode 100644 modules/betterbird/files/eu.betterbird.Betterbird.desktop create mode 100644 modules/betterbird/files/prefs.json create mode 100644 modules/betterbird/files/tags.json create mode 100644 modules/build-agent/files/build-agent.env create mode 100644 modules/build-agent/templates_test.go create mode 100755 modules/ca-trust/files/anchor create mode 100644 modules/ca-trust/files/mesh-ca-trust.service create mode 100644 modules/claude-code/cmd/claude-code/forge.go create mode 100644 modules/claude-code/cmd/claude-code/forge_test.go create mode 100644 modules/claude-code/cmd/claude-code/templates_test.go create mode 100644 modules/claude-code/files/agent-account create mode 100755 modules/claude-code/files/claude-agent create mode 100644 modules/claude-code/files/facts.json create mode 100644 modules/claude-code/files/settings.json create mode 100644 modules/claude-code/files/zsh-normal create mode 100644 modules/claude-licence-manager/cmd/claude-licence-manager/templates_test.go create mode 100644 modules/claude-licence-manager/files/database.url create mode 100644 modules/claude-licence-manager/files/settings.json create mode 100644 modules/clipmenu/files/bin/clipmenud-session create mode 100755 modules/clipmenu/files/bin/xsel create mode 100644 modules/clipmenu/files/env-cm-histlength create mode 100644 modules/clipmenu/files/env-cm-max-clips create mode 100644 modules/clipmenu/files/env-cm-selections create mode 100644 modules/clipmenu/files/systemd/clipmenud-mesh.conf create mode 100644 modules/confluence/files/config.json delete mode 100644 modules/de-spiegel/module.json create mode 100644 modules/desk-channel/cmd/desk-channel/templates_test.go create mode 100644 modules/desk-channel/files/settings.json create mode 100644 modules/dnsmasq/cmd/dnsmasq-tools/templates_test.go create mode 100644 modules/dnsmasq/files/dnsmasq.conf create mode 100644 modules/dnsmasq/files/mesh.conf create mode 100644 modules/docker/cmd/docker-tools/templates_test.go create mode 100644 modules/docker/files/daemon.json create mode 100644 modules/docker/files/docker-prune.service create mode 100644 modules/docker/files/docker-prune.timer create mode 100644 modules/docker/files/log-rotation.conf create mode 100644 modules/fail2ban/cmd/fail2ban-tools/templates_test.go create mode 100644 modules/fail2ban/files/fail2ban create mode 100644 modules/fail2ban/files/iptables-allports-dualchain.conf create mode 100644 modules/fail2ban/files/jail.local create mode 100644 modules/fail2ban/files/mesh-own-addresses.json create mode 100644 modules/fail2ban/files/own-addresses create mode 100644 modules/fail2ban/files/recidive.conf create mode 100644 modules/fail2ban/files/sshd.conf create mode 100644 modules/feh/files/xinitrc-normal create mode 100644 modules/fonts/cmd/fonts-tools/templates_test.go create mode 100644 modules/fonts/files/50-mesh-fonts.conf create mode 100644 modules/gitea/cmd/agent-teams/main.go create mode 100644 modules/gitea/cmd/agent-teams/teams.go create mode 100644 modules/gitea/cmd/agent-teams/teams_test.go create mode 100644 modules/gitea/cmd/agent-token/main.go create mode 100644 modules/gitea/cmd/agent-token/token.go create mode 100644 modules/gitea/cmd/agent-token/token_test.go create mode 100644 modules/gitea/cmd/forge/templates_test.go create mode 100644 modules/gitea/files/agent-teams.json create mode 100644 modules/gitea/files/config.json create mode 100644 modules/gitea/files/origins create mode 100644 modules/gitea/files/ready-authors create mode 100644 modules/gitea/files/server.env create mode 100644 modules/gitlab/files/config.json create mode 100644 modules/gnome-keyring/files/xinitrc-first create mode 100644 modules/grafana/files/admin.secret create mode 100644 modules/grafana/files/config.json create mode 100644 modules/grafana/files/influxdb-api create mode 100644 modules/grafana/files/mesh-influxdb.yaml create mode 100644 modules/grafana/files/oidc-client.secret create mode 100644 modules/grafana/files/oidc.env create mode 100644 modules/hello-web/files/index.html create mode 100644 modules/home-assistant/files/config.json create mode 100644 modules/home-assistant/files/provisions.env create mode 100644 modules/hostname/cmd/hostname-tools/templates_test.go create mode 100644 modules/hostname/files/hostname create mode 100644 modules/hostname/files/hosts rename modules/i3/{config => files}/config (100%) create mode 100644 modules/i3/files/env-xdg-current-desktop create mode 100644 modules/i3/files/env-xdg-session-desktop create mode 100755 modules/i3/files/wms/i3 create mode 100644 modules/i3/files/xinitrc-last delete mode 100644 modules/i3status-rust/files/block.tmpl create mode 100644 modules/i3status-rust/files/xinitrc-normal create mode 100644 modules/icecast/files/config.json create mode 100644 modules/icecast/files/icecast.xml create mode 100644 modules/influxdb/files/config.json create mode 100644 modules/influxdb/files/server.env create mode 100644 modules/invoicing/files/api.env create mode 100644 modules/jira/files/config.json create mode 100644 modules/kernel/files/huge-pages.sh create mode 100644 modules/kernel/files/mesh-kernel-huge-pages.service create mode 100644 modules/keycloak/cmd/keycloak-provider/moved.go create mode 100644 modules/keycloak/cmd/keycloak-provider/moved_test.go create mode 100644 modules/keycloak/cmd/keycloak-provider/templates_test.go create mode 100644 modules/keycloak/files/admin.env create mode 100644 modules/keycloak/files/config.json create mode 100644 modules/keycloak/files/consumers.json create mode 100644 modules/keycloak/files/database.env create mode 100644 modules/keycloak/files/hostname.env create mode 100644 modules/lab/files/lab.env rename modules/lemurs/{config => files}/config.toml (100%) create mode 100755 modules/lemurs/files/xserver create mode 100644 modules/letta/files/config.json create mode 100644 modules/letta/files/pgpass create mode 100644 modules/letta/files/server.env create mode 100644 modules/letta/files/start.sh create mode 100644 modules/local-model-consumer/files/openai.env create mode 100644 modules/localization/cmd/localization-tools/templates_test.go create mode 100644 modules/localization/files/locale.conf create mode 100644 modules/localization/files/vconsole.conf create mode 100644 modules/logrotate/cmd/logrotate-tools/templates_test.go create mode 100644 modules/logrotate/files/logrotate.conf create mode 100644 modules/mailu/files/admin.env create mode 100644 modules/mailu/files/config.json create mode 100644 modules/mailu/files/database.env create mode 100644 modules/mailu/files/mailu.env create mode 100644 modules/mailu/files/secret.env create mode 100644 modules/matrix/files/conduit.toml create mode 100644 modules/matrix/files/element.json create mode 100644 modules/memory-pressure/cmd/memory-pressure/templates_test.go create mode 100644 modules/memory-pressure/files/oomd.conf create mode 100644 modules/memory-pressure/files/root-slice.conf create mode 100644 modules/memory-pressure/files/sysctl.conf create mode 100644 modules/memory-pressure/files/user-manager.conf create mode 100644 modules/memory-pressure/files/zram-generator.conf create mode 100644 modules/mesh-catalog/files/database.url create mode 100644 modules/mesh-cli/files/zsh-normal create mode 100644 modules/mesh-config/README.md create mode 100644 modules/mesh-config/cmd/mesh-config/config.go create mode 100644 modules/mesh-config/cmd/mesh-config/config_test.go create mode 100644 modules/mesh-config/cmd/mesh-config/declared.go create mode 100644 modules/mesh-config/cmd/mesh-config/list.go create mode 100644 modules/mesh-config/cmd/mesh-config/main.go create mode 100644 modules/mesh-config/cmd/mesh-config/proposals.go create mode 100644 modules/mesh-config/cmd/mesh-config/proposals_test.go create mode 100644 modules/mesh-config/cmd/mesh-config/pulled_in_test.go create mode 100644 modules/mesh-config/cmd/mesh-config/resolve.go create mode 100644 modules/mesh-config/cmd/mesh-config/sources.go create mode 100644 modules/mesh-config/go.mod create mode 100644 modules/mesh-config/go.sum create mode 100644 modules/mesh-config/module.json create mode 100644 modules/mesh-issues/cmd/mesh-issues/templates_test.go create mode 100644 modules/mesh-issues/files/settings.json create mode 100644 modules/mesh-knowledge/cmd/mesh-knowledge/templates_test.go create mode 100644 modules/mesh-knowledge/files/database.url create mode 100644 modules/mesh-knowledge/files/settings.json create mode 100644 modules/mesh-mail/files/accounts.json create mode 100644 modules/mesh-mail/files/database-url create mode 100644 modules/mesh-mail/files/grants.json create mode 100644 modules/mesh-mail/files/store.json create mode 100644 modules/mesh-provisioner-client/README.md create mode 100644 modules/mesh-provisioner-client/cmd/mesh-provisioner-client/asks.go create mode 100644 modules/mesh-provisioner-client/cmd/mesh-provisioner-client/asks_test.go create mode 100644 modules/mesh-provisioner-client/cmd/mesh-provisioner-client/bars_golden_test.go create mode 100644 modules/mesh-provisioner-client/cmd/mesh-provisioner-client/bound_gitea_test.go create mode 100644 modules/mesh-provisioner-client/cmd/mesh-provisioner-client/client.go create mode 100644 modules/mesh-provisioner-client/cmd/mesh-provisioner-client/client_test.go create mode 100644 modules/mesh-provisioner-client/cmd/mesh-provisioner-client/collect.go create mode 100644 modules/mesh-provisioner-client/cmd/mesh-provisioner-client/collect_select_test.go create mode 100644 modules/mesh-provisioner-client/cmd/mesh-provisioner-client/collect_test.go create mode 100644 modules/mesh-provisioner-client/cmd/mesh-provisioner-client/consumers.go create mode 100644 modules/mesh-provisioner-client/cmd/mesh-provisioner-client/consumers_test.go create mode 100644 modules/mesh-provisioner-client/cmd/mesh-provisioner-client/environment.go create mode 100644 modules/mesh-provisioner-client/cmd/mesh-provisioner-client/environment_test.go create mode 100644 modules/mesh-provisioner-client/cmd/mesh-provisioner-client/fill.go create mode 100644 modules/mesh-provisioner-client/cmd/mesh-provisioner-client/i3_golden_test.go create mode 100644 modules/mesh-provisioner-client/cmd/mesh-provisioner-client/main.go create mode 100644 modules/mesh-provisioner-client/cmd/mesh-provisioner-client/moved_collectors_test.go create mode 100644 modules/mesh-provisioner-client/cmd/mesh-provisioner-client/moved_collectors_want_test.go create mode 100644 modules/mesh-provisioner-client/cmd/mesh-provisioner-client/moved_forge_test.go create mode 100644 modules/mesh-provisioner-client/cmd/mesh-provisioner-client/plan.go create mode 100644 modules/mesh-provisioner-client/cmd/mesh-provisioner-client/providers_test.go create mode 100644 modules/mesh-provisioner-client/cmd/mesh-provisioner-client/pulled_in_test.go create mode 100644 modules/mesh-provisioner-client/cmd/mesh-provisioner-client/secrets.go create mode 100644 modules/mesh-provisioner-client/cmd/mesh-provisioner-client/secrets_test.go create mode 100644 modules/mesh-provisioner-client/cmd/mesh-provisioner-client/template.go create mode 100644 modules/mesh-provisioner-client/cmd/mesh-provisioner-client/testdata/accounts.conf create mode 100644 modules/mesh-provisioner-client/cmd/mesh-provisioner-client/testdata/bars/battery.bottom-bar.toml create mode 100644 modules/mesh-provisioner-client/cmd/mesh-provisioner-client/testdata/bars/battery.top-bar.toml create mode 100644 modules/mesh-provisioner-client/cmd/mesh-provisioner-client/testdata/bars/both.bottom-bar.toml create mode 100644 modules/mesh-provisioner-client/cmd/mesh-provisioner-client/testdata/bars/both.top-bar.toml create mode 100644 modules/mesh-provisioner-client/cmd/mesh-provisioner-client/testdata/bars/none.bottom-bar.toml create mode 100644 modules/mesh-provisioner-client/cmd/mesh-provisioner-client/testdata/bars/none.top-bar.toml create mode 100644 modules/mesh-provisioner-client/cmd/mesh-provisioner-client/testdata/bars/power-meter.bottom-bar.toml create mode 100644 modules/mesh-provisioner-client/cmd/mesh-provisioner-client/testdata/bars/power-meter.top-bar.toml create mode 100644 modules/mesh-provisioner-client/cmd/mesh-provisioner-client/testdata/environment/all.posix create mode 100644 modules/mesh-provisioner-client/cmd/mesh-provisioner-client/testdata/environment/all.systemd create mode 100644 modules/mesh-provisioner-client/cmd/mesh-provisioner-client/testdata/environment/zsh.posix create mode 100644 modules/mesh-provisioner-client/cmd/mesh-provisioner-client/testdata/environment/zsh.systemd create mode 100644 modules/mesh-provisioner-client/cmd/mesh-provisioner-client/testdata/i3/all.config create mode 100644 modules/mesh-provisioner-client/cmd/mesh-provisioner-client/testdata/i3/i3-only.config create mode 100644 modules/mesh-provisioner-client/cmd/mesh-provisioner-client/testdata/xorg/all.xinitrc create mode 100644 modules/mesh-provisioner-client/cmd/mesh-provisioner-client/testdata/xorg/all.xresources create mode 100644 modules/mesh-provisioner-client/cmd/mesh-provisioner-client/testdata/xorg/i3-xterm.xinitrc create mode 100644 modules/mesh-provisioner-client/cmd/mesh-provisioner-client/testdata/xorg/i3-xterm.xresources create mode 100644 modules/mesh-provisioner-client/cmd/mesh-provisioner-client/xorg_golden_test.go create mode 100644 modules/mesh-provisioner-client/go.mod create mode 100644 modules/mesh-provisioner-client/go.sum create mode 100644 modules/mesh-provisioner-client/module.json create mode 100644 modules/mesh-provisioner/README.md create mode 100644 modules/mesh-provisioner/cmd/mesh-provisioner/main.go create mode 100644 modules/mesh-provisioner/cmd/mesh-provisioner/provisioner_test.go create mode 100644 modules/mesh-provisioner/cmd/mesh-provisioner/provisioning.go create mode 100644 modules/mesh-provisioner/cmd/mesh-provisioner/registry.go create mode 100644 modules/mesh-provisioner/go.mod create mode 100644 modules/mesh-provisioner/go.sum create mode 100644 modules/mesh-provisioner/module.json create mode 100644 modules/mesh-vault/README.md delete mode 100644 modules/mesh-vault/client.ts create mode 100644 modules/mesh-vault/cmd/mesh-vault/askers.go create mode 100644 modules/mesh-vault/cmd/mesh-vault/core.go create mode 100644 modules/mesh-vault/cmd/mesh-vault/core_test.go create mode 100644 modules/mesh-vault/cmd/mesh-vault/followups_test.go create mode 100644 modules/mesh-vault/cmd/mesh-vault/form_test.go create mode 100644 modules/mesh-vault/cmd/mesh-vault/intake.go create mode 100644 modules/mesh-vault/cmd/mesh-vault/intake_test.go create mode 100644 modules/mesh-vault/cmd/mesh-vault/issued_account_test.go create mode 100644 modules/mesh-vault/cmd/mesh-vault/keys.go create mode 100644 modules/mesh-vault/cmd/mesh-vault/ledger.go create mode 100644 modules/mesh-vault/cmd/mesh-vault/ledger_test.go create mode 100644 modules/mesh-vault/cmd/mesh-vault/main.go create mode 100644 modules/mesh-vault/cmd/mesh-vault/move.go create mode 100644 modules/mesh-vault/cmd/mesh-vault/move_test.go create mode 100644 modules/mesh-vault/cmd/mesh-vault/named_pair_test.go create mode 100644 modules/mesh-vault/cmd/mesh-vault/nodekey_test.go create mode 100644 modules/mesh-vault/cmd/mesh-vault/owncalls_test.go create mode 100644 modules/mesh-vault/cmd/mesh-vault/pair.go create mode 100644 modules/mesh-vault/cmd/mesh-vault/pair_test.go create mode 100644 modules/mesh-vault/cmd/mesh-vault/perholder.go create mode 100644 modules/mesh-vault/cmd/mesh-vault/perholder_bus_test.go create mode 100644 modules/mesh-vault/cmd/mesh-vault/perholder_test.go create mode 100644 modules/mesh-vault/cmd/mesh-vault/providers_test.go create mode 100644 modules/mesh-vault/cmd/mesh-vault/pulled_in_test.go create mode 100644 modules/mesh-vault/cmd/mesh-vault/reseal_health.go create mode 100644 modules/mesh-vault/cmd/mesh-vault/reseal_health_test.go create mode 100644 modules/mesh-vault/cmd/mesh-vault/retirement.go create mode 100644 modules/mesh-vault/cmd/mesh-vault/retirement_test.go create mode 100644 modules/mesh-vault/cmd/mesh-vault/rotate_one_test.go create mode 100644 modules/mesh-vault/cmd/mesh-vault/store.go create mode 100644 modules/mesh-vault/cmd/mesh-vault/testdata/retirement-state.json create mode 100644 modules/mesh-vault/cmd/mesh-vault/testdata/retirement-word.json create mode 100644 modules/mesh-vault/cmd/mesh-vault/tools.go create mode 100644 modules/mesh-vault/cmd/mesh-vault/tools_test.go create mode 100644 modules/mesh-vault/cmd/mesh-vault/vault.go create mode 100644 modules/mesh-vault/cmd/mesh-vault/vault_test.go create mode 100644 modules/mesh-vault/go.mod create mode 100644 modules/mesh-vault/go.sum delete mode 100644 modules/mesh-vault/index.ts delete mode 100644 modules/mesh-vault/package.json delete mode 100644 modules/mesh-vault/provisioner/index.ts delete mode 100644 modules/mesh-vault/test/ledger.test.ts delete mode 100644 modules/mesh-vault/tools/index.ts delete mode 100644 modules/mesh-vault/tsconfig.json create mode 100644 modules/mesh-watcher/cmd/mesh-watcher/templates_test.go create mode 100644 modules/mesh-watcher/files/settings.json create mode 100644 modules/messenger/cmd/messenger/templates_test.go create mode 100644 modules/messenger/files/settings.json create mode 100644 modules/minio/files/root.env create mode 100644 modules/model-usage/files/database.url create mode 100644 modules/mongodb/files/server-root.secret create mode 100644 modules/mosquitto/files/bootstrap.env create mode 100644 modules/mosquitto/files/mosquitto.conf create mode 100644 modules/mounts/cmd/mounts/templates_test.go create mode 100644 modules/mounts/files/mounts.conf create mode 100644 modules/mssql/files/backup.sql create mode 100644 modules/mssql/files/sa.env create mode 100644 modules/n8n/files/n8n-database.secret create mode 100644 modules/n8n/files/server.env create mode 100644 modules/nats/files/nats.conf create mode 100644 modules/nats/snapshot/templates_test.go create mode 100644 modules/networkmanager/cmd/uplink-tools/templates_test.go create mode 100644 modules/networkmanager/files/50-mesh.conf create mode 100644 modules/nextcloud/files/config.json create mode 100644 modules/nextcloud/files/server.env create mode 100644 modules/nfs-server/cmd/nfs-server/templates_test.go create mode 100644 modules/nfs-server/files/50-mesh.conf create mode 100644 modules/nfs-server/files/shares.conf create mode 100644 modules/nftables/files/mesh-filter.service create mode 100644 modules/nftables/files/mesh.conf create mode 100644 modules/node-env/files/50-mesh.conf create mode 100644 modules/node-env/files/environment.sh create mode 100644 modules/nodered/files/config.json create mode 100644 modules/nodered/files/mqtt.env create mode 100644 modules/nodered/files/settings.js create mode 100644 modules/nodered/files/settings.json create mode 100644 modules/only-office/files/server.env create mode 100644 modules/pacman/cmd/pacman-tools/templates_test.go create mode 100644 modules/pacman/files/pacman.conf create mode 100644 modules/pacman/files/reflector.conf create mode 100644 modules/polychromatic/files/display-session create mode 100644 modules/postgres/cmd/postgres-provider/moved.go create mode 100644 modules/postgres/cmd/postgres-provider/moved_test.go create mode 100644 modules/postgres/files/consumers.json create mode 100644 modules/power/cmd/power/templates_test.go create mode 100644 modules/power/templates/bar-battery create mode 100644 modules/power/templates/bar-power-draw create mode 100644 modules/power/templates/mesh-power/moments/after-boot create mode 100644 modules/power/templates/mesh-power/moments/after-wake create mode 100644 modules/power/templates/mesh-power/moments/before-shutdown create mode 100644 modules/power/templates/mesh-power/moments/before-sleep create mode 100644 modules/power/templates/mesh-power/moments/on-battery create mode 100644 modules/power/templates/mesh-power/moments/on-mains create mode 100644 modules/power/templates/systemd/logind.conf.d/power.conf create mode 100644 modules/power/templates/systemd/system/hibernate.target.d/mesh-power.conf create mode 100644 modules/power/templates/systemd/system/hybrid-sleep.target.d/mesh-power.conf create mode 100644 modules/power/templates/systemd/system/mesh-power-after-boot.service create mode 100644 modules/power/templates/systemd/system/mesh-power-after-wake.service create mode 100644 modules/power/templates/systemd/system/mesh-power-before-shutdown.service create mode 100644 modules/power/templates/systemd/system/mesh-power-before-sleep.service create mode 100644 modules/power/templates/systemd/system/mesh-power-supply.service create mode 100644 modules/power/templates/systemd/system/sleep.target.d/mesh-power.conf create mode 100644 modules/power/templates/systemd/system/suspend-then-hibernate.target.d/mesh-power.conf create mode 100644 modules/power/templates/systemd/system/suspend.target.d/mesh-power.conf create mode 100644 modules/power/templates/udev/rules.d/90-mesh-power.rules create mode 100644 modules/powerlevel10k/files/zsh-normal create mode 100644 modules/records/cmd/records/templates_test.go create mode 100644 modules/records/files/config.json create mode 100644 modules/records/files/origin create mode 100644 modules/redis/files/redis.conf create mode 100644 modules/restic/cmd/restic-backups/templates_test.go create mode 100644 modules/restic/files/backups.conf create mode 100644 modules/restic/files/data.conf create mode 100644 modules/restic/files/retention.conf create mode 100644 modules/route-adapter/cmd/route-provider/main.go create mode 100644 modules/route-adapter/cmd/route-provider/provider.go create mode 100644 modules/route-adapter/cmd/route-provider/provider_test.go create mode 100644 modules/route-adapter/files/config.json create mode 100644 modules/route-adapter/go.mod create mode 100644 modules/route-adapter/go.sum create mode 100644 modules/route-proxy/cmd/route-provider/main.go create mode 100644 modules/route-proxy/cmd/route-provider/provider.go create mode 100644 modules/route-proxy/cmd/route-provider/provider_test.go create mode 100644 modules/route-proxy/files/acme.env create mode 100644 modules/route-proxy/files/internal-acme.env create mode 100644 modules/route-proxy/go.mod create mode 100644 modules/route-proxy/go.sum create mode 100644 modules/screen-lock/files/xinitrc-normal create mode 100644 modules/searxng/files/config.json create mode 100644 modules/searxng/files/settings.yml create mode 100644 modules/showcase/templates/showcase.env create mode 100644 modules/slack/files/display-session create mode 100644 modules/ssh-client/cmd/ssh-client-tools/templates_test.go create mode 100644 modules/ssh-client/files/config create mode 100644 modules/sshd/files/10-mesh.conf create mode 100644 modules/step-ca/files/config.json create mode 100644 modules/step-ca/files/init.env create mode 100644 modules/sudo/cmd/sudo-tools/templates_test.go create mode 100644 modules/sudo/files/10-mesh-operator create mode 100644 modules/supabase/files/analytics-env.env create mode 100644 modules/supabase/files/analytics-reconcile.exs create mode 100644 modules/supabase/files/analytics-start.sh create mode 100644 modules/supabase/files/db-init/jwt.sql create mode 100644 modules/supabase/files/db-init/logs.sql create mode 100644 modules/supabase/files/db-init/pooler.sql create mode 100644 modules/supabase/files/db-init/realtime.sql create mode 100644 modules/supabase/files/db-init/roles.sql create mode 100644 modules/supabase/files/db-init/supabase.sql create mode 100644 modules/supabase/files/db-init/webhooks.sql create mode 100644 modules/supabase/files/functions-env.env create mode 100644 modules/supabase/files/functions-main/index.ts create mode 100644 modules/supabase/files/gotrue.env create mode 100644 modules/supabase/files/kong.yml create mode 100644 modules/supabase/files/meta-env.env create mode 100644 modules/supabase/files/pooler-env.env create mode 100644 modules/supabase/files/pooler.exs create mode 100644 modules/supabase/files/postgrest.conf create mode 100644 modules/supabase/files/realtime-env.env create mode 100644 modules/supabase/files/settings.json create mode 100644 modules/supabase/files/storage-env.env create mode 100644 modules/supabase/files/studio-env.env create mode 100644 modules/supabase/files/vector.yml create mode 100644 modules/systemd-networkd/cmd/uplink-tools/templates_test.go create mode 100644 modules/systemd-networkd/files/00-mesh0.network create mode 100644 modules/time-sync/cmd/time-sync-tools/templates_test.go create mode 100644 modules/time-sync/files/50-mesh.conf create mode 100644 modules/triggerhappy/cmd/triggerhappy-tools/templates_test.go create mode 100644 modules/triggerhappy/files/service.conf create mode 100644 modules/triggerhappy/files/triggers.conf create mode 100644 modules/umami/files/server.env create mode 100644 modules/unifi/files/config.json rename modules/xdg/{config => files}/mimeapps.list (100%) rename modules/xdg/{config => files}/user-dirs.conf (100%) rename modules/xdg/{config => files}/user-dirs.dirs (87%) rename modules/xdg/{config => files}/user-dirs.locale (100%) create mode 100644 modules/xorg/cmd/xorg/templates_test.go create mode 100755 modules/xorg/files/xinitrc create mode 100644 modules/xorg/files/xresources create mode 100644 modules/xterm/files/env-terminal create mode 100644 modules/xterm/files/xresources-normal create mode 100644 modules/zfs/cmd/zfs-tools/templates_test.go create mode 100644 modules/zfs/files/scrub.conf create mode 100644 modules/zsh-autosuggestions/files/zsh-normal create mode 100644 modules/zsh-syntax-highlighting/files/zsh-last create mode 100644 modules/zsh/files/env-editor create mode 100644 modules/zsh/files/env-path-local-bin create mode 100644 modules/zsh/files/env-path-scripts create mode 100644 modules/zsh/files/env-path-scripts-bin create mode 100644 modules/zsh/files/env-visual create mode 100644 modules/zsh/files/env-xdg-config-home create mode 100644 modules/zsh/files/tools.conf create mode 100644 modules/zsh/files/zshenv create mode 100644 modules/zsh/files/zshrc create mode 100644 predefined.json diff --git a/.gitignore b/.gitignore index 3a9d9d08..9adfdd46 100644 --- a/.gitignore +++ b/.gitignore @@ -21,3 +21,6 @@ modules/mesh-knowledge/cmd/mesh-knowledge/mesh-knowledge modules/mesh-knowledge/mesh-knowledge modules/mesh-reads/mesh-reads modules/mesh-reads/cmd/mesh-reads/mesh-reads + +# The provisioner client built in place (issue 671). +modules/mesh-provisioner-client/cmd/mesh-provisioner-client/mesh-provisioner-client diff --git a/RELEASE-57.md b/RELEASE-57.md new file mode 100644 index 00000000..140ccff6 --- /dev/null +++ b/RELEASE-57.md @@ -0,0 +1,6 @@ +# Release 57: config and provisioning + +This branch carries the work of hq design 57 (ADR 0348, ADR 0349) in this repository. Each package +merges here through its own reviewed pull request; the branch reaches main as one pull request, in one +delivery group with the branch of the same name in mesh-controller, mesh-host, mesh-sdk, mesh-catalog +and hq, after a rehearsal in the lab. This file is deleted by the last commit before that pull request. diff --git a/checks/config/go.mod b/checks/config/go.mod new file mode 100644 index 00000000..c651a52b --- /dev/null +++ b/checks/config/go.mod @@ -0,0 +1,3 @@ +module git.novox.be/novox/mesh-catalog/checks/config + +go 1.22 diff --git a/checks/config/main.go b/checks/config/main.go new file mode 100644 index 00000000..d332c4e9 --- /dev/null +++ b/checks/config/main.go @@ -0,0 +1,201 @@ +// Command config refuses the settings form's names in a module that has moved to config (novox/hq ADR 0348 +// clauses 1 and 11, design 57 package K6, issue 665). +// +// "Config" replaces "settings": a manifest declares its values under `config`, a mergeable file names the keys it +// takes under `takes-config`, and a file asks for a value with `${config:}`. A mergeable file naming no keys at +// all is the settings form's last shape: it takes every key of its module's layer and declares none, so mesh-config +// could set none of them (ADR 0348 §10, ADR 0326 §3); it is found here like an old name (issue 735). The old names — `settings`, +// `takes-settings` and `${setting:}` — stay accepted by the controller as aliases until every module has +// moved, and are then refused (ADR 0348 §1). The modules not yet moved are listed in the file `unmoved` beside this +// check, one per line: the catalogue's list ADR 0348 §11 names. Every other module says no old name, in its +// manifest or in a template file its manifest names; a module on the list that says none is taken off it; and a +// line the trunk's copy (the merge base's, given with -base) does not hold is refused, so the list only shrinks; +// while the trunk holds no copy, the modules in seed are the trunk's. +// When it is empty, the controller refuses the old names itself (design 57 package R1). +// +// Prose is not judged: a comment in a file's text that says "settings" composes into the machine's file, and +// rewording it would change what a node is sent. +// +// go run . [-base ] +// +// Exits 1 on a finding. +package main + +import ( + "bufio" + "encoding/json" + "flag" + "fmt" + "os" + "path/filepath" + "regexp" + "sort" + "strings" +) + +// The settings form's names (ADR 0348 §1). A key of a JSON object is a quoted string followed by a colon; a +// string value is never followed by one. +var ( + settingRef = regexp.MustCompile(`\$\{setting:[a-z0-9][a-z0-9_.-]*\}`) + takesSettingsKey = regexp.MustCompile(`"takes-settings"\s*:`) + oldPlaceholder = "${setting:…}" + oldTakes = "takes-settings" + oldDeclaration = "settings" + wholeLayer = "taking its whole layer" + listFile = "unmoved" + listPathInCatalog = "checks/config/" + listFile +) + +// Check answers, per module that says any, the settings form's names its manifest and its template files say. +func Check(root string) (map[string][]string, error) { + manifests, err := filepath.Glob(filepath.Join(root, "modules", "*", "module.json")) + if err != nil { + return nil, err + } + sort.Strings(manifests) + out := map[string][]string{} + for _, path := range manifests { + raw, err := os.ReadFile(path) + if err != nil { + return nil, err + } + var m struct { + Module string `json:"module"` + Settings json.RawMessage `json:"settings"` + Resources []map[string]any `json:"resources"` + } + if err := json.Unmarshal(raw, &m); err != nil { + return nil, fmt.Errorf("%s: %w", path, err) + } + var said []string + if len(m.Settings) > 0 && string(m.Settings) != "null" { + said = append(said, oldDeclaration) + } + if takesSettingsKey.Match(raw) { + said = append(said, oldTakes) + } + if settingRef.Match(raw) { + said = append(said, oldPlaceholder) + } + for _, r := range m.Resources { + // A mergeable file naming no keys takes every key of the layer (ADR 0326 §3), declaring none of them. + if how, _ := r["merge"].(string); how != "" { + _, oldList := r["takes-settings"] + _, newList := r["takes-config"] + if !oldList && !newList { + said = append(said, fmt.Sprintf("%v %s", r["id"], wholeLayer)) + } + } + name, _ := r["template"].(string) + if name == "" { + continue + } + text, err := os.ReadFile(filepath.Join(filepath.Dir(path), filepath.FromSlash(name))) + if err != nil { + return nil, fmt.Errorf("%s names the template %s: %w", m.Module, name, err) + } + if settingRef.Match(text) { + said = append(said, oldPlaceholder+" in "+name) + } + } + if len(said) > 0 { + sort.Strings(said) + out[m.Module] = said + } + } + return out, nil +} + +// seed is the list as it was first written, standing for the trunk's copy until the trunk holds one. +// The modules taking a whole layer were found after the list was first written and before the trunk held a copy +// (issue 735), so they stand in it too. +var seed = map[string]bool{ + "betterbird": true, "fail2ban": true, "influxdb": true, "keycloak": true, "mailu": true, "searxng": true, + "baserow": true, "confluence": true, "gitea": true, "gitlab": true, "grafana": true, "home-assistant": true, + "icecast": true, "jira": true, "letta": true, "matrix": true, "nodered": true, "supabase": true, "unifi": true, +} + +// Unmoved is the modules a list names, one per line; a line starting with # and a blank line are not modules. +func Unmoved(path string) (map[string]bool, error) { + f, err := os.Open(path) + if err != nil { + return nil, err + } + defer f.Close() + out := map[string]bool{} + s := bufio.NewScanner(f) + for s.Scan() { + line := strings.TrimSpace(s.Text()) + if line == "" || strings.HasPrefix(line, "#") { + continue + } + out[line] = true + } + return out, s.Err() +} + +// Judge answers the modules that say an old name and are not on the list (refused), the modules on the list that +// say none (stale), and the modules on the list that the trunk's list does not hold (added). +func Judge(found map[string][]string, unmoved, base map[string]bool) (refused, stale, added []string) { + for m := range found { + if !unmoved[m] { + refused = append(refused, m) + } + } + for m := range unmoved { + if _, says := found[m]; !says { + stale = append(stale, m) + } + if !base[m] { + added = append(added, m) + } + } + sort.Strings(refused) + sort.Strings(stale) + sort.Strings(added) + return refused, stale, added +} + +func main() { + basePath := flag.String("base", "", "the trunk's copy of unmoved (the merge base's); seed when absent") + flag.Parse() + root := "." + if flag.NArg() > 0 { + root = flag.Arg(0) + } + found, err := Check(root) + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + unmoved, err := Unmoved(listFile) + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + base := seed + if *basePath != "" { + if base, err = Unmoved(*basePath); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + } + refused, stale, added := Judge(found, unmoved, base) + for _, m := range refused { + fmt.Printf("%s says %s, the settings form's names, and is not on %s: write config, takes-config and "+ + "${config:} throughout, and name the keys each mergeable file takes (novox/hq ADR 0348 §1, §11)\n", m, strings.Join(found[m], ", "), listPathInCatalog) + } + for _, m := range stale { + fmt.Printf("%s names %s, which says no old name: take it off the list\n", listPathInCatalog, m) + } + for _, m := range added { + fmt.Printf("%s gains %s, which the trunk's copy does not hold: the list only shrinks — move the module to "+ + "config instead (novox/hq ADR 0348 §11)\n", listPathInCatalog, m) + } + if len(refused) > 0 || len(stale) > 0 || len(added) > 0 { + os.Exit(1) + } + fmt.Printf("config: no moved module says settings, takes-settings or ${setting:}, or takes a whole layer "+ + "(%d not yet moved)\n", + len(unmoved)) +} diff --git a/checks/config/main_test.go b/checks/config/main_test.go new file mode 100644 index 00000000..d650a16b --- /dev/null +++ b/checks/config/main_test.go @@ -0,0 +1,131 @@ +package main + +import ( + "os" + "path/filepath" + "reflect" + "strings" + "testing" +) + +func module(t *testing.T, root, name, body string, files map[string]string) { + t.Helper() + dir := filepath.Join(root, "modules", name) + if err := os.MkdirAll(dir, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "module.json"), []byte(body), 0o644); err != nil { + t.Fatal(err) + } + for path, text := range files { + full := filepath.Join(dir, filepath.FromSlash(path)) + if err := os.MkdirAll(filepath.Dir(full), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(full, []byte(text), 0o644); err != nil { + t.Fatal(err) + } + } +} + +// Each of the settings form's names is found where it stands: the declaration, a mergeable file's keys, a +// placeholder in the manifest, and a placeholder in a template file the manifest names (novox/hq ADR 0348 §1). +func TestEveryOldNameIsFound(t *testing.T) { + root := t.TempDir() + module(t, root, "declares", `{"module":"declares","settings":{"size":{"kind":"preference","default":1,"why":"w"}},`+ + `"resources":[{"id":"f","type":"file","content":"${setting:size}"}]}`, nil) + module(t, root, "takes", `{"module":"takes","resources":[{"id":"f","type":"file","merge":"json",`+ + `"takes-settings":["a"],"content":"{}"}]}`, nil) + module(t, root, "asks", `{"module":"asks","serves":{"x":{"d":"${setting:domain}"}}}`, nil) + module(t, root, "templated", `{"module":"templated","config":{"size":{"type":"number","description":"d",`+ + `"default":1}},"resources":[{"id":"f","type":"file","template":"files/conf"}]}`, + map[string]string{"files/conf": "size = ${setting:size}\n"}) + module(t, root, "moved", `{"module":"moved","config":{"size":{"type":"number","description":"d","default":1}},`+ + `"resources":[{"id":"f","type":"file","template":"files/conf"},{"id":"g","type":"file","merge":"json",`+ + `"takes-config":[],"content":"{\"settings\": 1}"}]}`, + map[string]string{"files/conf": "size = ${config:size}\n# change the settings, never this file\n"}) + module(t, root, "none", `{"module":"none","resources":[{"id":"f","type":"file","content":"no settings here"}]}`, nil) + got, err := Check(root) + if err != nil { + t.Fatal(err) + } + want := map[string][]string{ + "asks": {"${setting:…}"}, + "declares": {"${setting:…}", "settings"}, + "takes": {"takes-settings"}, + "templated": {"${setting:…} in files/conf"}, + } + if !reflect.DeepEqual(got, want) { + t.Fatalf("found %v, want %v", got, want) + } +} + +// A module not on the list says no old name; one on the list that says none is taken off it; a line the trunk's +// list does not hold is refused, so the list only shrinks until it is empty (novox/hq ADR 0348 §11). +func TestTheListOfModulesNotYetMovedOnlyShrinks(t *testing.T) { + found := map[string][]string{"old": {"settings"}, "forgotten": {"takes-settings"}} + unmoved := map[string]bool{"old": true, "done": true, "new": true} + base := map[string]bool{"old": true, "done": true, "forgotten": true} + refused, stale, added := Judge(found, unmoved, base) + if !reflect.DeepEqual(refused, []string{"forgotten"}) || !reflect.DeepEqual(stale, []string{"done", "new"}) || + !reflect.DeepEqual(added, []string{"new"}) { + t.Fatalf("refused %v, stale %v, added %v", refused, stale, added) + } +} + +func TestTheListIsReadWithoutItsComments(t *testing.T) { + path := filepath.Join(t.TempDir(), "unmoved") + if err := os.WriteFile(path, []byte("# why\n\nmailu\n searxng \n"), 0o644); err != nil { + t.Fatal(err) + } + got, err := Unmoved(path) + if err != nil || !reflect.DeepEqual(got, map[string]bool{"mailu": true, "searxng": true}) { + t.Fatalf("%v %v", got, err) + } +} + +// The catalogue itself: every moved module says config alone, and the list names exactly the modules that +// still say settings. +func TestTheCatalogueSaysConfigInEveryMovedModule(t *testing.T) { + found, err := Check(filepath.Join("..", "..")) + if err != nil { + t.Fatal(err) + } + unmoved, err := Unmoved("unmoved") + if err != nil { + t.Fatal(err) + } + refused, stale, added := Judge(found, unmoved, seed) + for _, m := range refused { + t.Errorf("%s is not on checks/config/unmoved and says %s", m, strings.Join(found[m], ", ")) + } + for _, m := range stale { + t.Errorf("checks/config/unmoved names %s, which says no old name", m) + } + for _, m := range added { + t.Errorf("checks/config/unmoved gains %s, which its first copy did not hold", m) + } +} + +// A mergeable file that names no keys takes every key of its module's layer (novox/hq ADR 0326 §3): the settings +// form's last shape, which declares none of what it takes, so mesh-config could set none of it (ADR 0348 §10). It +// is found like an old name, in a module that says no other (issue 735); one naming its keys, an empty list +// included, is not. +func TestAMergeableFileTakingTheWholeLayerIsFound(t *testing.T) { + root := t.TempDir() + module(t, root, "whole", `{"module":"whole","resources":[{"id":"runtime-config","type":"file","merge":"json",`+ + `"template":"files/config.json"}]}`, map[string]string{"files/config.json": "{}\n"}) + module(t, root, "names", `{"module":"names","config":{"url":{"type":"string","description":"d","default":""}},`+ + `"resources":[{"id":"runtime-config","type":"file","merge":"json","takes-config":["url"],"content":"{}"}]}`, nil) + module(t, root, "none", `{"module":"none","resources":[{"id":"c","type":"file","merge":"json",`+ + `"takes-config":[],"content":"{}"}]}`, nil) + module(t, root, "plain", `{"module":"plain","resources":[{"id":"c","type":"file","content":"{}"}]}`, nil) + got, err := Check(root) + if err != nil { + t.Fatal(err) + } + want := map[string][]string{"whole": {"runtime-config taking its whole layer"}} + if !reflect.DeepEqual(got, want) { + t.Fatalf("found %v, want %v", got, want) + } +} diff --git a/checks/config/unmoved b/checks/config/unmoved new file mode 100644 index 00000000..c8ca7e28 --- /dev/null +++ b/checks/config/unmoved @@ -0,0 +1,5 @@ +# The modules still written in the settings form (`settings`, `takes-settings`, `${setting:}`, or a mergeable +# file naming no keys, which takes its whole layer), one per line: the catalogue's list of modules not yet moved to +# config (novox/hq ADR 0348 §11). Every other module says config alone. This list only shrinks: a module that says no +# old name is taken off, and a line the trunk's copy does not hold is refused. When it is empty, the controller +# refuses the old names (design 57 package R1). diff --git a/checks/predefined/go.mod b/checks/predefined/go.mod new file mode 100644 index 00000000..da843dfb --- /dev/null +++ b/checks/predefined/go.mod @@ -0,0 +1,3 @@ +module git.novox.be/novox/mesh-catalog/checks/predefined + +go 1.22 diff --git a/checks/predefined/main.go b/checks/predefined/main.go new file mode 100644 index 00000000..9b6d4854 --- /dev/null +++ b/checks/predefined/main.go @@ -0,0 +1,153 @@ +// Command predefined refuses a change to predefined.json that removes a key a module still names (novox/hq ADR +// 0348 clause 7, design 57 package K2, issue 661). +// +// Mesh config and node config hold values only for the keys on the catalogue's list of predefined keys, at the +// root of the repository; a module's config value may fall back to one, `"predefined": ""`. A key removed +// while a manifest still names it would leave that module falling back to nothing: the controller refuses its +// build then, at the next merge that touches it. So the removal is refused here, in the pull request that makes +// it, naming each module that names the key. +// +// **What it cannot know.** Whether a key still holds a value at the mesh or a node is mesh-config's to say (design +// 57 package K3): it holds the values, and the catalogue's check reads no store. Until mesh-config refuses that +// itself, removing a key nobody names is let through here, and said. +// +// The list it compares with is the trunk's at the merge base, given with -base; without one, nothing was removed. +// The controller the mesh runs reads the list itself (type, validation, every manifest against it) in the +// module check; this check reads only its keys. +// +// go run . [-base ] +// +// Exits 1 on a finding. +package main + +import ( + "encoding/json" + "errors" + "flag" + "fmt" + "io/fs" + "os" + "path/filepath" + "sort" + "strings" +) + +// File is the list's file at the root of the catalogue. +const File = "predefined.json" + +// Keys is the keys of a list, read as a JSON object; an absent file has none. +func Keys(path string) (map[string]bool, error) { + raw, err := os.ReadFile(path) + if errors.Is(err, fs.ErrNotExist) { + return map[string]bool{}, nil + } + if err != nil { + return nil, err + } + var keys map[string]json.RawMessage + if err := json.Unmarshal(raw, &keys); err != nil { + return nil, fmt.Errorf("%s is not a JSON object of keys: %w", path, err) + } + out := map[string]bool{} + for k := range keys { + out[k] = true + } + return out, nil +} + +// Named is, per predefined key, the modules whose manifests name it: under `config`, or its alias `settings`. +func Named(root string) (map[string][]string, error) { + manifests, err := filepath.Glob(filepath.Join(root, "modules", "*", "module.json")) + if err != nil { + return nil, err + } + sort.Strings(manifests) + named := map[string][]string{} + for _, path := range manifests { + raw, err := os.ReadFile(path) + if err != nil { + return nil, err + } + var m struct { + Module string `json:"module"` + Config map[string]json.RawMessage `json:"config"` + Settings map[string]json.RawMessage `json:"settings"` + } + if err := json.Unmarshal(raw, &m); err != nil { + return nil, fmt.Errorf("%s: %w", path, err) + } + for _, declared := range []map[string]json.RawMessage{m.Config, m.Settings} { + for _, d := range declared { + var value struct { + Predefined string `json:"predefined"` + } + if json.Unmarshal(d, &value) == nil && value.Predefined != "" { + named[value.Predefined] = append(named[value.Predefined], m.Module) + } + } + } + } + return named, nil +} + +// Removed is each key the base list holds and the tree's does not, with the modules still naming it; a removed +// key nobody names is answered with none. +func Removed(base, tree map[string]bool, named map[string][]string) map[string][]string { + out := map[string][]string{} + for k := range base { + if !tree[k] { + out[k] = named[k] + } + } + return out +} + +func main() { + basePath := flag.String("base", "", "the trunk's predefined.json at the merge base") + flag.Parse() + if flag.NArg() != 1 { + fmt.Fprintln(os.Stderr, "usage: predefined [-base ] ") + os.Exit(2) + } + root := flag.Arg(0) + tree, err := Keys(filepath.Join(root, File)) + if err != nil { + fmt.Println(err) + os.Exit(1) + } + if *basePath == "" { + fmt.Printf("%s: %d key(s); no trunk list given, so none was removed\n", File, len(tree)) + return + } + base, err := Keys(*basePath) + if err != nil { + fmt.Println(err) + os.Exit(1) + } + named, err := Named(root) + if err != nil { + fmt.Println(err) + os.Exit(1) + } + removed := Removed(base, tree, named) + keys := make([]string, 0, len(removed)) + for k := range removed { + keys = append(keys, k) + } + sort.Strings(keys) + failed := false + for _, k := range keys { + if modules := removed[k]; len(modules) > 0 { + failed = true + fmt.Printf("%s: the key %q is removed and still named by %s: move each off it first (novox/hq ADR 0348 "+ + "clause 7)\n", File, k, strings.Join(modules, ", ")) + } else { + fmt.Printf("%s: the key %q is removed and no module names it; whether the mesh or a node still holds a "+ + "value for it is mesh-config's to refuse, not this check's\n", File, k) + } + } + if failed { + os.Exit(1) + } + fmt.Printf("%s: %d key(s), none removed that a module names\n", File, len(tree)) +} diff --git a/checks/predefined/main_test.go b/checks/predefined/main_test.go new file mode 100644 index 00000000..3f9c1325 --- /dev/null +++ b/checks/predefined/main_test.go @@ -0,0 +1,100 @@ +package main + +import ( + "os" + "os/exec" + "path/filepath" + "strings" + "testing" +) + +// A catalogue whose dunst falls back to font-size, and the trunk's list holding font-size and +// agent-managed-settings. +func aCatalogue(t *testing.T, list string) (root, base string) { + t.Helper() + root = t.TempDir() + write := func(path, text string) { + t.Helper() + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, []byte(text), 0o644); err != nil { + t.Fatal(err) + } + } + write(filepath.Join(root, "modules", "dunst", "module.json"), `{"module": "dunst", "config": {"font-size": + {"type": "number", "description": "points", "default": 10, "predefined": "font-size"}}}`) + write(filepath.Join(root, "modules", "claude-code", "module.json"), `{"module": "claude-code", "settings": { + "managed_settings": {"kind": "preference", "default": "{}", "why": "none"}}}`) + write(filepath.Join(root, "modules", "plain", "module.json"), `{"module": "plain"}`) + if list != "" { + write(filepath.Join(root, File), list) + } + base = filepath.Join(t.TempDir(), "base.json") + write(base, `{"font-size": {"type": "number"}, "agent-managed-settings": {"type": "object"}}`) + return root, base +} + +func run(t *testing.T, args ...string) (string, error) { + t.Helper() + out, err := exec.Command("go", append([]string{"run", "."}, args...)...).CombinedOutput() + return string(out), err +} + +func TestARemovalOfAKeyAModuleStillNamesIsRefused(t *testing.T) { + // The planted removal: font-size goes while dunst still falls back to it. + root, base := aCatalogue(t, `{"agent-managed-settings": {"type": "object"}}`) + out, err := run(t, "-base", base, root) + if err == nil || !strings.Contains(out, `the key "font-size" is removed and still named by dunst`) { + t.Fatalf("a removal of a named key passed:\n%s", out) + } +} + +func TestARemovalNobodyNamesPassesAndSaysWhatItCannotKnow(t *testing.T) { + root, base := aCatalogue(t, `{"font-size": {"type": "number"}}`) + out, err := run(t, "-base", base, root) + if err != nil || !strings.Contains(out, `"agent-managed-settings" is removed and no module names it`) || + !strings.Contains(out, "mesh-config's to refuse") { + t.Fatalf("a removal nobody names: %v\n%s", err, out) + } +} + +func TestAListUnchangedOrGrownPasses(t *testing.T) { + for _, list := range []string{ + `{"font-size": {"type": "number"}, "agent-managed-settings": {"type": "object"}}`, + `{"font-size": {"type": "number"}, "agent-managed-settings": {"type": "object"}, "accent": {"type": "string"}}`, + } { + root, base := aCatalogue(t, list) + if out, err := run(t, "-base", base, root); err != nil || !strings.Contains(out, "none removed") { + t.Fatalf("%s: %v\n%s", list, err, out) + } + } + // No trunk list given: nothing was removed. + root, _ := aCatalogue(t, `{}`) + if out, err := run(t, root); err != nil || !strings.Contains(out, "none was removed") { + t.Fatalf("without a base: %v\n%s", err, out) + } +} + +func TestTheFileRemovedWholeIsARemovalOfEveryKey(t *testing.T) { + root, base := aCatalogue(t, "") + out, err := run(t, "-base", base, root) + if err == nil || !strings.Contains(out, `"font-size" is removed and still named by dunst`) { + t.Fatalf("the list removed whole passed:\n%s", out) + } +} + +func TestNamedReadsBothForms(t *testing.T) { + root, _ := aCatalogue(t, "") + if err := os.WriteFile(filepath.Join(root, "modules", "plain", "module.json"), []byte(`{"module": "plain", + "settings": {"size": {"predefined": "font-size"}}}`), 0o644); err != nil { + t.Fatal(err) + } + named, err := Named(root) + if err != nil { + t.Fatal(err) + } + if got := strings.Join(named["font-size"], ","); got != "dunst,plain" { + t.Fatalf("font-size named by %s", got) + } +} diff --git a/checks/secrets/main.go b/checks/secrets/main.go index 4a317967..a73c7059 100644 --- a/checks/secrets/main.go +++ b/checks/secrets/main.go @@ -7,7 +7,8 @@ // until somebody removed it by hand. The controller refuses a setting of a key the file keeps as `protected`, // naming the layer and the key, never the value. This check holds every such file to that: each top-level key // whose value holds `${secret:…}`, anywhere below it, is either listed in the file's `protected`, or left out -// of the file's `takes-settings` (novox/hq ADR 0326), so that no layer reaches it. +// of the file's `takes-settings` (novox/hq ADR 0326), so that no layer reaches it. A file whose text is a template +// in files/ is judged by the template's text. // // A secret nested below a key the file takes for its other values (`server.secret_key` below `server`) cannot // be protected alone today: the controller protects top-level keys. Such a file is named in the file `known` @@ -64,6 +65,15 @@ func Check(root string) ([]Finding, error) { } id, _ := r["id"].(string) content, _ := r["content"].(string) + // A file whose text is a template in its module's files/ (novox/hq ADR 0349 §2) is judged by that text, + // exactly as it would be inline. + if name, _ := r["template"].(string); name != "" { + text, err := os.ReadFile(filepath.Join(filepath.Dir(path), filepath.FromSlash(name))) + if err != nil { + return nil, fmt.Errorf("%s: %s names the template %s: %w", m.Module, id, name, err) + } + content = string(text) + } var doc map[string]any if strings.TrimSpace(content) == "" { continue diff --git a/checks/secrets/main_test.go b/checks/secrets/main_test.go index 48b0bd23..efa9750a 100644 --- a/checks/secrets/main_test.go +++ b/checks/secrets/main_test.go @@ -51,6 +51,35 @@ func TestASecretASettingCouldReplaceIsRefused(t *testing.T) { } } +// A mergeable file whose text is a template in files/ (novox/hq ADR 0349 §2) is judged by the template's text, +// exactly as the same text inline; a template the module's directory lacks is an error, never a silent pass. +func TestAMergeableFilesTemplateIsJudgedAsItsText(t *testing.T) { + root := t.TempDir() + manifest(t, root, "a", `{"id":"cfg","type":"file","merge":"json","template":"files/cfg.json"}`) + manifest(t, root, "c", `{"id":"cfg","type":"file","merge":"json","protected":["password"],"template":"files/cfg.json"}`) + for _, m := range []string{"a", "c"} { + dir := filepath.Join(root, "modules", m, "files") + if err := os.MkdirAll(dir, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "cfg.json"), []byte(`{"password": "${secret:admin}"}`), 0o644); err != nil { + t.Fatal(err) + } + } + found, err := Check(root) + if err != nil { + t.Fatal(err) + } + if len(found) != 1 || found[0].String() != "a cfg password" { + t.Fatalf("found %v, want [a cfg password]", found) + } + + manifest(t, root, "g", `{"id":"cfg","type":"file","merge":"json","template":"files/gone.json"}`) + if _, err := Check(root); err == nil { + t.Fatal("a template the module's directory lacks passed the check") + } +} + func TestTheKnownListOnlyShrinks(t *testing.T) { found := []Finding{{"b", "cfg", "server.secret_key"}, {"a", "cfg", "password"}} refused, stale := Judge(found, map[string]bool{"b cfg server.secret_key": true, "gone cfg token": true}) diff --git a/checks/templates/go.mod b/checks/templates/go.mod new file mode 100644 index 00000000..f6c19294 --- /dev/null +++ b/checks/templates/go.mod @@ -0,0 +1,3 @@ +module git.novox.be/novox/mesh-catalog/checks/templates + +go 1.22 diff --git a/checks/templates/inline b/checks/templates/inline new file mode 100644 index 00000000..9d845aee --- /dev/null +++ b/checks/templates/inline @@ -0,0 +1,9 @@ +# The modules that still keep a file's text inline in module.json (`content`), one per line: the catalogue's list of +# modules whose file text has not yet moved to a template in files/ (novox/hq ADR 0349 §2, design 57), or in the +# templates directory its manifest names (`templates`, issue 731). Every other module's file resources name their +# template. This list only shrinks: a module that keeps no file text inline is taken off, and a line the trunk's copy +# does not hold is refused. When it is empty, no manifest in the catalogue carries file text (design 57 package T5, +# issue 658). +# +# A window-manager addition (`contributions`, such as files/i3/*.conf) is not a file resource and takes no template; +# it moves with the collections (design 57 package L2, issue 681) and is not judged here. diff --git a/checks/templates/main.go b/checks/templates/main.go new file mode 100644 index 00000000..792d9e01 --- /dev/null +++ b/checks/templates/main.go @@ -0,0 +1,178 @@ +// Command templates refuses a file's text kept inline in a manifest, outside the modules still to move (novox/hq +// ADR 0349 §2, design 57 package T5, issue 658). +// +// A file resource names its text as a template in its module's templates directory (`"template": "/"`; +// the directory is the manifest's `"templates"`, files/ when it says none), which the module's build packs into an +// archive; the controller reads each template back into the file's content, so the +// mesh holds and composes what it would for the same text inline. A module whose file text has not yet moved is +// listed in the file `inline` beside this check, one per line. Every other module's file resources say no +// `content`; a module on the list that keeps none inline is taken off it; and a line the trunk's copy (the merge +// base's, given with -base) does not hold is refused, so the list only shrinks; while the trunk holds no copy, the +// modules in seed are the trunk's. When it is empty, no manifest in the catalogue carries file text. +// +// Only file resources are judged: a window-manager addition (`contributions`) takes no template and moves with the +// collections (design 57 package L2, issue 681). +// +// go run . [-base ] +// +// Every refusal names the refused module's own templates directory. Exits 1 on a finding. +package main + +import ( + "bufio" + "encoding/json" + "flag" + "fmt" + "os" + "path/filepath" + "sort" + "strings" +) + +const ( + listFile = "inline" + listPathInCatalog = "checks/templates/" + listFile +) + +// Check answers, per module that keeps any, the ids of its file resources that say their text inline. +func Check(root string) (map[string][]string, error) { + manifests, err := filepath.Glob(filepath.Join(root, "modules", "*", "module.json")) + if err != nil { + return nil, err + } + sort.Strings(manifests) + out := map[string][]string{} + for _, path := range manifests { + raw, err := os.ReadFile(path) + if err != nil { + return nil, err + } + var m struct { + Module string `json:"module"` + Resources []map[string]any `json:"resources"` + } + if err := json.Unmarshal(raw, &m); err != nil { + return nil, fmt.Errorf("%s: %w", path, err) + } + for _, r := range m.Resources { + if _, inline := r["content"]; inline && r["type"] == "file" { + out[m.Module] = append(out[m.Module], fmt.Sprint(r["id"])) + } + } + } + return out, nil +} + +// TemplatesDir answers the directory a module keeps its templates in, with a trailing slash: its manifest's +// `templates`, files/ when the manifest says none or cannot be read here. +func TemplatesDir(root, module string) string { + dir := "files/" + raw, err := os.ReadFile(filepath.Join(root, "modules", module, "module.json")) + if err != nil { + return dir + } + var m struct { + Templates string `json:"templates"` + } + if json.Unmarshal(raw, &m) == nil && m.Templates != "" { + dir = strings.TrimSuffix(m.Templates, "/") + "/" + } + return dir +} + +// Refusal is the finding for a module that keeps file text inline and is not on the list. +func Refusal(module string, ids []string, dir string) string { + return fmt.Sprintf("%s keeps the text of %s inline and is not on %s: keep each file's text in %s and name it as "+ + "the file's template, packed by the build's archive from %s (novox/hq ADR 0349 §2)", + module, strings.Join(ids, ", "), listPathInCatalog, dir, dir) +} + +// Gained is the finding for a module the list gains against the trunk's copy. +func Gained(module, dir string) string { + return fmt.Sprintf("%s gains %s, which the trunk's copy does not hold: the list only shrinks — move its file text "+ + "to %s instead (novox/hq ADR 0349 §2)", listPathInCatalog, module, dir) +} + +// seed is the list as it was first written, standing for the trunk's copy until the trunk holds one. +var seed = map[string]bool{"asus-zephyrus-g14": true, "power": true, "showcase": true} + +// Listed is the modules a list names, one per line; a line starting with # and a blank line are not modules. +func Listed(path string) (map[string]bool, error) { + f, err := os.Open(path) + if err != nil { + return nil, err + } + defer f.Close() + out := map[string]bool{} + s := bufio.NewScanner(f) + for s.Scan() { + line := strings.TrimSpace(s.Text()) + if line == "" || strings.HasPrefix(line, "#") { + continue + } + out[line] = true + } + return out, s.Err() +} + +// Judge answers the modules that keep file text inline and are not on the list (refused), the modules on the list +// that keep none (stale), and the modules on the list that the trunk's list does not hold (added). +func Judge(found map[string][]string, listed, base map[string]bool) (refused, stale, added []string) { + for m := range found { + if !listed[m] { + refused = append(refused, m) + } + } + for m := range listed { + if _, keeps := found[m]; !keeps { + stale = append(stale, m) + } + if !base[m] { + added = append(added, m) + } + } + sort.Strings(refused) + sort.Strings(stale) + sort.Strings(added) + return refused, stale, added +} + +func main() { + basePath := flag.String("base", "", "the trunk's copy of inline (the merge base's); seed when absent") + flag.Parse() + root := "." + if flag.NArg() > 0 { + root = flag.Arg(0) + } + found, err := Check(root) + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + listed, err := Listed(listFile) + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + base := seed + if *basePath != "" { + if base, err = Listed(*basePath); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + } + refused, stale, added := Judge(found, listed, base) + for _, m := range refused { + fmt.Println(Refusal(m, found[m], TemplatesDir(root, m))) + } + for _, m := range stale { + fmt.Printf("%s names %s, which keeps no file text inline: take it off the list\n", listPathInCatalog, m) + } + for _, m := range added { + fmt.Println(Gained(m, TemplatesDir(root, m))) + } + if len(refused) > 0 || len(stale) > 0 || len(added) > 0 { + os.Exit(1) + } + fmt.Printf("templates: no module keeps a file's text inline but the %d listed\n", len(listed)) +} diff --git a/checks/templates/main_test.go b/checks/templates/main_test.go new file mode 100644 index 00000000..c7d0d5b3 --- /dev/null +++ b/checks/templates/main_test.go @@ -0,0 +1,102 @@ +package main + +import ( + "os" + "path/filepath" + "reflect" + "strings" + "testing" +) + +func module(t *testing.T, root, name, body string) { + t.Helper() + dir := filepath.Join(root, "modules", name) + if err := os.MkdirAll(dir, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "module.json"), []byte(body), 0o644); err != nil { + t.Fatal(err) + } +} + +// A file resource that says its text inline is found, by module and resource; one naming its template, a +// resource that is not a file and a window-manager addition are not (novox/hq ADR 0349 §2). +func TestEveryInlineFileTextIsFound(t *testing.T) { + root := t.TempDir() + module(t, root, "inline", `{"module":"inline","resources":[{"id":"a","type":"file","content":"x"},`+ + `{"id":"b","type":"file","template":"files/b"},{"id":"c","type":"file","content":""}]}`) + module(t, root, "templated", `{"module":"templated","resources":[{"id":"a","type":"file","template":"files/a"},`+ + `{"id":"d","type":"directory","path":"/x"}],"contributions":{"i3":[{"content":"bindsym x"}]}}`) + got, err := Check(root) + if err != nil { + t.Fatal(err) + } + want := map[string][]string{"inline": {"a", "c"}} + if !reflect.DeepEqual(got, want) { + t.Fatalf("found %v, want %v", got, want) + } +} + +// A module not on the list keeps no file text inline; one on the list that keeps none is taken off it; a line the +// trunk's list does not hold is refused, so the list only shrinks until it is empty. +func TestTheListOfModulesKeepingTextInlineOnlyShrinks(t *testing.T) { + found := map[string][]string{"old": {"a"}, "forgotten": {"b"}} + inline := map[string]bool{"old": true, "done": true, "new": true} + base := map[string]bool{"old": true, "done": true, "forgotten": true} + refused, stale, added := Judge(found, inline, base) + if !reflect.DeepEqual(refused, []string{"forgotten"}) || !reflect.DeepEqual(stale, []string{"done", "new"}) || + !reflect.DeepEqual(added, []string{"new"}) { + t.Fatalf("refused %v, stale %v, added %v", refused, stale, added) + } +} + +func TestTheListIsReadWithoutItsComments(t *testing.T) { + path := filepath.Join(t.TempDir(), "inline") + if err := os.WriteFile(path, []byte("# why\n\npower\n showcase \n"), 0o644); err != nil { + t.Fatal(err) + } + got, err := Listed(path) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(got, map[string]bool{"power": true, "showcase": true}) { + t.Fatalf("read %v", got) + } +} + +// The catalogue itself: no module keeps a file's text inline but those the list names, and the list names none +// that keeps it no longer (design 57 package T5, issue 658: the ten modules without a build moved). +func TestTheCatalogueKeepsFileTextInlineOnlyWhereListed(t *testing.T) { + found, err := Check("../..") + if err != nil { + t.Fatal(err) + } + inline, err := Listed(listFile) + if err != nil { + t.Fatal(err) + } + refused, stale, _ := Judge(found, inline, inline) + if len(refused) > 0 || len(stale) > 0 { + t.Fatalf("keeping file text inline and not listed: %v; listed and keeping none: %v", refused, stale) + } +} + +// A refusal names the refused module's own templates directory, not files/ when the module names another (issue 809). +func TestARefusalNamesTheModulesOwnTemplatesDirectory(t *testing.T) { + root := t.TempDir() + module(t, root, "own", `{"module":"own","templates":"t/","resources":[{"id":"a","type":"file","content":"x"}]}`) + module(t, root, "plain", `{"module":"plain","resources":[{"id":"a","type":"file","content":"x"}]}`) + module(t, root, "bare", `{"module":"bare","templates":"templates","resources":[]}`) + for module, dir := range map[string]string{"own": "t/", "plain": "files/", "bare": "templates/"} { + if got := TemplatesDir(root, module); got != dir { + t.Fatalf("%s: directory %q, want %q", module, got, dir) + } + } + msg := Refusal("own", []string{"a"}, TemplatesDir(root, "own")) + if !strings.Contains(msg, "t/") || strings.Contains(msg, "files/") { + t.Fatalf("refusal %q", msg) + } + if msg := Gained("own", TemplatesDir(root, "own")); !strings.Contains(msg, "t/") || strings.Contains(msg, "files/") { + t.Fatalf("gained %q", msg) + } +} diff --git a/health-undeclared b/health-undeclared index 29ac652b..3fbd022b 100644 --- a/health-undeclared +++ b/health-undeclared @@ -1,4 +1,4 @@ -62 +61 The long-running resources of this catalogue that do not say how they are ready (`health`, novox/hq ADR 0240 rule 8), as `module check` counts them. It may only go down: merge-check.sh fails a change that raises it, and one that lowers it writes the new number on the first line. From 2026-11-18, or once it is 0, a long-running diff --git a/merge-check.sh b/merge-check.sh index 549c22b0..0e4e4cdc 100755 --- a/merge-check.sh +++ b/merge-check.sh @@ -33,7 +33,18 @@ # 6. the secrets (novox/hq issue 342): no mergeable file fills a key with a vault secret that a setting could # replace — the key is `protected`, or left out of the file's `takes-settings`. checks/secrets/known names # the nested ones the controller cannot protect alone yet, and only shrinks. -# 7. the mail-store seat (novox/hq ADR 0378 §3, issue 800): no manifest but modules/mesh-mail claims it. +# 7. the list of predefined keys (novox/hq ADR 0348 clause 7): a change to predefined.json removes no key a +# module still names, judged against the trunk's copy at the merge base. Whether a removed key still holds a +# value at the mesh or a node is mesh-config's to refuse; this check reads no store. The list itself and +# every manifest against it are judged by the controller's module check in step 1. +# 8. the config form (novox/hq ADR 0348 clauses 1 and 11): no module says `settings`, `takes-settings` or +# `${setting:}`, in its manifest or a template it names, or has a mergeable file naming no keys (which +# takes its whole layer), unless checks/config/unmoved lists it as not yet moved; that list only shrinks, +# judged against the trunk's copy at the merge base. +# 9. the file text (novox/hq ADR 0349 §2): no module keeps a file's text inline in its manifest (`content`) +# unless checks/templates/inline lists it as not yet moved; each names its template in files/. That list +# only shrinks, judged against the trunk's copy at the merge base. +# 10. the mail-store seat (novox/hq ADR 0378 §3, issue 800): no manifest but modules/mesh-mail claims it. set -eu (cd checks/words && go test -count=1 . && go run . ../..) @@ -54,6 +65,42 @@ else echo "NOT COMPARED WITH THE TRUNK: no merge base with $secrets_ref; checks/secrets/known is held to the list it was first written with" fi (cd checks/secrets && go test -count=1 . && if [ -n "$secrets_base" ]; then go run . -base "$secrets_base" ../..; else go run . ../..; fi) +# The list of predefined keys (novox/hq ADR 0348 clause 7): no key removed that a module still names. The trunk's +# list at the merge base is what it is judged against; a trunk without the file had none. +predefined_base="" +if merge_base=$(git merge-base HEAD "origin/${MESH_CHECK_BASE:-main}" 2>/dev/null); then + predefined_base=$(mktemp) + if ! git show "$merge_base:predefined.json" >"$predefined_base" 2>/dev/null; then + echo '{}' >"$predefined_base" + fi +else + echo "NOT COMPARED WITH THE TRUNK: no merge base with origin/${MESH_CHECK_BASE:-main}; no removal from predefined.json can be seen" +fi +(cd checks/predefined && go test -count=1 . && if [ -n "$predefined_base" ]; then go run . -base "$predefined_base" ../..; else go run . ../..; fi) +# The config form (novox/hq ADR 0348 §1, §11): a moved module says no old name, and the list of modules not yet moved +# only shrinks, judged against the trunk's copy at the merge base; a trunk without one is the list's first copy. +unmoved_base="" +if merge_base=$(git merge-base HEAD "origin/${MESH_CHECK_BASE:-main}" 2>/dev/null); then + if git cat-file -e "$merge_base:checks/config/unmoved" 2>/dev/null; then + unmoved_base=$(mktemp) + git show "$merge_base:checks/config/unmoved" >"$unmoved_base" + fi +else + echo "NOT COMPARED WITH THE TRUNK: no merge base with origin/${MESH_CHECK_BASE:-main}; checks/config/unmoved is held to the list it was first written with" +fi +(cd checks/config && go test -count=1 . && if [ -n "$unmoved_base" ]; then go run . -base "$unmoved_base" ../..; else go run . ../..; fi) +# The file text (novox/hq ADR 0349 §2): a file's text is a template in files/, outside the modules not yet moved; the +# list of those only shrinks, judged against the trunk's copy at the merge base, a trunk without one being its first. +inline_base="" +if merge_base=$(git merge-base HEAD "origin/${MESH_CHECK_BASE:-main}" 2>/dev/null); then + if git cat-file -e "$merge_base:checks/templates/inline" 2>/dev/null; then + inline_base=$(mktemp) + git show "$merge_base:checks/templates/inline" >"$inline_base" + fi +else + echo "NOT COMPARED WITH THE TRUNK: no merge base with origin/${MESH_CHECK_BASE:-main}; checks/templates/inline is held to the list it was first written with" +fi +(cd checks/templates && go test -count=1 . && if [ -n "$inline_base" ]; then go run . -base "$inline_base" ../..; else go run . ../..; fi) # The lists that narrow the paths check may not grow unseen: the base branch's copies are what they are # judged against (MESH_CHECK_BASE, main when unset), read from the clone the check runs in. base_lists="" diff --git a/modules/adwaita/README.md b/modules/adwaita/README.md index c8ff628a..040a46d2 100644 --- a/modules/adwaita/README.md +++ b/modules/adwaita/README.md @@ -22,11 +22,11 @@ for GTK, Qt, the portal and the cursor, dark by default. It claims no seat, beca | path | class | from | |---|---|---| -| `~/.config/gtk-3.0/settings.ini` | owned (the found file kept once) | [`config/gtk-settings.ini`](config/gtk-settings.ini) | +| `~/.config/gtk-3.0/settings.ini` | owned (the found file kept once) | [`files/gtk-settings.ini`](files/gtk-settings.ini) | | `~/.config/gtk-4.0/settings.ini` | owned | the same file | -| `~/.config/qt6ct/qt6ct.conf` | owned | [`config/qt6ct.conf`](config/qt6ct.conf) | -| `~/.config/xdg-desktop-portal/portals.conf` | owned | [`config/portals.conf`](config/portals.conf) | -| `~/.icons/default/index.theme` | owned | [`config/cursor-index.theme`](config/cursor-index.theme): the cursor for programs that read neither `XCURSOR_THEME` nor the resources | +| `~/.config/qt6ct/qt6ct.conf` | owned | [`files/qt6ct.conf`](files/qt6ct.conf) | +| `~/.config/xdg-desktop-portal/portals.conf` | owned | [`files/portals.conf`](files/portals.conf) | +| `~/.icons/default/index.theme` | owned | [`files/cursor-index.theme`](files/cursor-index.theme): the cursor for programs that read neither `XCURSOR_THEME` nor the resources | **`qt6ct.conf` is the mesh's now.** A change made in qt6ct's own window is replaced at the next push, and the window's saved geometry goes with it. The theme is this module's to say. Settings will make it diff --git a/modules/adwaita/cmd/adwaita-tools/adwaita_test.go b/modules/adwaita/cmd/adwaita-tools/adwaita_test.go index fc1b3264..be9d84b9 100644 --- a/modules/adwaita/cmd/adwaita-tools/adwaita_test.go +++ b/modules/adwaita/cmd/adwaita-tools/adwaita_test.go @@ -18,10 +18,18 @@ type manifest struct { Environment struct { Variables map[string]string `json:"variables"` } `json:"environment"` + Items []struct { + Collection string `json:"collection"` + Attributes map[string]string `json:"attributes"` + Template string `json:"template"` + } `json:"items"` Shell []struct { For, Slot, Code string } `json:"shell"` Resources []map[string]any `json:"resources"` + Build struct { + Artifacts []map[string]any `json:"artifacts"` + } `json:"build"` } func readManifest(t *testing.T) manifest { @@ -33,6 +41,23 @@ func readManifest(t *testing.T) manifest { if err := json.Unmarshal(raw, &m); err != nil { t.Fatal(err) } + // The module's code and variables are the templates of its items (design 57 L3, issues 795 and 807). + for _, it := range m.Items { + text, err := os.ReadFile(filepath.Join("..", "..", it.Template)) + if err != nil { + t.Fatal(err) + } + parts := strings.Split(it.Collection, ":") + switch { + case len(parts) == 3 && parts[0] == "shell": + m.Shell = append(m.Shell, struct{ For, Slot, Code string }{parts[1], parts[2], string(text)}) + case it.Collection == "environment" && it.Attributes["kind"] == "variable": + if m.Environment.Variables == nil { + m.Environment.Variables = map[string]string{} + } + m.Environment.Variables[it.Attributes["name"]] = strings.TrimSuffix(string(text), "\n") + } + } return m } @@ -96,9 +121,10 @@ func TestItOwnsTheFilesItsSourcesHoldAndNoQt5Duplicate(t *testing.T) { pkgs = append(pkgs, r["package"].(string)) continue } - raw, _ := os.ReadFile(filepath.Join("..", "..", "config", sources[id])) - if r["content"] != string(raw) || r["into"] != nil || r["owner"] != "${machine:account}" { - t.Errorf("%s is not config/%s, whole and the account's", id, sources[id]) + _, inline := r["content"] + if _, err := os.Stat(filepath.Join("..", "..", "files", sources[id])); err != nil || inline || + r["template"] != "files/"+sources[id] || r["into"] != nil || r["owner"] != "${machine:account}" { + t.Errorf("%s is not files/%s, whole, as its template, and the account's", id, sources[id]) } if strings.Contains(r["path"].(string), "qt5ct") { t.Error("qt5ct: both workstations run Qt 6 programs through qt6ct; a second tool is a duplicate") @@ -107,7 +133,7 @@ func TestItOwnsTheFilesItsSourcesHoldAndNoQt5Duplicate(t *testing.T) { if strings.Join(pkgs, ",") != "gnome-themes-extra,adwaita-icon-theme,adwaita-cursors,qt6ct,xdg-desktop-portal-gtk" { t.Fatalf("%v", pkgs) } - qt := readINI(filepath.Join("..", "..", "config", "qt6ct.conf")) + qt := readINI(filepath.Join("..", "..", "files", "qt6ct.conf")) if qt["Appearance"]["style"] != "Fusion" || qt["Appearance"]["custom_palette"] != "true" || !strings.Contains(qt["Fonts"]["general"], "Inter,11") { t.Fatalf("%v", qt) } @@ -116,7 +142,7 @@ func TestItOwnsTheFilesItsSourcesHoldAndNoQt5Duplicate(t *testing.T) { t.Fatalf("qt6ct ships no %s", qt["Appearance"]["color_scheme_path"]) } } - gtk := readINI(filepath.Join("..", "..", "config", "gtk-settings.ini"))["Settings"] + gtk := readINI(filepath.Join("..", "..", "files", "gtk-settings.ini"))["Settings"] if gtk["gtk-theme-name"] != "Adwaita" || gtk["gtk-application-prefer-dark-theme"] != "1" || gtk["gtk-font-name"] != "Inter 11" { t.Fatalf("%v", gtk) } @@ -244,3 +270,31 @@ func asJSON(v any) string { b, _ := json.Marshal(v) return string(b) } + +// No file resource carries its text inline: each names its template in files/, and the build packs files/ +// into the one archive the controller reads the templates back from (novox/hq ADR 0349 §2). +func TestEveryFileTakesItsTextFromATemplate(t *testing.T) { + m := readManifest(t) + for _, r := range m.Resources { + if _, inline := r["content"]; inline { + t.Errorf("resource %v keeps its text inline: keep it in files/ and name it as the template", r["id"]) + } + if name, ok := r["template"].(string); ok { + if _, err := os.Stat(filepath.Join("..", "..", name)); err != nil { + t.Errorf("resource %v names %s: %v", r["id"], name, err) + } + } + } + packs := 0 + for _, a := range m.Build.Artifacts { + if a["kind"] == "archive" && a["from"] == "files" { + packs++ + if a["name"] != "files" { + t.Errorf("the templates archive is named %v, not files", a["name"]) + } + } + } + if packs != 1 { + t.Errorf("the build packs files/ into %d archives, not one", packs) + } +} diff --git a/modules/adwaita/config/cursor-index.theme b/modules/adwaita/files/cursor-index.theme similarity index 100% rename from modules/adwaita/config/cursor-index.theme rename to modules/adwaita/files/cursor-index.theme diff --git a/modules/adwaita/files/env-gtk-theme b/modules/adwaita/files/env-gtk-theme new file mode 100644 index 00000000..b98c8093 --- /dev/null +++ b/modules/adwaita/files/env-gtk-theme @@ -0,0 +1 @@ +Adwaita:dark diff --git a/modules/adwaita/files/env-gtk2-rc-files b/modules/adwaita/files/env-gtk2-rc-files new file mode 100644 index 00000000..8a782fb8 --- /dev/null +++ b/modules/adwaita/files/env-gtk2-rc-files @@ -0,0 +1 @@ +/usr/share/themes/Adwaita-dark/gtk-2.0/gtkrc diff --git a/modules/adwaita/files/env-qt-qpa-platformtheme b/modules/adwaita/files/env-qt-qpa-platformtheme new file mode 100644 index 00000000..349e5d6b --- /dev/null +++ b/modules/adwaita/files/env-qt-qpa-platformtheme @@ -0,0 +1 @@ +qt6ct diff --git a/modules/adwaita/files/env-qt-select b/modules/adwaita/files/env-qt-select new file mode 100644 index 00000000..1e8b3149 --- /dev/null +++ b/modules/adwaita/files/env-qt-select @@ -0,0 +1 @@ +6 diff --git a/modules/adwaita/files/env-qt-style-override b/modules/adwaita/files/env-qt-style-override new file mode 100644 index 00000000..a0da7db8 --- /dev/null +++ b/modules/adwaita/files/env-qt-style-override @@ -0,0 +1 @@ +Fusion diff --git a/modules/adwaita/files/env-xcursor-size b/modules/adwaita/files/env-xcursor-size new file mode 100644 index 00000000..a45fd52c --- /dev/null +++ b/modules/adwaita/files/env-xcursor-size @@ -0,0 +1 @@ +24 diff --git a/modules/adwaita/files/env-xcursor-theme b/modules/adwaita/files/env-xcursor-theme new file mode 100644 index 00000000..cb9676b8 --- /dev/null +++ b/modules/adwaita/files/env-xcursor-theme @@ -0,0 +1 @@ +Adwaita diff --git a/modules/adwaita/config/gtk-settings.ini b/modules/adwaita/files/gtk-settings.ini similarity index 100% rename from modules/adwaita/config/gtk-settings.ini rename to modules/adwaita/files/gtk-settings.ini diff --git a/modules/adwaita/config/portals.conf b/modules/adwaita/files/portals.conf similarity index 100% rename from modules/adwaita/config/portals.conf rename to modules/adwaita/files/portals.conf diff --git a/modules/adwaita/config/qt6ct.conf b/modules/adwaita/files/qt6ct.conf similarity index 100% rename from modules/adwaita/config/qt6ct.conf rename to modules/adwaita/files/qt6ct.conf diff --git a/modules/adwaita/files/xinitrc-normal b/modules/adwaita/files/xinitrc-normal new file mode 100644 index 00000000..864b3cf6 --- /dev/null +++ b/modules/adwaita/files/xinitrc-normal @@ -0,0 +1,10 @@ +# The appearance (module adwaita): GSettings is where the portal reads dark or light, and the portal is +# the only way it reaches Electron, Chromium, Firefox and flatpaks. Set at every session start to the +# module's default; adwaita_appearance switches it for a session. +gsettings set org.gnome.desktop.interface color-scheme 'prefer-dark' || true +gsettings set org.gnome.desktop.interface gtk-theme 'Adwaita' || true +gsettings set org.gnome.desktop.interface icon-theme 'Adwaita' || true +gsettings set org.gnome.desktop.interface cursor-theme 'Adwaita' || true +gsettings set org.gnome.desktop.interface cursor-size 24 || true +gsettings set org.gnome.desktop.interface font-name 'Inter 11' || true +gsettings set org.gnome.desktop.interface monospace-font-name 'JetBrainsMono Nerd Font 11' || true diff --git a/modules/adwaita/files/xresources-normal b/modules/adwaita/files/xresources-normal new file mode 100644 index 00000000..02e7ec33 --- /dev/null +++ b/modules/adwaita/files/xresources-normal @@ -0,0 +1,3 @@ +! adwaita: the cursor, for X programs that take it from the resources. +Xcursor.theme: Adwaita +Xcursor.size: 24 diff --git a/modules/adwaita/module.json b/modules/adwaita/module.json index c02971ea..bf81c833 100644 --- a/modules/adwaita/module.json +++ b/modules/adwaita/module.json @@ -1,6 +1,7 @@ { "module": "adwaita", "version": "1", + "moved": true, "capabilities": [ "package-manager" ], @@ -10,27 +11,79 @@ "adwaita_icons", "adwaita_portal_check" ], - "environment": { - "variables": { - "GTK_THEME": "Adwaita:dark", - "GTK2_RC_FILES": "/usr/share/themes/Adwaita-dark/gtk-2.0/gtkrc", - "QT_QPA_PLATFORMTHEME": "qt6ct", - "QT_STYLE_OVERRIDE": "Fusion", - "QT_SELECT": "6", - "XCURSOR_THEME": "Adwaita", - "XCURSOR_SIZE": "24" - } - }, - "shell": [ + "items": [ { - "for": "xresources", - "slot": "normal", - "code": "! adwaita: the cursor, for X programs that take it from the resources.\nXcursor.theme: Adwaita\nXcursor.size: 24\n" + "collection": "environment", + "name": "gtk-theme", + "attributes": { + "kind": "variable", + "name": "GTK_THEME" + }, + "template": "files/env-gtk-theme" }, { - "for": "xinitrc", - "slot": "normal", - "code": "# The appearance (module adwaita): GSettings is where the portal reads dark or light, and the portal is\n# the only way it reaches Electron, Chromium, Firefox and flatpaks. Set at every session start to the\n# module's default; adwaita_appearance switches it for a session.\ngsettings set org.gnome.desktop.interface color-scheme 'prefer-dark' || true\ngsettings set org.gnome.desktop.interface gtk-theme 'Adwaita' || true\ngsettings set org.gnome.desktop.interface icon-theme 'Adwaita' || true\ngsettings set org.gnome.desktop.interface cursor-theme 'Adwaita' || true\ngsettings set org.gnome.desktop.interface cursor-size 24 || true\ngsettings set org.gnome.desktop.interface font-name 'Inter 11' || true\ngsettings set org.gnome.desktop.interface monospace-font-name 'JetBrainsMono Nerd Font 11' || true\n" + "collection": "environment", + "name": "gtk2-rc-files", + "attributes": { + "kind": "variable", + "name": "GTK2_RC_FILES" + }, + "template": "files/env-gtk2-rc-files" + }, + { + "collection": "environment", + "name": "qt-qpa-platformtheme", + "attributes": { + "kind": "variable", + "name": "QT_QPA_PLATFORMTHEME" + }, + "template": "files/env-qt-qpa-platformtheme" + }, + { + "collection": "environment", + "name": "qt-style-override", + "attributes": { + "kind": "variable", + "name": "QT_STYLE_OVERRIDE" + }, + "template": "files/env-qt-style-override" + }, + { + "collection": "environment", + "name": "qt-select", + "attributes": { + "kind": "variable", + "name": "QT_SELECT" + }, + "template": "files/env-qt-select" + }, + { + "collection": "environment", + "name": "xcursor-theme", + "attributes": { + "kind": "variable", + "name": "XCURSOR_THEME" + }, + "template": "files/env-xcursor-theme" + }, + { + "collection": "environment", + "name": "xcursor-size", + "attributes": { + "kind": "variable", + "name": "XCURSOR_SIZE" + }, + "template": "files/env-xcursor-size" + }, + { + "collection": "shell:xresources:normal", + "name": "xresources-normal", + "template": "files/xresources-normal" + }, + { + "collection": "shell:xinitrc:normal", + "name": "xinitrc-normal", + "template": "files/xinitrc-normal" } ], "resources": [ @@ -65,7 +118,7 @@ "path": "${machine:account-home}/.config/gtk-3.0/settings.ini", "owner": "${machine:account}", "mode": "0644", - "content": "# Written by the mesh (module adwaita, novox/hq ADR 0208), for GTK 3 and GTK 4 alike. Replaced at\n# every push; adwaita_appearance switches dark and light for the running session.\n[Settings]\ngtk-theme-name=Adwaita\ngtk-icon-theme-name=Adwaita\ngtk-cursor-theme-name=Adwaita\ngtk-cursor-theme-size=24\ngtk-font-name=Inter 11\ngtk-application-prefer-dark-theme=1\n" + "template": "files/gtk-settings.ini" }, { "id": "gtk4", @@ -73,7 +126,7 @@ "path": "${machine:account-home}/.config/gtk-4.0/settings.ini", "owner": "${machine:account}", "mode": "0644", - "content": "# Written by the mesh (module adwaita, novox/hq ADR 0208), for GTK 3 and GTK 4 alike. Replaced at\n# every push; adwaita_appearance switches dark and light for the running session.\n[Settings]\ngtk-theme-name=Adwaita\ngtk-icon-theme-name=Adwaita\ngtk-cursor-theme-name=Adwaita\ngtk-cursor-theme-size=24\ngtk-font-name=Inter 11\ngtk-application-prefer-dark-theme=1\n" + "template": "files/gtk-settings.ini" }, { "id": "qt6ct", @@ -81,7 +134,7 @@ "path": "${machine:account-home}/.config/qt6ct/qt6ct.conf", "owner": "${machine:account}", "mode": "0644", - "content": "[Appearance]\ncolor_scheme_path=/usr/share/qt6ct/colors/darker.conf\ncustom_palette=true\nicon_theme=Adwaita\nstandard_dialogs=default\nstyle=Fusion\n\n[Fonts]\nfixed=\"JetBrainsMono Nerd Font,11,-1,5,50,0,0,0,0,0\"\ngeneral=\"Inter,11,-1,5,50,0,0,0,0,0\"\n\n[Interface]\nactivate_item_on_single_click=1\nbuttonbox_layout=0\ncursor_flash_time=1000\ndialog_buttons_have_icons=1\ndouble_click_interval=400\ngui_effects=@Invalid()\nkeyboard_scheme=2\nmenus_have_icons=true\nshow_shortcuts_in_context_menus=true\nstylesheets=@Invalid()\ntoolbutton_style=4\nunderline_shortcut=1\nwheel_scroll_lines=3\n\n[Troubleshooting]\nforce_raster_widgets=1\nignored_applications=@Invalid()\n" + "template": "files/qt6ct.conf" }, { "id": "portals", @@ -89,7 +142,7 @@ "path": "${machine:account-home}/.config/xdg-desktop-portal/portals.conf", "owner": "${machine:account}", "mode": "0644", - "content": "# Written by the mesh (module adwaita, novox/hq ADR 0208). Replaced at every push.\n#\n# Which portal backend answers each interface. i3 is not a desktop xdg-desktop-portal knows, so with\n# no preference it uses whichever backend happens to be installed: fine while gtk is the only one,\n# wrong the day another arrives as somebody else's dependency. Named instead. gtk also serves\n# org.freedesktop.appearance (dark or light) from GSettings, which the session's start sets.\n[preferred]\ndefault=gtk\n# Secrets for sandboxed programs come from the keyring's backend. Without this line no backend answers\n# the interface: gtk does not implement it, and gnome-keyring's names only GNOME as its desktop.\norg.freedesktop.impl.portal.Secret=gnome-keyring\n" + "template": "files/portals.conf" }, { "id": "cursor", @@ -97,7 +150,7 @@ "path": "${machine:account-home}/.icons/default/index.theme", "owner": "${machine:account}", "mode": "0644", - "content": "# Written by the mesh (module adwaita, novox/hq ADR 0208): the default cursor theme, for programs that\n# read neither XCURSOR_THEME nor the X resources.\n[Icon Theme]\nName=Default\nInherits=Adwaita\n" + "template": "files/cursor-index.theme" } ], "build": { @@ -112,6 +165,11 @@ "loads": [ "adwaita-tools" ] + }, + { + "name": "files", + "kind": "archive", + "from": "files" } ] } diff --git a/modules/artifact-store-tools/cmd/artifact-store-tools/templates_test.go b/modules/artifact-store-tools/cmd/artifact-store-tools/templates_test.go new file mode 100644 index 00000000..7d0718b5 --- /dev/null +++ b/modules/artifact-store-tools/cmd/artifact-store-tools/templates_test.go @@ -0,0 +1,64 @@ +package main + +import ( + "encoding/json" + "os" + "path/filepath" + "testing" +) + +// The module's directory, where its manifest and its files/ are. +const moduleDir = "../.." + +// templateText is the text a file resource places on the machine: its template's, in the module's files/ +// (novox/hq ADR 0349 §2). A resource that names no template, or a template that is not there, fails the test. +func templateText(t *testing.T, r map[string]any) string { + t.Helper() + name, _ := r["template"].(string) + if name == "" { + t.Fatalf("resource %v names no template", r["id"]) + } + raw, err := os.ReadFile(filepath.Join(moduleDir, filepath.FromSlash(name))) + if err != nil { + t.Fatalf("resource %v names %s: %v", r["id"], name, err) + } + return string(raw) +} + +// No file resource carries its text inline: each names its template in files/, and the build packs files/ +// into the one archive the controller reads the templates back from (novox/hq ADR 0349 §2). +func TestEveryFileTakesItsTextFromATemplate(t *testing.T) { + raw, err := os.ReadFile(filepath.Join(moduleDir, "module.json")) + if err != nil { + t.Fatal(err) + } + var m struct { + Resources []map[string]any `json:"resources"` + Build struct { + Artifacts []map[string]any `json:"artifacts"` + } `json:"build"` + } + if err := json.Unmarshal(raw, &m); err != nil { + t.Fatal(err) + } + for _, r := range m.Resources { + if _, inline := r["content"]; inline { + t.Errorf("resource %v keeps its text inline: keep it in files/ and name it as the template", r["id"]) + } + if r["type"] == "file" { + templateText(t, r) + } + } + packs := 0 + for _, a := range m.Build.Artifacts { + if a["kind"] == "archive" && a["from"] == "files" { + packs++ + if a["name"] != "files" { + t.Errorf("the templates archive is named %v, not files", a["name"]) + } + } + } + if packs != 1 { + t.Errorf("the build packs files/ into %d archives, not one", packs) + } +} diff --git a/modules/artifact-store-tools/files/store-names b/modules/artifact-store-tools/files/store-names new file mode 100644 index 00000000..29cab310 --- /dev/null +++ b/modules/artifact-store-tools/files/store-names @@ -0,0 +1,4 @@ +# The names the artifact store had before, from the setting older-store-names, written by the mesh. +# artifact_store_references reads every definition under these too (novox/hq issue 323). +# Replaced on every push; change the setting, never this file. +older=${config:older-store-names} diff --git a/modules/artifact-store-tools/module.json b/modules/artifact-store-tools/module.json index d17ac5d2..fb6b34bb 100644 --- a/modules/artifact-store-tools/module.json +++ b/modules/artifact-store-tools/module.json @@ -13,13 +13,14 @@ "artifact_store_references", "artifact_store_collect" ], - "settings": { - "older-store-names": { - "kind": "preference", - "default": "none", - "why": "the names the artifact store had before, as a definition may still pin an image under them: host names or host:port, separated by spaces; none when it had no other. artifact_store_references reads every definition under these as well as under the store's current name, so an image a definition pins under one is said to be named by a definition (novox/hq issue 323)" - } - }, + "config": { + "older-store-names": { + "type": "string", + "description": "The names the artifact store had before, as host names or host:port separated by spaces, or none when it had no other. artifact_store_references reads every definition under these as well as under the store's current name, so an image a definition pins under one is said to be named by a definition (novox/hq issue 323).", + "pattern": "none|[A-Za-z0-9.:-]+( [A-Za-z0-9.:-]+)*", + "default": "none" + } + }, "resources": [ { "id": "state", @@ -33,7 +34,7 @@ "path": "${dir:state}/store-names", "mode": "0600", "trusted": false, - "content": "# The names the artifact store had before, from the setting older-store-names, written by the mesh.\n# artifact_store_references reads every definition under these too (novox/hq issue 323).\n# Replaced on every push; change the setting, never this file.\nolder=${setting:older-store-names}\n" + "template": "files/store-names" } ], "build": { @@ -52,6 +53,11 @@ "MESH_ARTIFACT_STORE_CONTAINER": "mesh-registry", "MESH_ARTIFACT_STORE_NAMES_FILE": "${dir:state}/store-names" } + }, + { + "name": "files", + "kind": "archive", + "from": "files" } ] } diff --git a/modules/asus-zephyrus-g14/cmd/zephyrus/keys_test.go b/modules/asus-zephyrus-g14/cmd/zephyrus/keys_test.go index d289c14e..d435c882 100644 --- a/modules/asus-zephyrus-g14/cmd/zephyrus/keys_test.go +++ b/modules/asus-zephyrus-g14/cmd/zephyrus/keys_test.go @@ -34,11 +34,9 @@ func TestKeysListsTriggersI3LinesAndFirmwareKeys(t *testing.T) { home := "/home/operator" t.Setenv("MESH_OPERATOR_HOME", home) // The i3 module's file as the controller composes it: other modules' lines around this one's. - var own string - for _, c := range m.Contributions { - if c.Seat == "node-display-session" { - own += c.Content - } + own := m.i3Lines() + if own == "" || len(m.Contributions) != 0 { + t.Fatalf("the lines for the window manager are one item, and no contribution is left: %q %v", own, m.Contributions) } put(t, f.root, home+"/"+I3Config, "set $mod Mod4\n# rofi\nbindsym $mod+d exec rofi\n# "+ModuleName+"\n"+own+ "# triggerhappy-not-here\nbindsym $mod+Shift+z exec other\n#########################################\ninclude x\n") diff --git a/modules/asus-zephyrus-g14/cmd/zephyrus/manifest_test.go b/modules/asus-zephyrus-g14/cmd/zephyrus/manifest_test.go index 410c78ac..d300d0f6 100644 --- a/modules/asus-zephyrus-g14/cmd/zephyrus/manifest_test.go +++ b/modules/asus-zephyrus-g14/cmd/zephyrus/manifest_test.go @@ -15,19 +15,41 @@ type manifest struct { Tools []string `json:"tools"` Emits []string `json:"emits"` Resources []map[string]any `json:"resources"` - // Contributions to other modules' seats (novox/hq ADR 0212): the vendor keys go to node-hotkeys. - Contributions []struct { - Seat string `json:"seat"` - Kind string `json:"kind"` - Content string `json:"content"` - } `json:"contributions"` + // Items of collections (novox/hq ADR 0349 clause 13): the vendor keys are an item of the node-hotkeys + // triggers, its text the template file beside the manifest. + Items []struct { + Collection string `json:"collection"` + Template string `json:"template"` + } `json:"items"` + // Old-form contributions to other modules' seats (novox/hq ADR 0212): none is left. + Contributions []any `json:"contributions"` } -// triggers is the module's contribution to node-hotkeys: its vendor keys. +// i3Lines is the module's lines for the window manager: the text of the template of its item of the collection +// contribution:node-display-session:config (design 57 L3, issue 848). +func (m manifest) i3Lines() string { + for _, it := range m.Items { + if it.Collection == "contribution:node-display-session:config" { + raw, err := os.ReadFile(filepath.Join("..", "..", it.Template)) + if err != nil { + return "" + } + return string(raw) + } + } + return "" +} + +// triggers is the module's contribution to node-hotkeys: its vendor keys, the text of the template of its +// item of the collection contribution:node-hotkeys:trigger. func (m manifest) triggers() string { - for _, c := range m.Contributions { - if c.Seat == "node-hotkeys" && c.Kind == "trigger" { - return c.Content + for _, it := range m.Items { + if it.Collection == "contribution:node-hotkeys:trigger" { + raw, err := os.ReadFile(filepath.Join("..", "..", it.Template)) + if err != nil { + return "" + } + return string(raw) } } return "" diff --git a/modules/asus-zephyrus-g14/cmd/zephyrus/templates_test.go b/modules/asus-zephyrus-g14/cmd/zephyrus/templates_test.go new file mode 100644 index 00000000..cc5a736f --- /dev/null +++ b/modules/asus-zephyrus-g14/cmd/zephyrus/templates_test.go @@ -0,0 +1,83 @@ +package main + +import ( + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" +) + +// Each file's text is a template in templates/, the directory the manifest names (novox/hq ADR 0349 §2, issue 731), +// never in files/: files/ is the archive the machine unpacks, and stays exactly what the machine is given. +func TestEveryFileTakesItsTextFromATemplateOutsideTheUnpackedFiles(t *testing.T) { + raw, err := os.ReadFile("../../module.json") + if err != nil { + t.Fatal(err) + } + var m struct { + Templates string `json:"templates"` + Resources []map[string]any `json:"resources"` + Build struct { + Artifacts []map[string]any `json:"artifacts"` + } `json:"build"` + } + if err := json.Unmarshal(raw, &m); err != nil { + t.Fatal(err) + } + if m.Templates != "templates" { + t.Fatalf("the manifest names the templates directory %q, not templates", m.Templates) + } + files := 0 + for _, r := range m.Resources { + if _, inline := r["content"]; inline { + t.Errorf("resource %v keeps its text inline: keep it in templates/ and name it as the template", r["id"]) + } + if r["type"] != "file" { + continue + } + files++ + name, _ := r["template"].(string) + if !strings.HasPrefix(name, "templates/") { + t.Errorf("resource %v names the template %q, not one in templates/", r["id"], name) + continue + } + if st, err := os.Lstat(filepath.Join("../..", name)); err != nil || !st.Mode().IsRegular() { + t.Errorf("resource %v names the template %s, which is not a plain file: %v", r["id"], name, err) + } + } + if files == 0 { + t.Fatal("no file resource") + } + packs := 0 + for _, a := range m.Build.Artifacts { + if a["kind"] == "archive" && a["from"] == "templates" { + packs++ + if a["name"] != "templates" { + t.Errorf("the templates archive is named %v, not templates", a["name"]) + } + } + } + if packs != 1 { + t.Fatalf("the build packs templates/ into %d archives, not one", packs) + } + // The templates archive is read by the controller, never unpacked onto the machine. + for _, r := range m.Resources { + if r["artifact"] == "templates" { + t.Errorf("resource %v places the templates archive on the machine", r["id"]) + } + } + // And files/ holds what the machine runs, no template. + err = filepath.WalkDir("../../files", func(at string, d os.DirEntry, err error) error { + if err != nil { + return err + } + if !d.IsDir() && !strings.HasPrefix(filepath.ToSlash(at), "../../files/bin/") { + t.Errorf("files/ holds %s, which is not one of the scripts the machine unpacks", at) + } + return nil + }) + if err != nil { + t.Fatal(err) + } +} diff --git a/modules/asus-zephyrus-g14/module.json b/modules/asus-zephyrus-g14/module.json index 13a2fbf8..77869927 100644 --- a/modules/asus-zephyrus-g14/module.json +++ b/modules/asus-zephyrus-g14/module.json @@ -1,6 +1,8 @@ { "module": "asus-zephyrus-g14", "version": "1", + "moved": true, + "templates": "templates", "capabilities": [ "package-manager", "service-manager" @@ -59,14 +61,14 @@ "type": "file", "path": "/etc/modprobe.d/g14-nvidia-power.conf", "mode": "0644", - "content": "# Managed by the mesh (module asus-zephyrus-g14). Replaced on every push; edit the catalogue instead.\n#\n# The discrete GPU's driver options on the ROG Zephyrus G14 (GA403, RTX 40 series, hybrid graphics).\n#\n# NVreg_DynamicPowerManagement=0x00 turns runtime D3 off. With it on, a change of power source sends\n# the driver an ACPI notification it fails to handle on this model (\"RmHandleDNotifierEvent: Failed to\n# handle ACPI D-Notifier event, status=0x62\"), and the GPU stops making progress until the machine is\n# powered off. Off costs a few idle watts in hybrid mode and keeps the machine up.\n#\n# NVreg_PreserveVideoMemoryAllocations=1 saves video memory across suspend, so what used the GPU still\n# works after waking. It needs nvidia-suspend, -hibernate and -resume to run around a sleep, which this\n# module's drop-ins on the sleep services ask for.\n#\n# A change here applies when the driver next loads: at the next boot.\noptions nvidia NVreg_PreserveVideoMemoryAllocations=1\noptions nvidia NVreg_DynamicPowerManagement=0x00\n" + "template": "templates/modprobe.d/g14-nvidia-power.conf" }, { "id": "video-options", "type": "file", "path": "/etc/modprobe.d/video-brightness-switch.conf", "mode": "0644", - "content": "# Managed by the mesh (module asus-zephyrus-g14). Replaced on every push; edit the catalogue instead.\n#\n# The ACPI video driver does not change the backlight itself on the brightness keys: on this model it\n# moves the wrong one. The keys are triggerhappy's (see /etc/triggerhappy/triggers.d/asus-g14.conf).\n# Applies when the module next loads: at the next boot.\noptions video brightness_switch_enabled=0\n" + "template": "templates/modprobe.d/video-brightness-switch.conf" }, { "id": "suspend-drop-ins", @@ -79,7 +81,7 @@ "type": "file", "path": "/etc/systemd/system/systemd-suspend.service.d/asus-zephyrus-g14-nvidia.conf", "mode": "0644", - "content": "# Managed by the mesh (module asus-zephyrus-g14). Replaced on every push; edit the catalogue instead.\n#\n# The NVIDIA driver's own sleep actions, asked for by the sleep itself rather than enabled as\n# links: the mesh declares files and never makes links (novox/hq ADR 0012), and the host's service\n# shape must not start these units by hand, which would put the GPU to sleep with the machine awake.\n[Unit]\nWants=nvidia-suspend.service nvidia-resume.service\n" + "template": "templates/systemd/system/systemd-suspend.service.d/asus-zephyrus-g14-nvidia.conf" }, { "id": "hibernate-drop-ins", @@ -92,7 +94,7 @@ "type": "file", "path": "/etc/systemd/system/systemd-hibernate.service.d/asus-zephyrus-g14-nvidia.conf", "mode": "0644", - "content": "# Managed by the mesh (module asus-zephyrus-g14). Replaced on every push; edit the catalogue instead.\n#\n# The NVIDIA driver's own sleep actions, asked for by the sleep itself rather than enabled as\n# links: the mesh declares files and never makes links (novox/hq ADR 0012), and the host's service\n# shape must not start these units by hand, which would put the GPU to sleep with the machine awake.\n[Unit]\nWants=nvidia-hibernate.service nvidia-resume.service\n" + "template": "templates/systemd/system/systemd-hibernate.service.d/asus-zephyrus-g14-nvidia.conf" }, { "id": "suspend-then-hibernate-drop-ins", @@ -105,7 +107,7 @@ "type": "file", "path": "/etc/systemd/system/systemd-suspend-then-hibernate.service.d/asus-zephyrus-g14-nvidia.conf", "mode": "0644", - "content": "# Managed by the mesh (module asus-zephyrus-g14). Replaced on every push; edit the catalogue instead.\n#\n# The NVIDIA driver's own sleep actions, asked for by the sleep itself rather than enabled as\n# links: the mesh declares files and never makes links (novox/hq ADR 0012), and the host's service\n# shape must not start these units by hand, which would put the GPU to sleep with the machine awake.\n[Unit]\nWants=nvidia-suspend-then-hibernate.service nvidia-resume.service\n" + "template": "templates/systemd/system/systemd-suspend-then-hibernate.service.d/asus-zephyrus-g14-nvidia.conf" }, { "id": "powerd-drop-ins", @@ -118,14 +120,14 @@ "type": "file", "path": "/etc/systemd/system/nvidia-powerd.service.d/asus-zephyrus-g14.conf", "mode": "0644", - "content": "# Managed by the mesh (module asus-zephyrus-g14). Replaced on every push; edit the catalogue instead.\n#\n# nvidia-powerd (Dynamic Boost) was the first error in the chain that hung this model's GPU on a change\n# of power source, and asusd starts it on mains. It runs only when the kernel command line says\n# zephyrus.nvidia-powerd — an explicit opt-in, at boot.\n[Unit]\nConditionKernelCommandLine=zephyrus.nvidia-powerd\n" + "template": "templates/systemd/system/nvidia-powerd.service.d/asus-zephyrus-g14.conf" }, { "id": "backlight-rule", "type": "file", "path": "/etc/udev/rules.d/90-backlight.rules", "mode": "0644", - "content": "# Managed by the mesh (module asus-zephyrus-g14). Replaced on every push; edit the catalogue instead.\n#\n# The backlights are writable by the video group, so the brightness keys and the module's brightness tool\n# move the panel without root.\nACTION==\"add\", SUBSYSTEM==\"backlight\", RUN+=\"/usr/bin/chgrp video /sys/class/backlight/%k/brightness\", RUN+=\"/usr/bin/chmod g+w /sys/class/backlight/%k/brightness\"\n" + "template": "templates/udev/rules.d/90-backlight.rules" }, { "id": "udev", @@ -151,7 +153,7 @@ "type": "file", "path": "/etc/UPower/UPower.conf.d/50-asus-zephyrus-g14.conf", "mode": "0644", - "content": "# Managed by the mesh (module asus-zephyrus-g14). Replaced on every push; edit the catalogue instead.\n#\n# On low battery the machine suspends rather than powering off, at 7 % — s2idle still draws a little,\n# so it leaves headroom. A drop-in over the package's own UPower.conf, which stays the package's.\n[UPower]\nUsePercentageForPolicy=true\nPercentageLow=15.0\nPercentageCritical=10.0\nPercentageAction=7.0\nCriticalPowerAction=Suspend\nAllowRiskyCriticalPowerAction=true\n" + "template": "templates/UPower/UPower.conf.d/50-asus-zephyrus-g14.conf" }, { "id": "upower", @@ -174,7 +176,22 @@ "type": "file", "path": "/etc/X11/xorg.conf.d/30-asus-zephyrus-g14-touchpad.conf", "mode": "0644", - "content": "# Managed by the mesh (module asus-zephyrus-g14). Replaced on every push; edit the catalogue instead.\n#\n# The touchpad's settings, applied by X every time the device appears — at login and after every\n# resume, when the device is initialised again. This replaces the predecessor's sleep hook, which ran\n# xinput after a resume as a named person on a guessed display.\nSection \"InputClass\"\n Identifier \"asus-zephyrus-g14 touchpad\"\n MatchIsTouchpad \"on\"\n Option \"Tapping\" \"on\"\n Option \"NaturalScrolling\" \"true\"\n Option \"AccelSpeed\" \"0.15\"\nEndSection\n" + "template": "templates/X11/xorg.conf.d/30-asus-zephyrus-g14-touchpad.conf" + } + ], + "items": [ + { + "collection": "shell:after-wake:normal", + "template": "templates/after-wake" + }, + { + "collection": "contribution:node-hotkeys:trigger", + "template": "templates/hotkeys-triggers" + }, + { + "collection": "contribution:node-display-session:config", + "name": "display-session", + "template": "templates/display-session" } ], "build": { @@ -194,31 +211,12 @@ "name": "scripts", "kind": "archive", "from": "files" + }, + { + "name": "templates", + "kind": "archive", + "from": "templates" } ] - }, - "contributions": [ - { - "seat": "node-hotkeys", - "kind": "trigger", - "content": "# The ROG Zephyrus G14's vendor keys, which reach no X client: media (the M-keys), panel brightness,\n# and the touchpad key. Each runs this module's own script, as the operator's account.\nKEY_PROG1\t1\t/usr/local/lib/asus-zephyrus-g14/bin/zephyrus-session /usr/local/lib/asus-zephyrus-g14/bin/zephyrus-media play-pause\nKEY_PROG3\t1\t/usr/local/lib/asus-zephyrus-g14/bin/zephyrus-session /usr/local/lib/asus-zephyrus-g14/bin/zephyrus-media previous\nKEY_PROG4\t1\t/usr/local/lib/asus-zephyrus-g14/bin/zephyrus-session /usr/local/lib/asus-zephyrus-g14/bin/zephyrus-media next\nKEY_BRIGHTNESSDOWN\t1\t/usr/local/lib/asus-zephyrus-g14/bin/zephyrus-backlight -\nKEY_BRIGHTNESSDOWN\t2\t/usr/local/lib/asus-zephyrus-g14/bin/zephyrus-backlight -\nKEY_BRIGHTNESSUP\t1\t/usr/local/lib/asus-zephyrus-g14/bin/zephyrus-backlight +\nKEY_BRIGHTNESSUP\t2\t/usr/local/lib/asus-zephyrus-g14/bin/zephyrus-backlight +\nKEY_F21\t1\t/usr/local/lib/asus-zephyrus-g14/bin/zephyrus-touchpad reset\n" - }, - { - "seat": "node-display-session", - "kind": "config", - "content": "# The laptop's own lines in i3 (module asus-zephyrus-g14, novox/hq ADR 0208, ADR 0210). Owned by the\n# mesh: replaced at every push. Once the controller places contributions to node-display-session\n# (ADR 0210), these become the module's contribution instead of a file in i3's directory.\n#\n# The keys the firmware turns into ordinary key presses. The vendor keys that reach no X client are\n# triggerhappy's (/etc/triggerhappy/triggers.d/asus-g14.conf): M4 and Fn+F4/F5 for media, Fn+F7/F8 for\n# the panel, Fn+F10 for the touchpad. The keyboard backlight (Fn+F2/F3) is the firmware's and asusd's.\n\n# Fn+F6, the screenshot key: the firmware sends Super+Shift+S. Released before it runs, because the\n# screenshot grabs the pointer to select a region, which fails while the key is still held.\nbindsym --release $mod+Shift+s exec --no-startup-id $XDG_CONFIG_HOME/i3/scripts/screenshot.sh\n\n# Fn+F9, the display key: the firmware sends Super+P. Odd workspaces to the panel, even ones to the\n# external output.\nbindsym $mod+p exec --no-startup-id /usr/local/lib/asus-zephyrus-g14/bin/zephyrus-display order\n\n# The keyboard backlight's level, shown when it changes.\nexec --no-startup-id /usr/local/lib/asus-zephyrus-g14/bin/zephyrus-kbd-notify\n" - }, - { - "seat": "node-display-session", - "kind": "config", - "content": "# The laptop's own lines in i3 (module asus-zephyrus-g14, novox/hq ADR 0208, ADR 0210). Owned by the\n# mesh: replaced at every push. Once the controller places contributions to node-display-session\n# (ADR 0210), these become the module's contribution instead of a file in i3's directory.\n#\n# The model's panel and touchpad.\n\n# The internal panel is the primary output, where the bars' tray goes. Which monitors are on and where\n# is the display server's (autorandr, run at every session start).\nexec --no-startup-id /usr/local/lib/asus-zephyrus-g14/bin/zephyrus-display primary\n\n# The touchpad's settings again, by hand. X applies them itself whenever the device appears.\nbindsym $mod+Shift+x exec --no-startup-id /usr/local/lib/asus-zephyrus-g14/bin/zephyrus-touchpad reset\n" - } - ], - "shell": [ - { - "for": "after-wake", - "slot": "normal", - "code": "# The touchpad's settings again after waking, in the operator's session: X applies the module's\n# input class when the device appears, and this covers a wake that does not initialise it again.\n# Runs as root from the power module; the reset itself runs as the session's owner.\nsleep 2\nowner=$(ps -o user= -C i3 | head -n 1)\n[ -n \"$owner\" ] && runuser -u \"$owner\" -- /usr/local/lib/asus-zephyrus-g14/bin/zephyrus-touchpad reset\ntrue\n" - } - ] + } } diff --git a/modules/asus-zephyrus-g14/templates/UPower/UPower.conf.d/50-asus-zephyrus-g14.conf b/modules/asus-zephyrus-g14/templates/UPower/UPower.conf.d/50-asus-zephyrus-g14.conf new file mode 100644 index 00000000..f3d6bef5 --- /dev/null +++ b/modules/asus-zephyrus-g14/templates/UPower/UPower.conf.d/50-asus-zephyrus-g14.conf @@ -0,0 +1,11 @@ +# Managed by the mesh (module asus-zephyrus-g14). Replaced on every push; edit the catalogue instead. +# +# On low battery the machine suspends rather than powering off, at 7 % — s2idle still draws a little, +# so it leaves headroom. A drop-in over the package's own UPower.conf, which stays the package's. +[UPower] +UsePercentageForPolicy=true +PercentageLow=15.0 +PercentageCritical=10.0 +PercentageAction=7.0 +CriticalPowerAction=Suspend +AllowRiskyCriticalPowerAction=true diff --git a/modules/asus-zephyrus-g14/templates/X11/xorg.conf.d/30-asus-zephyrus-g14-touchpad.conf b/modules/asus-zephyrus-g14/templates/X11/xorg.conf.d/30-asus-zephyrus-g14-touchpad.conf new file mode 100644 index 00000000..f30830c6 --- /dev/null +++ b/modules/asus-zephyrus-g14/templates/X11/xorg.conf.d/30-asus-zephyrus-g14-touchpad.conf @@ -0,0 +1,12 @@ +# Managed by the mesh (module asus-zephyrus-g14). Replaced on every push; edit the catalogue instead. +# +# The touchpad's settings, applied by X every time the device appears — at login and after every +# resume, when the device is initialised again. This replaces the predecessor's sleep hook, which ran +# xinput after a resume as a named person on a guessed display. +Section "InputClass" + Identifier "asus-zephyrus-g14 touchpad" + MatchIsTouchpad "on" + Option "Tapping" "on" + Option "NaturalScrolling" "true" + Option "AccelSpeed" "0.15" +EndSection diff --git a/modules/asus-zephyrus-g14/templates/after-wake b/modules/asus-zephyrus-g14/templates/after-wake new file mode 100644 index 00000000..af60a9d9 --- /dev/null +++ b/modules/asus-zephyrus-g14/templates/after-wake @@ -0,0 +1,7 @@ +# The touchpad's settings again after waking, in the operator's session: X applies the module's +# input class when the device appears, and this covers a wake that does not initialise it again. +# Runs as root from the power module; the reset itself runs as the session's owner. +sleep 2 +owner=$(ps -o user= -C i3 | head -n 1) +[ -n "$owner" ] && runuser -u "$owner" -- /usr/local/lib/asus-zephyrus-g14/bin/zephyrus-touchpad reset +true diff --git a/modules/asus-zephyrus-g14/templates/display-session b/modules/asus-zephyrus-g14/templates/display-session new file mode 100644 index 00000000..38f9e5bd --- /dev/null +++ b/modules/asus-zephyrus-g14/templates/display-session @@ -0,0 +1,30 @@ +# The laptop's own lines in i3 (module asus-zephyrus-g14, novox/hq ADR 0208, ADR 0210). Owned by the +# mesh: replaced at every push. Once the controller places contributions to node-display-session +# (ADR 0210), these become the module's contribution instead of a file in i3's directory. +# +# The keys the firmware turns into ordinary key presses. The vendor keys that reach no X client are +# triggerhappy's (/etc/triggerhappy/triggers.d/asus-g14.conf): M4 and Fn+F4/F5 for media, Fn+F7/F8 for +# the panel, Fn+F10 for the touchpad. The keyboard backlight (Fn+F2/F3) is the firmware's and asusd's. + +# Fn+F6, the screenshot key: the firmware sends Super+Shift+S. Released before it runs, because the +# screenshot grabs the pointer to select a region, which fails while the key is still held. +bindsym --release $mod+Shift+s exec --no-startup-id $XDG_CONFIG_HOME/i3/scripts/screenshot.sh + +# Fn+F9, the display key: the firmware sends Super+P. Odd workspaces to the panel, even ones to the +# external output. +bindsym $mod+p exec --no-startup-id /usr/local/lib/asus-zephyrus-g14/bin/zephyrus-display order + +# The keyboard backlight's level, shown when it changes. +exec --no-startup-id /usr/local/lib/asus-zephyrus-g14/bin/zephyrus-kbd-notify +# The laptop's own lines in i3 (module asus-zephyrus-g14, novox/hq ADR 0208, ADR 0210). Owned by the +# mesh: replaced at every push. Once the controller places contributions to node-display-session +# (ADR 0210), these become the module's contribution instead of a file in i3's directory. +# +# The model's panel and touchpad. + +# The internal panel is the primary output, where the bars' tray goes. Which monitors are on and where +# is the display server's (autorandr, run at every session start). +exec --no-startup-id /usr/local/lib/asus-zephyrus-g14/bin/zephyrus-display primary + +# The touchpad's settings again, by hand. X applies them itself whenever the device appears. +bindsym $mod+Shift+x exec --no-startup-id /usr/local/lib/asus-zephyrus-g14/bin/zephyrus-touchpad reset diff --git a/modules/asus-zephyrus-g14/templates/hotkeys-triggers b/modules/asus-zephyrus-g14/templates/hotkeys-triggers new file mode 100644 index 00000000..5e4d165c --- /dev/null +++ b/modules/asus-zephyrus-g14/templates/hotkeys-triggers @@ -0,0 +1,10 @@ +# The ROG Zephyrus G14's vendor keys, which reach no X client: media (the M-keys), panel brightness, +# and the touchpad key. Each runs this module's own script, as the operator's account. +KEY_PROG1 1 /usr/local/lib/asus-zephyrus-g14/bin/zephyrus-session /usr/local/lib/asus-zephyrus-g14/bin/zephyrus-media play-pause +KEY_PROG3 1 /usr/local/lib/asus-zephyrus-g14/bin/zephyrus-session /usr/local/lib/asus-zephyrus-g14/bin/zephyrus-media previous +KEY_PROG4 1 /usr/local/lib/asus-zephyrus-g14/bin/zephyrus-session /usr/local/lib/asus-zephyrus-g14/bin/zephyrus-media next +KEY_BRIGHTNESSDOWN 1 /usr/local/lib/asus-zephyrus-g14/bin/zephyrus-backlight - +KEY_BRIGHTNESSDOWN 2 /usr/local/lib/asus-zephyrus-g14/bin/zephyrus-backlight - +KEY_BRIGHTNESSUP 1 /usr/local/lib/asus-zephyrus-g14/bin/zephyrus-backlight + +KEY_BRIGHTNESSUP 2 /usr/local/lib/asus-zephyrus-g14/bin/zephyrus-backlight + +KEY_F21 1 /usr/local/lib/asus-zephyrus-g14/bin/zephyrus-touchpad reset diff --git a/modules/asus-zephyrus-g14/templates/modprobe.d/g14-nvidia-power.conf b/modules/asus-zephyrus-g14/templates/modprobe.d/g14-nvidia-power.conf new file mode 100644 index 00000000..78941022 --- /dev/null +++ b/modules/asus-zephyrus-g14/templates/modprobe.d/g14-nvidia-power.conf @@ -0,0 +1,16 @@ +# Managed by the mesh (module asus-zephyrus-g14). Replaced on every push; edit the catalogue instead. +# +# The discrete GPU's driver options on the ROG Zephyrus G14 (GA403, RTX 40 series, hybrid graphics). +# +# NVreg_DynamicPowerManagement=0x00 turns runtime D3 off. With it on, a change of power source sends +# the driver an ACPI notification it fails to handle on this model ("RmHandleDNotifierEvent: Failed to +# handle ACPI D-Notifier event, status=0x62"), and the GPU stops making progress until the machine is +# powered off. Off costs a few idle watts in hybrid mode and keeps the machine up. +# +# NVreg_PreserveVideoMemoryAllocations=1 saves video memory across suspend, so what used the GPU still +# works after waking. It needs nvidia-suspend, -hibernate and -resume to run around a sleep, which this +# module's drop-ins on the sleep services ask for. +# +# A change here applies when the driver next loads: at the next boot. +options nvidia NVreg_PreserveVideoMemoryAllocations=1 +options nvidia NVreg_DynamicPowerManagement=0x00 diff --git a/modules/asus-zephyrus-g14/templates/modprobe.d/video-brightness-switch.conf b/modules/asus-zephyrus-g14/templates/modprobe.d/video-brightness-switch.conf new file mode 100644 index 00000000..5b2de1d5 --- /dev/null +++ b/modules/asus-zephyrus-g14/templates/modprobe.d/video-brightness-switch.conf @@ -0,0 +1,6 @@ +# Managed by the mesh (module asus-zephyrus-g14). Replaced on every push; edit the catalogue instead. +# +# The ACPI video driver does not change the backlight itself on the brightness keys: on this model it +# moves the wrong one. The keys are triggerhappy's (see /etc/triggerhappy/triggers.d/asus-g14.conf). +# Applies when the module next loads: at the next boot. +options video brightness_switch_enabled=0 diff --git a/modules/asus-zephyrus-g14/templates/systemd/system/nvidia-powerd.service.d/asus-zephyrus-g14.conf b/modules/asus-zephyrus-g14/templates/systemd/system/nvidia-powerd.service.d/asus-zephyrus-g14.conf new file mode 100644 index 00000000..4466dd08 --- /dev/null +++ b/modules/asus-zephyrus-g14/templates/systemd/system/nvidia-powerd.service.d/asus-zephyrus-g14.conf @@ -0,0 +1,7 @@ +# Managed by the mesh (module asus-zephyrus-g14). Replaced on every push; edit the catalogue instead. +# +# nvidia-powerd (Dynamic Boost) was the first error in the chain that hung this model's GPU on a change +# of power source, and asusd starts it on mains. It runs only when the kernel command line says +# zephyrus.nvidia-powerd — an explicit opt-in, at boot. +[Unit] +ConditionKernelCommandLine=zephyrus.nvidia-powerd diff --git a/modules/asus-zephyrus-g14/templates/systemd/system/systemd-hibernate.service.d/asus-zephyrus-g14-nvidia.conf b/modules/asus-zephyrus-g14/templates/systemd/system/systemd-hibernate.service.d/asus-zephyrus-g14-nvidia.conf new file mode 100644 index 00000000..1f56e3b8 --- /dev/null +++ b/modules/asus-zephyrus-g14/templates/systemd/system/systemd-hibernate.service.d/asus-zephyrus-g14-nvidia.conf @@ -0,0 +1,7 @@ +# Managed by the mesh (module asus-zephyrus-g14). Replaced on every push; edit the catalogue instead. +# +# The NVIDIA driver's own sleep actions, asked for by the sleep itself rather than enabled as +# links: the mesh declares files and never makes links (novox/hq ADR 0012), and the host's service +# shape must not start these units by hand, which would put the GPU to sleep with the machine awake. +[Unit] +Wants=nvidia-hibernate.service nvidia-resume.service diff --git a/modules/asus-zephyrus-g14/templates/systemd/system/systemd-suspend-then-hibernate.service.d/asus-zephyrus-g14-nvidia.conf b/modules/asus-zephyrus-g14/templates/systemd/system/systemd-suspend-then-hibernate.service.d/asus-zephyrus-g14-nvidia.conf new file mode 100644 index 00000000..5813188a --- /dev/null +++ b/modules/asus-zephyrus-g14/templates/systemd/system/systemd-suspend-then-hibernate.service.d/asus-zephyrus-g14-nvidia.conf @@ -0,0 +1,7 @@ +# Managed by the mesh (module asus-zephyrus-g14). Replaced on every push; edit the catalogue instead. +# +# The NVIDIA driver's own sleep actions, asked for by the sleep itself rather than enabled as +# links: the mesh declares files and never makes links (novox/hq ADR 0012), and the host's service +# shape must not start these units by hand, which would put the GPU to sleep with the machine awake. +[Unit] +Wants=nvidia-suspend-then-hibernate.service nvidia-resume.service diff --git a/modules/asus-zephyrus-g14/templates/systemd/system/systemd-suspend.service.d/asus-zephyrus-g14-nvidia.conf b/modules/asus-zephyrus-g14/templates/systemd/system/systemd-suspend.service.d/asus-zephyrus-g14-nvidia.conf new file mode 100644 index 00000000..92a01117 --- /dev/null +++ b/modules/asus-zephyrus-g14/templates/systemd/system/systemd-suspend.service.d/asus-zephyrus-g14-nvidia.conf @@ -0,0 +1,7 @@ +# Managed by the mesh (module asus-zephyrus-g14). Replaced on every push; edit the catalogue instead. +# +# The NVIDIA driver's own sleep actions, asked for by the sleep itself rather than enabled as +# links: the mesh declares files and never makes links (novox/hq ADR 0012), and the host's service +# shape must not start these units by hand, which would put the GPU to sleep with the machine awake. +[Unit] +Wants=nvidia-suspend.service nvidia-resume.service diff --git a/modules/asus-zephyrus-g14/templates/udev/rules.d/90-backlight.rules b/modules/asus-zephyrus-g14/templates/udev/rules.d/90-backlight.rules new file mode 100644 index 00000000..0a9a894d --- /dev/null +++ b/modules/asus-zephyrus-g14/templates/udev/rules.d/90-backlight.rules @@ -0,0 +1,5 @@ +# Managed by the mesh (module asus-zephyrus-g14). Replaced on every push; edit the catalogue instead. +# +# The backlights are writable by the video group, so the brightness keys and the module's brightness tool +# move the panel without root. +ACTION=="add", SUBSYSTEM=="backlight", RUN+="/usr/bin/chgrp video /sys/class/backlight/%k/brightness", RUN+="/usr/bin/chmod g+w /sys/class/backlight/%k/brightness" diff --git a/modules/baserow/files/config.json b/modules/baserow/files/config.json new file mode 100644 index 00000000..955ac244 --- /dev/null +++ b/modules/baserow/files/config.json @@ -0,0 +1,4 @@ +{ + "password": "${secret:admin}", + "host": "${bound:route:name}" +} diff --git a/modules/baserow/files/server.env b/modules/baserow/files/server.env new file mode 100644 index 00000000..7b716b42 --- /dev/null +++ b/modules/baserow/files/server.env @@ -0,0 +1,7 @@ +DATABASE_HOST=${bound:postgres-database:at} +DATABASE_PORT=${bound:postgres-database:port} +DATABASE_NAME=${bound:postgres-database:as} +DATABASE_USER=${bound:postgres-database:as} +DATABASE_PASSWORD_FILE=/run/secrets/database +DISABLE_EMBEDDED_PSQL=true +BASEROW_PUBLIC_URL=https://${bound:route:name} diff --git a/modules/baserow/module.json b/modules/baserow/module.json index 366ab9c0..e19568b6 100644 --- a/modules/baserow/module.json +++ b/modules/baserow/module.json @@ -46,6 +46,26 @@ } ] }, + "config": { + "host": { + "type": "string", + "description": "The Host header the module's tools send to Baserow, a host name; the file's default is the name the module's route was given.", + "trusted": true, + "default": "${bound:route:name}" + }, + "email": { + "type": "string", + "description": "The email address of the Baserow account the module's tools sign in as; unset, the code falls back to its environment.", + "trusted": true, + "default": "" + }, + "url": { + "type": "string", + "description": "The address the module's tools reach Baserow at, as a URL; unset, the code falls back to its environment, else this machine's Baserow on port 80.", + "trusted": true, + "default": "" + } + }, "resources": [ { "id": "mesh-state", @@ -70,7 +90,7 @@ "type": "file", "path": "${dir:state}/server.env", "mode": "0600", - "content": "DATABASE_HOST=${bound:postgres-database:at}\nDATABASE_PORT=${bound:postgres-database:port}\nDATABASE_NAME=${bound:postgres-database:as}\nDATABASE_USER=${bound:postgres-database:as}\nDATABASE_PASSWORD_FILE=/run/secrets/database\nDISABLE_EMBEDDED_PSQL=true\nBASEROW_PUBLIC_URL=https://${bound:route:name}\n" + "template": "files/server.env" }, { "id": "net", @@ -102,8 +122,9 @@ "type": "file", "path": "${dir:mesh-state}/config.json", "mode": "0600", - "content": "{\n \"password\": \"${secret:admin}\",\n \"host\": \"${bound:route:name}\"\n}\n", + "template": "files/config.json", "merge": "json", + "takes-config": ["host", "email", "url"], "protected": ["password"] } ], @@ -123,6 +144,11 @@ "MESH_BASEROW_URL": "http://127.0.0.1:${port:80}", "MESH_BASEROW_CONFIG_FILE": "${dir:mesh-state}/config.json" } + }, + { + "name": "files", + "kind": "archive", + "from": "files" } ] } diff --git a/modules/betterbird/README.md b/modules/betterbird/README.md index 4e2a2acd..628f32e3 100644 --- a/modules/betterbird/README.md +++ b/modules/betterbird/README.md @@ -208,7 +208,7 @@ node set up from nothing, the declared accounts are written at the second start Five more settings, mesh-wide with a node's layer over each (an entry set to `null` in a node's layer is not on that node). `identities`, `calendars`, `address-books` and `filters` reach the machine as -`~/.config/betterbird/configuration.json`, trusted as `accounts` is; `tags` as `tags.json`, not trusted (§5). +`~/.config/betterbird/configuration.json`, trusted as `accounts` is; `tags` as `tags.json`, trusted too since issue 751 (the operator's decision), so a change to it is the operator's to approve. **No value holds a password, a token or a cookie**: a key shaped like one is refused, and so is an address whose user part carries a password (§2). A setting holds no line break, so a signature is a list of its lines. diff --git a/modules/betterbird/cmd/betterbird-tools/configuration.go b/modules/betterbird/cmd/betterbird-tools/configuration.go index 9a1620d3..97e0ad52 100644 --- a/modules/betterbird/cmd/betterbird-tools/configuration.go +++ b/modules/betterbird/cmd/betterbird-tools/configuration.go @@ -7,7 +7,8 @@ package main // // - ~/.config/betterbird/configuration.json: identities, calendars, address-books and filters, trusted as // accounts is (§5), because each names where a password, a reply or forwarded mail goes; -// - ~/.config/betterbird/tags.json: tags, which names no server and sends no mail, and is not trusted. +// - ~/.config/betterbird/tags.json: tags, which names no server and sends no mail; trusted too since issue 751, +// so a change to it is the operator's to approve. // // An entry is keyed by a name of the operator's (lower-case letters and digits, as an account's), a tag by its // own key. A node's layer may set an entry to null to take it out on that node, and may give an identity, a diff --git a/modules/betterbird/cmd/betterbird-tools/manifest_test.go b/modules/betterbird/cmd/betterbird-tools/manifest_test.go index 1a7846d0..37e420b5 100644 --- a/modules/betterbird/cmd/betterbird-tools/manifest_test.go +++ b/modules/betterbird/cmd/betterbird-tools/manifest_test.go @@ -25,7 +25,7 @@ type manifest struct { Upgrade map[string]string `json:"upgrade"` Requires []string `json:"requires"` Invokes []string `json:"invokes"` - Settings map[string]any `json:"settings"` + Config map[string]any `json:"config"` Tools []string `json:"tools"` Resources []map[string]any `json:"resources"` Build struct { @@ -114,13 +114,21 @@ func TestItPlacesBetterbirdWhereTheCommunityPackageDidAndOnlyAfterItIsGone(t *te } } -// ADR 0329 §4–§6, ADR 0326, issue 586: every setting reaches the machine as a mergeable file of the account's that -// names the keys it takes; four kinds and accounts and prefs are trusted, tags is not; no setting has a default in the -// definition, since each is the operator's own value; and each file's default declares nothing. -func TestTheSettingsAreMergeableFilesThatNameTheirKeysAndTagsAloneIsUntrusted(t *testing.T) { +// ADR 0329 §4–§6, ADR 0326, issue 586, ADR 0348: every value reaches the machine as a mergeable file of the +// account's that names the keys it takes, and every one is trusted, tags too since issue 751 (the operator's +// decision); each is an object whose default is the file's own, empty, never required, since a node without the +// operator's setup still runs Betterbird; and each file's default declares nothing. +func TestTheValuesAreTrustedMergeableFilesThatNameTheirKeys(t *testing.T) { m, _ := readManifest(t) - if len(m.Settings) != 0 { - t.Fatalf("no setting has a default in the definition: the operator's setup is the settings' mesh-wide layer (issue 586, ADR 0262): %v", m.Settings) + if _, said := m.Config["tags"]; !said || len(m.Config) != 7 { + t.Fatalf("the seven values are declared under config: %v", m.Config) + } + for key, raw := range m.Config { + d, _ := raw.(map[string]any) + def, _ := d["default"].(map[string]any) + if d["type"] != "object" || d["trusted"] != true || d["required"] != nil || def == nil || len(def) != 0 { + t.Errorf("%s is a trusted object whose default is empty and which is not required: %v", key, d) + } } files := map[string]struct { path, mode string @@ -130,7 +138,7 @@ func TestTheSettingsAreMergeableFilesThatNameTheirKeysAndTagsAloneIsUntrusted(t "prefs": {prefsFile, "0644", []any{"prefs"}, true}, "accounts": {accountsFile, "0600", []any{"accounts"}, true}, "configuration": {configurationFile, "0600", []any{"identities", "calendars", "address-books", "filters"}, true}, - "tags": {tagsFile, "0644", []any{"tags"}, false}, + "tags": {tagsFile, "0644", []any{"tags"}, true}, } merging := 0 for _, r := range m.Resources { @@ -144,27 +152,27 @@ func TestTheSettingsAreMergeableFilesThatNameTheirKeysAndTagsAloneIsUntrusted(t for id, want := range files { r := resource(t, m, id) if r["path"] != "${machine:account-home}/"+want.path || r["merge"] != "json" || r["trusted"] != want.trusted || - r["mode"] != want.mode || r["owner"] != "${machine:account}" || !reflect.DeepEqual(r["takes-settings"], want.takes) { + r["mode"] != want.mode || r["owner"] != "${machine:account}" || !reflect.DeepEqual(r["takes-config"], want.takes) { t.Errorf("%s: %v", id, r) } var content map[string]any - if err := json.Unmarshal([]byte(r["content"].(string)), &content); err != nil || len(content) != len(want.takes) { + if err := json.Unmarshal([]byte(templateText(t, r)), &content); err != nil || len(content) != len(want.takes) { t.Errorf("%s's default holds each key it takes, empty: %v %v", id, content, err) } } - got, err := parseAccounts([]byte(resource(t, m, "accounts")["content"].(string))) + got, err := parseAccounts([]byte(templateText(t, resource(t, m, "accounts")))) if err != nil || len(got) != 0 { t.Fatalf("its default declares no account: %v %v", got, err) } - prefs, err := parseDeclaredPrefs([]byte(resource(t, m, "prefs")["content"].(string))) + prefs, err := parseDeclaredPrefs([]byte(templateText(t, resource(t, m, "prefs")))) if err != nil || len(prefs) != 0 { t.Fatalf("its default declares no preference: %v %v", prefs, err) } var c Configuration - if err := decodeInto([]byte(resource(t, m, "configuration")["content"].(string)), configurationFile, &c); err != nil { + if err := decodeInto([]byte(templateText(t, resource(t, m, "configuration"))), configurationFile, &c); err != nil { t.Fatal(err) } - if err := decodeInto([]byte(resource(t, m, "tags")["content"].(string)), tagsFile, &c); err != nil { + if err := decodeInto([]byte(templateText(t, resource(t, m, "tags"))), tagsFile, &c); err != nil { t.Fatal(err) } } @@ -268,13 +276,13 @@ func TestThePolicyTurnsTheUpdaterOffAndInstallsTheMeshsOwlEverywhere(t *testing. func TestTheStartPreparesThenStartsBetterbirdAndTheLauncherUsesIt(t *testing.T) { m, _ := readManifest(t) start := resource(t, m, "start") - content := start["content"].(string) + content := templateText(t, start) if start["path"] != "/usr/local/bin/betterbird" || start["mode"] != "0755" || !strings.Contains(content, "\n"+preparation+" prepare ||") || !strings.HasSuffix(content, "\nexec "+startBin+" \"$@\"\n") { t.Fatalf("the start runs the preparation, says when it did not complete, and starts Betterbird either way: %v", start) } launcher := resource(t, m, "launcher") - lc := launcher["content"].(string) + lc := templateText(t, launcher) if launcher["path"] != "/usr/local/share/applications/eu.betterbird.Betterbird.desktop" || !strings.Contains(lc, "\nExec=/usr/local/bin/betterbird %u\n") || !strings.Contains(lc, "\nIcon="+installDir+"/") || !strings.Contains(lc, "\nStartupWMClass=eu.betterbird.Betterbird\n") { diff --git a/modules/betterbird/cmd/betterbird-tools/templates_test.go b/modules/betterbird/cmd/betterbird-tools/templates_test.go new file mode 100644 index 00000000..7d0718b5 --- /dev/null +++ b/modules/betterbird/cmd/betterbird-tools/templates_test.go @@ -0,0 +1,64 @@ +package main + +import ( + "encoding/json" + "os" + "path/filepath" + "testing" +) + +// The module's directory, where its manifest and its files/ are. +const moduleDir = "../.." + +// templateText is the text a file resource places on the machine: its template's, in the module's files/ +// (novox/hq ADR 0349 §2). A resource that names no template, or a template that is not there, fails the test. +func templateText(t *testing.T, r map[string]any) string { + t.Helper() + name, _ := r["template"].(string) + if name == "" { + t.Fatalf("resource %v names no template", r["id"]) + } + raw, err := os.ReadFile(filepath.Join(moduleDir, filepath.FromSlash(name))) + if err != nil { + t.Fatalf("resource %v names %s: %v", r["id"], name, err) + } + return string(raw) +} + +// No file resource carries its text inline: each names its template in files/, and the build packs files/ +// into the one archive the controller reads the templates back from (novox/hq ADR 0349 §2). +func TestEveryFileTakesItsTextFromATemplate(t *testing.T) { + raw, err := os.ReadFile(filepath.Join(moduleDir, "module.json")) + if err != nil { + t.Fatal(err) + } + var m struct { + Resources []map[string]any `json:"resources"` + Build struct { + Artifacts []map[string]any `json:"artifacts"` + } `json:"build"` + } + if err := json.Unmarshal(raw, &m); err != nil { + t.Fatal(err) + } + for _, r := range m.Resources { + if _, inline := r["content"]; inline { + t.Errorf("resource %v keeps its text inline: keep it in files/ and name it as the template", r["id"]) + } + if r["type"] == "file" { + templateText(t, r) + } + } + packs := 0 + for _, a := range m.Build.Artifacts { + if a["kind"] == "archive" && a["from"] == "files" { + packs++ + if a["name"] != "files" { + t.Errorf("the templates archive is named %v, not files", a["name"]) + } + } + } + if packs != 1 { + t.Errorf("the build packs files/ into %d archives, not one", packs) + } +} diff --git a/modules/betterbird/files/accounts.json b/modules/betterbird/files/accounts.json new file mode 100644 index 00000000..e182cc3d --- /dev/null +++ b/modules/betterbird/files/accounts.json @@ -0,0 +1,3 @@ +{ + "accounts": {} +} diff --git a/modules/betterbird/files/betterbird b/modules/betterbird/files/betterbird new file mode 100755 index 00000000..3ce088a8 --- /dev/null +++ b/modules/betterbird/files/betterbird @@ -0,0 +1,10 @@ +#!/bin/sh +# Betterbird's start, written by the mesh (module betterbird, novox/hq ADR 0316). Replaced at every push. +# +# Before Betterbird starts, the module's tools place its user.js in the profile (the settings prefs, accounts, identities, calendars, +# address-books and tags) and write what Betterbird rewrites itself: the account lists and each declared account's filters. They do +# it only while no Betterbird of this account runs, because Betterbird rewrites its saved preferences when it quits; a +# second start while one runs only hands over to it. What the preparation could not do it says here, and +# betterbird_check says it too. Betterbird starts either way: its configuration never holds the mail back. +/usr/local/lib/mesh-betterbird/betterbird-tools prepare || echo "betterbird: the mesh's preparation did not complete; betterbird_check says why" >&2 +exec /opt/betterbird/betterbird "$@" diff --git a/modules/betterbird/files/configuration.json b/modules/betterbird/files/configuration.json new file mode 100644 index 00000000..70204462 --- /dev/null +++ b/modules/betterbird/files/configuration.json @@ -0,0 +1,6 @@ +{ + "identities": {}, + "calendars": {}, + "address-books": {}, + "filters": {} +} diff --git a/modules/betterbird/files/eu.betterbird.Betterbird.desktop b/modules/betterbird/files/eu.betterbird.Betterbird.desktop new file mode 100644 index 00000000..e409fcf8 --- /dev/null +++ b/modules/betterbird/files/eu.betterbird.Betterbird.desktop @@ -0,0 +1,25 @@ +[Desktop Entry] +Version=1.0 +Type=Application +Name=Betterbird +GenericName=Mail Client +Comment=Send and receive mail with Betterbird +Exec=/usr/local/bin/betterbird %u +Icon=/opt/betterbird/chrome/icons/default/default128.png +Terminal=false +Categories=Network;Email; +MimeType=message/rfc822;x-scheme-handler/mailto;application/x-xpinstall;application/x-extension-ics;text/calendar;text/vcard;text/x-vcard;x-scheme-handler/webcal;x-scheme-handler/webcals;x-scheme-handler/mid; +StartupNotify=true +StartupWMClass=eu.betterbird.Betterbird +Keywords=Email;E-mail;Calendar;Thunderbird; +Actions=ComposeMessage;OpenAddressBook; +# Written by the mesh (module betterbird, novox/hq ADR 0316 §3), in place of the community package's entry: +# it starts Betterbird through the module's start, and names the icon in the installation, with no link. + +[Desktop Action ComposeMessage] +Name=Write new message +Exec=/usr/local/bin/betterbird -compose + +[Desktop Action OpenAddressBook] +Name=Open address book +Exec=/usr/local/bin/betterbird -addressbook diff --git a/modules/betterbird/files/prefs.json b/modules/betterbird/files/prefs.json new file mode 100644 index 00000000..45fd9732 --- /dev/null +++ b/modules/betterbird/files/prefs.json @@ -0,0 +1,3 @@ +{ + "prefs": {} +} diff --git a/modules/betterbird/files/tags.json b/modules/betterbird/files/tags.json new file mode 100644 index 00000000..4772f099 --- /dev/null +++ b/modules/betterbird/files/tags.json @@ -0,0 +1,3 @@ +{ + "tags": {} +} diff --git a/modules/betterbird/module.json b/modules/betterbird/module.json index 45383ab4..19319ef4 100644 --- a/modules/betterbird/module.json +++ b/modules/betterbird/module.json @@ -33,6 +33,50 @@ "betterbird_drift", "betterbird_reminder_test" ], + "config": { + "prefs": { + "type": "object", + "description": "Betterbird's preferences the operator sets, by their own names (mailnews.*, calendar.* and the like), laid into prefs.json; an account's, a server's and a tag's are refused, as accounts, the four kinds and tags own them (novox/hq ADR 0329 §6).", + "trusted": true, + "default": {} + }, + "accounts": { + "type": "object", + "description": "The mail accounts Betterbird is set up with, each by a name of the operator's: its address, its sender's name and its incoming and outgoing servers; no password, token or cookie (novox/hq ADR 0329 §2).", + "trusted": true, + "default": {} + }, + "identities": { + "type": "object", + "description": "The identities beside an account's own, each by a name of the operator's: the address and name it sends as, its signature as a list of lines, and the profile's key to adopt on a node.", + "trusted": true, + "default": {} + }, + "calendars": { + "type": "object", + "description": "The calendars Betterbird shows, each by a name of the operator's: where it is and how it is reached, and the profile's key to adopt on a node.", + "trusted": true, + "default": {} + }, + "address-books": { + "type": "object", + "description": "The address books Betterbird uses, each by a name of the operator's: where it is and how it is reached, and the profile's key to adopt on a node.", + "trusted": true, + "default": {} + }, + "filters": { + "type": "object", + "description": "The message filters of each account, by account: what each matches and what it does with the mail.", + "trusted": true, + "default": {} + }, + "tags": { + "type": "object", + "description": "The tags Betterbird offers, each by its own key: its name, colour and order; trusted since issue 751, so a change to it is the operator's to approve.", + "trusted": true, + "default": {} + } + }, "resources": [ { "id": "configuration-dir", @@ -48,10 +92,10 @@ "owner": "${machine:account}", "mode": "0644", "merge": "json", - "takes-settings": [ + "takes-config": [ "prefs" ], - "content": "{\n \"prefs\": {}\n}\n", + "template": "files/prefs.json", "trusted": true }, { @@ -61,10 +105,10 @@ "owner": "${machine:account}", "mode": "0600", "merge": "json", - "takes-settings": [ + "takes-config": [ "accounts" ], - "content": "{\n \"accounts\": {}\n}\n", + "template": "files/accounts.json", "trusted": true }, { @@ -74,13 +118,13 @@ "owner": "${machine:account}", "mode": "0600", "merge": "json", - "takes-settings": [ + "takes-config": [ "identities", "calendars", "address-books", "filters" ], - "content": "{\n \"identities\": {},\n \"calendars\": {},\n \"address-books\": {},\n \"filters\": {}\n}\n", + "template": "files/configuration.json", "trusted": true }, { @@ -90,11 +134,11 @@ "owner": "${machine:account}", "mode": "0644", "merge": "json", - "takes-settings": [ + "takes-config": [ "tags" ], - "content": "{\n \"tags\": {}\n}\n", - "trusted": false + "template": "files/tags.json", + "trusted": true }, { "id": "community-package", @@ -137,14 +181,14 @@ "type": "file", "path": "/usr/local/bin/betterbird", "mode": "0755", - "content": "#!/bin/sh\n# Betterbird's start, written by the mesh (module betterbird, novox/hq ADR 0316). Replaced at every push.\n#\n# Before Betterbird starts, the module's tools place its user.js in the profile (the settings prefs, accounts, identities, calendars,\n# address-books and tags) and write what Betterbird rewrites itself: the account lists and each declared account's filters. They do\n# it only while no Betterbird of this account runs, because Betterbird rewrites its saved preferences when it quits; a\n# second start while one runs only hands over to it. What the preparation could not do it says here, and\n# betterbird_check says it too. Betterbird starts either way: its configuration never holds the mail back.\n/usr/local/lib/mesh-betterbird/betterbird-tools prepare || echo \"betterbird: the mesh's preparation did not complete; betterbird_check says why\" >&2\nexec /opt/betterbird/betterbird \"$@\"\n" + "template": "files/betterbird" }, { "id": "launcher", "type": "file", "path": "/usr/local/share/applications/eu.betterbird.Betterbird.desktop", "mode": "0644", - "content": "[Desktop Entry]\nVersion=1.0\nType=Application\nName=Betterbird\nGenericName=Mail Client\nComment=Send and receive mail with Betterbird\nExec=/usr/local/bin/betterbird %u\nIcon=/opt/betterbird/chrome/icons/default/default128.png\nTerminal=false\nCategories=Network;Email;\nMimeType=message/rfc822;x-scheme-handler/mailto;application/x-xpinstall;application/x-extension-ics;text/calendar;text/vcard;text/x-vcard;x-scheme-handler/webcal;x-scheme-handler/webcals;x-scheme-handler/mid;\nStartupNotify=true\nStartupWMClass=eu.betterbird.Betterbird\nKeywords=Email;E-mail;Calendar;Thunderbird;\nActions=ComposeMessage;OpenAddressBook;\n# Written by the mesh (module betterbird, novox/hq ADR 0316 §3), in place of the community package's entry:\n# it starts Betterbird through the module's start, and names the icon in the installation, with no link.\n\n[Desktop Action ComposeMessage]\nName=Write new message\nExec=/usr/local/bin/betterbird -compose\n\n[Desktop Action OpenAddressBook]\nName=Open address book\nExec=/usr/local/bin/betterbird -addressbook\n" + "template": "files/eu.betterbird.Betterbird.desktop" } ], "build": { @@ -191,6 +235,11 @@ "name": "defaults", "kind": "archive", "from": "install/defaults-pref" + }, + { + "name": "files", + "kind": "archive", + "from": "files" } ] } diff --git a/modules/build-agent/files/build-agent.env b/modules/build-agent/files/build-agent.env new file mode 100644 index 00000000..15ee1bbb --- /dev/null +++ b/modules/build-agent/files/build-agent.env @@ -0,0 +1,8 @@ +MESH_BROKER_FILE=/run/mesh/broker +MESH_NODE=${machine:name} +MESH_REGISTRY=${bound:artifact-store:at}:${bound:artifact-store:port} +MESH_PACKAGE_BINDING=/run/mesh/package-registry.json +MESH_NPM_TOKEN_FILE=/run/mesh/package-registry.secret +MESH_GENESIS_PACKAGE_REGISTRY=/run/mesh/genesis-package-registry.json +MESH_WORKSPACE=${dir:workspace} +MESH_JUDGE_CACHE=${dir:judge-cache} diff --git a/modules/build-agent/manifest_test.go b/modules/build-agent/manifest_test.go index 06e22fb1..59e041d9 100644 --- a/modules/build-agent/manifest_test.go +++ b/modules/build-agent/manifest_test.go @@ -75,7 +75,10 @@ func TestTheRegistrysBindsAndSecretsAreOnlyWhenBound(t *testing.T) { t.Fatalf("the agent is not told where the registry is once it binds: binds %v, secrets %v", m.Binds, m.Secrets) } for _, r := range m.Resources { - content, _ := r["content"].(string) + if r["type"] != "file" { + continue + } + content := templateText(t, r) if strings.Contains(content, "${bound:npm-package-registry:") || strings.Contains(content, "${secret:npm-package-registry}") { if r["only-when-bound"] != "npm-package-registry" { t.Errorf("%v names the registry's binding and is delivered while it is absent", r["id"]) @@ -106,7 +109,7 @@ func TestTheGenesisDocumentIsDeclaredAndIssuedOutside(t *testing.T) { inside := "/run/mesh/" + strings.TrimPrefix(own.Path, "${dir:mesh-state}/") for _, r := range m.Resources { if r["id"] == "agent-env" { - content, _ := r["content"].(string) + content := templateText(t, r) if !strings.Contains(content, "MESH_GENESIS_PACKAGE_REGISTRY="+inside+"\n") { t.Errorf("the agent is not told where the genesis document is (%s):\n%s", inside, content) } @@ -133,10 +136,10 @@ func TestTheJudgeCacheIsADirectoryOfItsOwnMountedAtTheSamePath(t *testing.T) { } `json:"own"` } `json:"data"` Resources []struct { - ID string `json:"id"` - Type string `json:"type"` - Content string `json:"content"` - Volumes []string `json:"volumes"` + ID string `json:"id"` + Type string `json:"type"` + Template string `json:"template"` + Volumes []string `json:"volumes"` } `json:"resources"` } if err := json.Unmarshal(raw, &m); err != nil { @@ -153,7 +156,7 @@ func TestTheJudgeCacheIsADirectoryOfItsOwnMountedAtTheSamePath(t *testing.T) { case r.ID == "judge-cache" && r.Type == "directory": dir = true case r.ID == "agent-env": - env = strings.Contains(r.Content, "MESH_JUDGE_CACHE=${dir:judge-cache}\n") + env = strings.Contains(templateText(t, map[string]any{"id": r.ID, "template": r.Template}), "MESH_JUDGE_CACHE=${dir:judge-cache}\n") case r.ID == "server": for _, v := range r.Volumes { if v == "${dir:judge-cache}:${dir:judge-cache}" { diff --git a/modules/build-agent/module.json b/modules/build-agent/module.json index 7cae885c..b960eacf 100644 --- a/modules/build-agent/module.json +++ b/modules/build-agent/module.json @@ -70,7 +70,7 @@ "type": "file", "path": "${dir:mesh-state}/build-agent.env", "mode": "0600", - "content": "MESH_BROKER_FILE=/run/mesh/broker\nMESH_NODE=${machine:name}\nMESH_REGISTRY=${bound:artifact-store:at}:${bound:artifact-store:port}\nMESH_PACKAGE_BINDING=/run/mesh/package-registry.json\nMESH_NPM_TOKEN_FILE=/run/mesh/package-registry.secret\nMESH_GENESIS_PACKAGE_REGISTRY=/run/mesh/genesis-package-registry.json\nMESH_WORKSPACE=${dir:workspace}\nMESH_JUDGE_CACHE=${dir:judge-cache}\n" + "template": "files/build-agent.env" }, { "id": "server", @@ -115,6 +115,11 @@ "repository": "novox/mesh-controller", "ref": "main" } + }, + { + "name": "files", + "kind": "archive", + "from": "files" } ], "on": [ diff --git a/modules/build-agent/templates_test.go b/modules/build-agent/templates_test.go new file mode 100644 index 00000000..896a9770 --- /dev/null +++ b/modules/build-agent/templates_test.go @@ -0,0 +1,64 @@ +package buildagent + +import ( + "encoding/json" + "os" + "path/filepath" + "testing" +) + +// The module's directory, where its manifest and its files/ are. +const moduleDir = "." + +// templateText is the text a file resource places on the machine: its template's, in the module's files/ +// (novox/hq ADR 0349 §2). A resource that names no template, or a template that is not there, fails the test. +func templateText(t *testing.T, r map[string]any) string { + t.Helper() + name, _ := r["template"].(string) + if name == "" { + t.Fatalf("resource %v names no template", r["id"]) + } + raw, err := os.ReadFile(filepath.Join(moduleDir, filepath.FromSlash(name))) + if err != nil { + t.Fatalf("resource %v names %s: %v", r["id"], name, err) + } + return string(raw) +} + +// No file resource carries its text inline: each names its template in files/, and the build packs files/ +// into the one archive the controller reads the templates back from (novox/hq ADR 0349 §2). +func TestEveryFileTakesItsTextFromATemplate(t *testing.T) { + raw, err := os.ReadFile(filepath.Join(moduleDir, "module.json")) + if err != nil { + t.Fatal(err) + } + var m struct { + Resources []map[string]any `json:"resources"` + Build struct { + Artifacts []map[string]any `json:"artifacts"` + } `json:"build"` + } + if err := json.Unmarshal(raw, &m); err != nil { + t.Fatal(err) + } + for _, r := range m.Resources { + if _, inline := r["content"]; inline { + t.Errorf("resource %v keeps its text inline: keep it in files/ and name it as the template", r["id"]) + } + if r["type"] == "file" { + templateText(t, r) + } + } + packs := 0 + for _, a := range m.Build.Artifacts { + if a["kind"] == "archive" && a["from"] == "files" { + packs++ + if a["name"] != "files" { + t.Errorf("the templates archive is named %v, not files", a["name"]) + } + } + } + if packs != 1 { + t.Errorf("the build packs files/ into %d archives, not one", packs) + } +} diff --git a/modules/ca-trust/files/anchor b/modules/ca-trust/files/anchor new file mode 100755 index 00000000..72576d2f --- /dev/null +++ b/modules/ca-trust/files/anchor @@ -0,0 +1,58 @@ +#!/bin/sh +# The mesh's internal certificate authority, trusted by this machine. +# +# Written by the mesh from the ca-trust module's manifest (novox/hq ADR 0147). +# Editing it here lasts until the next apply. +# +# There is no prior trust to verify the fetch against — this is the thing that +# establishes it — so it is made over the mesh's own private network, which is +# what authenticates it (novox/hq ADR 0098, the same reasoning that lets the +# route proxy fetch this root for itself). What comes back is checked here: a +# body that is not a certificate is refused now, rather than believed and then +# failed by whatever reads the trust store next. +set -eu + +ROOTS='https://${bound:internal-acme-ca:at}:${bound:internal-acme-ca:port}${bound:internal-acme-ca:roots}' +ANCHORS=/etc/ca-certificates/trust-source/anchors +ANCHOR="$ANCHORS/mesh-internal-ca.crt" + +# Arch's layout, said out loud rather than assumed: a machine that keeps its +# anchors elsewhere fails here, visibly, instead of writing a file nothing +# reads. That failure is the signal that this belongs in the host, where one +# operating system's difference lives (novox/hq ADR 0147, option 2). +[ -d "$ANCHORS" ] || { + echo "this machine keeps no trust anchors in $ANCHORS; ca-trust is written for that layout" >&2 + exit 1 +} + +case "${1:-}" in +install) + tmp=$(mktemp) + trap 'rm -f "$tmp"' EXIT + # The authority may still be starting, or this machine may have come up + # before it: two minutes of asking, then an honest failure. + n=0 + while [ "$n" -lt 60 ]; do + if curl --fail --silent --show-error --insecure --max-time 10 \ + --output "$tmp" "$ROOTS" && + grep -q 'BEGIN CERTIFICATE' "$tmp"; then + install -m 0644 "$tmp" "$ANCHOR" + update-ca-trust + exit 0 + fi + n=$((n + 1)) + sleep 2 + done + echo "the authority at $ROOTS did not serve a certificate within two minutes" >&2 + exit 1 + ;; +remove) + # What stopping the unit does, and therefore what being unassigned does. + rm -f "$ANCHOR" + update-ca-trust + ;; +*) + echo "usage: $(basename "$0") install|remove" >&2 + exit 2 + ;; +esac diff --git a/modules/ca-trust/files/mesh-ca-trust.service b/modules/ca-trust/files/mesh-ca-trust.service new file mode 100644 index 00000000..2411560c --- /dev/null +++ b/modules/ca-trust/files/mesh-ca-trust.service @@ -0,0 +1,16 @@ +[Unit] +Description=The mesh's internal certificate authority, trusted by this machine +# novox/hq ADR 0147. Starting this unit places the mesh's root among this +# machine's trust anchors; stopping it takes the root away again, which is what +# the host does when the module is no longer assigned here. +Wants=network-online.target +After=network-online.target + +[Service] +Type=oneshot +RemainAfterExit=yes +ExecStart=${dir:state}/anchor install +ExecStop=${dir:state}/anchor remove + +[Install] +WantedBy=multi-user.target diff --git a/modules/ca-trust/module.json b/modules/ca-trust/module.json index 1f7baf81..94e64fe4 100644 --- a/modules/ca-trust/module.json +++ b/modules/ca-trust/module.json @@ -32,14 +32,14 @@ "type": "file", "path": "${dir:state}/anchor", "mode": "0755", - "content": "#!/bin/sh\n# The mesh's internal certificate authority, trusted by this machine.\n#\n# Written by the mesh from the ca-trust module's manifest (novox/hq ADR 0147).\n# Editing it here lasts until the next apply.\n#\n# There is no prior trust to verify the fetch against \u2014 this is the thing that\n# establishes it \u2014 so it is made over the mesh's own private network, which is\n# what authenticates it (novox/hq ADR 0098, the same reasoning that lets the\n# route proxy fetch this root for itself). What comes back is checked here: a\n# body that is not a certificate is refused now, rather than believed and then\n# failed by whatever reads the trust store next.\nset -eu\n\nROOTS='https://${bound:internal-acme-ca:at}:${bound:internal-acme-ca:port}${bound:internal-acme-ca:roots}'\nANCHORS=/etc/ca-certificates/trust-source/anchors\nANCHOR=\"$ANCHORS/mesh-internal-ca.crt\"\n\n# Arch's layout, said out loud rather than assumed: a machine that keeps its\n# anchors elsewhere fails here, visibly, instead of writing a file nothing\n# reads. That failure is the signal that this belongs in the host, where one\n# operating system's difference lives (novox/hq ADR 0147, option 2).\n[ -d \"$ANCHORS\" ] || {\n\techo \"this machine keeps no trust anchors in $ANCHORS; ca-trust is written for that layout\" >&2\n\texit 1\n}\n\ncase \"${1:-}\" in\ninstall)\n\ttmp=$(mktemp)\n\ttrap 'rm -f \"$tmp\"' EXIT\n\t# The authority may still be starting, or this machine may have come up\n\t# before it: two minutes of asking, then an honest failure.\n\tn=0\n\twhile [ \"$n\" -lt 60 ]; do\n\t\tif curl --fail --silent --show-error --insecure --max-time 10 \\\n\t\t\t--output \"$tmp\" \"$ROOTS\" &&\n\t\t\tgrep -q 'BEGIN CERTIFICATE' \"$tmp\"; then\n\t\t\tinstall -m 0644 \"$tmp\" \"$ANCHOR\"\n\t\t\tupdate-ca-trust\n\t\t\texit 0\n\t\tfi\n\t\tn=$((n + 1))\n\t\tsleep 2\n\tdone\n\techo \"the authority at $ROOTS did not serve a certificate within two minutes\" >&2\n\texit 1\n\t;;\nremove)\n\t# What stopping the unit does, and therefore what being unassigned does.\n\trm -f \"$ANCHOR\"\n\tupdate-ca-trust\n\t;;\n*)\n\techo \"usage: $(basename \"$0\") install|remove\" >&2\n\texit 2\n\t;;\nesac\n" + "template": "files/anchor" }, { "id": "unit", "type": "file", "path": "/etc/systemd/system/mesh-ca-trust.service", "mode": "0644", - "content": "[Unit]\nDescription=The mesh's internal certificate authority, trusted by this machine\n# novox/hq ADR 0147. Starting this unit places the mesh's root among this\n# machine's trust anchors; stopping it takes the root away again, which is what\n# the host does when the module is no longer assigned here.\nWants=network-online.target\nAfter=network-online.target\n\n[Service]\nType=oneshot\nRemainAfterExit=yes\nExecStart=${dir:state}/anchor install\nExecStop=${dir:state}/anchor remove\n\n[Install]\nWantedBy=multi-user.target\n" + "template": "files/mesh-ca-trust.service" }, { "id": "trust", @@ -52,5 +52,14 @@ "unit" ] } - ] + ], + "build": { + "artifacts": [ + { + "name": "files", + "kind": "archive", + "from": "files" + } + ] + } } diff --git a/modules/claude-code/README.md b/modules/claude-code/README.md index 94e65a68..c0423f3d 100644 --- a/modules/claude-code/README.md +++ b/modules/claude-code/README.md @@ -43,7 +43,7 @@ whenever the node's tool runtime collects the module's tools: | file | holds | |---|---| | `managed-mcp.json` | the tool servers every session loads: the mesh's console as `mesh`, and the servers set in this module's `mcp_servers` setting. **Exclusive**: a server not listed here does not load — not one added with `claude mcp add`, not a project's `.mcp.json`, not a plugin's | -| `managed-settings.json` | the keys set in this module's `managed_settings` setting — at the controller's terminal alone (hq issue 340) — under the mesh's own keys: the repositories' attribution convention, the claude.ai connectors kept beside the managed servers, the key-helper while the node holds an API-key licence, and the two that name the `nox-mesh` marketplace and enable its plugin | +| `managed-settings.json` | the keys set in this module's `managed_settings` setting — at the controller's terminal alone (hq issue 340) — under the mesh's own keys: the repositories' attribution convention, the claude.ai connectors kept beside the managed servers, the key-helper while the node holds an API-key licence, git's credential file while the node holds the forge's read credential (below), and the two that name the `nox-mesh` marketplace and enable its plugin | | `CLAUDE.md` | how a session on this mesh works, this node's name and role, the conventions, the **"instead of" table** — then the instruction sections registered for every node and for this one | | `marketplace/` | the `nox-mesh` plugin (hq ADR 0216): the skills, subagents, commands and output styles registered for every node and for this one, offered in a session as `nox-mesh:` — and the **guard on the agent's shell** (`guard/`), the mesh's own hook and the plugin's only one (hq issue 340). Replaced whole, staged beside and swapped in | @@ -55,6 +55,17 @@ same content, and never through a directory that is a symbolic link. A placed fi alone, and one the person deleted stays deleted until the item is unregistered (hq ADR 0182). The status tool reads the rest of the home's items to report them; nothing else is read or written. +**The forge's read credential** (hq ADR 0365 §4, issue 737). The module requires `gitea-read`, optionally: the forge +mints this node's agent a token of its own account, `agent-`, scoped to read repositories alone, and the vault +keeps it for this module on this machine alone (issue 749); it arrives in the state as `gitea-read.secret`, git's +credential-store lines. Every few seconds the module places them at `~/.config/mesh/forge.credentials` in the agent's +home, mode 0600, written as the agent's account; puts the file back when git's store helper or a person changed it; +and takes it back when none is held. A value that is not credential lines (one the mesh made before the forge was +asked) is never placed. While one is held, `managed-settings.json` gives every session's environment +`GIT_CONFIG_COUNT`, `GIT_CONFIG_KEY_0=credential.helper` and `GIT_CONFIG_VALUE_0=store --file=''`: the +path, never the token. So `git clone` and `git fetch` of what the agent's account may read work with nothing typed, +printed or put on a command line; pushing stays `gitea_commit`'s, and the forge refuses the token a push. + ## Over NATS Everything between this module and the rest of the mesh is NATS, in three kinds: an **event** says that diff --git a/modules/claude-code/cmd/claude-code/forge.go b/modules/claude-code/cmd/claude-code/forge.go new file mode 100644 index 00000000..112c033c --- /dev/null +++ b/modules/claude-code/cmd/claude-code/forge.go @@ -0,0 +1,142 @@ +package main + +// The forge's read credential in the agent's home (novox/hq ADR 0365 §4, issue 737). +// +// The forge's module mints each node's agent a token of its own account, `agent-`, scoped to read +// repositories alone, and the vault keeps it for this module on this machine alone (provision `gitea-read`, issue +// 749). What arrives in this module's state is git's credential-store lines for it, one per origin the forge is +// reached at. This module places them in the agent's home, a file of mode 0600 written as the agent's account (ADR +// 0266: no other module writes there), and points every session's git at it through the managed settings' +// environment: the file's path, never its content. So an agent clones and fetches what its account may read, and no +// token is printed, typed on a command line or put in an environment a process listing shows. +// +// Git's own `store` helper reads the file; on a token the forge rejects it erases that line, and on a password a +// person typed elsewhere it adds one. Both are undone at the next look (every few seconds): the file holds what the +// forge issued, and nothing else. A value that is not credential lines — one the mesh made while the forge had not +// been asked yet — is not placed, and what was placed before is taken back. + +import ( + "bytes" + "errors" + "fmt" + "io/fs" + "net/url" + "os" + "path/filepath" + "strings" +) + +// ForgeProvision is the provision whose credential this is, and the file this module's manifest has it delivered to. +const ( + ForgeProvision = "gitea-read" + forgeSecret = "gitea-read.secret" +) + +func (p Paths) forgeSecret() string { return filepath.Join(p.State, forgeSecret) } + +// forgeCredentials is where the agent's git reads the credential: under the agent's home, in a directory of the +// mesh's own. +func (p Paths) forgeCredentials() string { + return filepath.Join(p.Home, ".config", "mesh", "forge.credentials") +} + +// ForgeCredentialLines are the credential lines held, checked: each an http(s) origin with an account and a token, +// and nothing else. Anything else is refused, said without the value. +func ForgeCredentialLines(raw string) (string, error) { + var lines []string + for _, line := range strings.Split(raw, "\n") { + line = strings.TrimSpace(line) + if line == "" { + continue + } + u, err := url.Parse(line) + if err != nil || (u.Scheme != "https" && u.Scheme != "http") || u.Host == "" || u.User == nil || + u.User.Username() == "" || u.Path != "" || u.RawQuery != "" || u.Fragment != "" { + return "", errors.New("what the mesh holds for the forge's read credential is not git credential lines " + + "(the forge has not issued it yet, or something else was given)") + } + if pw, ok := u.User.Password(); !ok || pw == "" { + return "", errors.New("a forge credential line names no token") + } + lines = append(lines, line) + } + if len(lines) == 0 { + return "", errors.New("the forge's read credential is empty") + } + return strings.Join(lines, "\n") + "\n", nil +} + +// ForgeCredentialHeld says whether this machine holds a forge read credential to place. +func ForgeCredentialHeld(p Paths) bool { + raw, err := os.ReadFile(p.forgeSecret()) + if err != nil { + return false + } + _, err = ForgeCredentialLines(string(raw)) + return err == nil +} + +// PlaceForgeCredential puts the forge's read credential in the agent's home as it is held — written, rewritten when +// it differs, taken back when none is held — and says what it did, or "" when nothing changed. Never the value. +func PlaceForgeCredential(p Paths) (string, error) { + path := p.forgeCredentials() + raw, err := os.ReadFile(p.forgeSecret()) + var lines string + var why error + switch { + case errors.Is(err, fs.ErrNotExist): + why = errors.New("this machine holds no forge read credential") + case err != nil: + return "", fmt.Errorf("the forge's read credential cannot be read: %w", err) + default: + lines, why = ForgeCredentialLines(string(raw)) + } + if why != nil { + if _, err := fsys.Stat(path); errors.Is(err, fs.ErrNotExist) { + return "", nil + } + if err := fsys.Remove(path); err != nil && !errors.Is(err, fs.ErrNotExist) { + return "", fmt.Errorf("the forge credential at %s could not be taken back: %w", path, err) + } + return fmt.Sprintf("forge credential: taken back from %s: %v", path, why), nil + } + if held, err := fsys.ReadFile(path); err == nil && bytes.Equal(held, []byte(lines)) { + if info, err := fsys.Stat(path); err == nil && info.Mode().Perm() == 0o600 { + return "", nil + } + } + if err := fsys.MkdirAll(filepath.Dir(path), 0o700); err != nil { + return "", fmt.Errorf("the directory for the forge credential could not be made: %w", err) + } + tmp := path + ".mesh-tmp" + if err := fsys.WriteFile(tmp, []byte(lines), 0o600); err != nil { + return "", fmt.Errorf("the forge credential could not be written: %w", err) + } + if err := fsys.Chmod(tmp, 0o600); err != nil { + _ = fsys.Remove(tmp) + return "", fmt.Errorf("the forge credential could not be made 0600: %w", err) + } + if err := fsys.Rename(tmp, path); err != nil { + _ = fsys.Remove(tmp) + return "", fmt.Errorf("the forge credential could not be put in place: %w", err) + } + return fmt.Sprintf("forge credential: placed at %s for %d origin(s)", path, strings.Count(lines, "\n")), nil +} + +// forgeEnv is what every session's environment carries for git to read the credential: the file's path in git's +// configuration from the environment, never the credential. Laid over the operator's `env`, the mesh's keys winning. +func forgeEnv(managed map[string]any, credentials string) { + if credentials == "" { + return + } + env := map[string]any{} + if held, ok := managed["env"].(map[string]any); ok { + for k, v := range held { + env[k] = v + } + } + env["GIT_CONFIG_COUNT"] = "1" + env["GIT_CONFIG_KEY_0"] = "credential.helper" + env["GIT_CONFIG_VALUE_0"] = "store --file=" + shellQuote(credentials) + managed["env"] = env +} diff --git a/modules/claude-code/cmd/claude-code/forge_test.go b/modules/claude-code/cmd/claude-code/forge_test.go new file mode 100644 index 00000000..8c504620 --- /dev/null +++ b/modules/claude-code/cmd/claude-code/forge_test.go @@ -0,0 +1,262 @@ +package main + +import ( + "encoding/json" + "net/http" + "net/http/cgi" + "net/http/httptest" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" +) + +const agentToken = "0123456789abcdef0123456789abcdef01234567" + +// privateForge is a forge serving one private repository over git's smart http, to agent-anchor with its token +// alone, as the forge does for an account a team lets read it. +func privateForge(t *testing.T) (origin, bare string) { + t.Helper() + backend := filepath.Join(gitExecPath(t), "git-http-backend") + if _, err := os.Stat(backend); err != nil { + t.Skipf("git-http-backend is not installed: %v", err) + } + root := t.TempDir() + bare = filepath.Join(root, "private.git") + work := t.TempDir() + runGit(t, work, nil, "git", "init", "-q", "-b", "main", ".") + commit(t, work, "one") + runGit(t, root, nil, "git", "clone", "-q", "--bare", work, bare) + cgiHandler := &cgi.Handler{Path: backend, Env: []string{"GIT_PROJECT_ROOT=" + root, "GIT_HTTP_EXPORT_ALL=1"}} + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + user, pass, ok := r.BasicAuth() + if !ok || user != "agent-anchor" || pass != agentToken { + w.Header().Set("WWW-Authenticate", `Basic realm="Gitea"`) + http.Error(w, "unauthorized", http.StatusUnauthorized) + return + } + if r.Method == http.MethodPost && strings.HasSuffix(r.URL.Path, "git-receive-pack") { + http.Error(w, "read only", http.StatusForbidden) + return + } + cgiHandler.ServeHTTP(w, r) + })) + t.Cleanup(srv.Close) + return srv.URL, bare +} + +func gitExecPath(t *testing.T) string { + t.Helper() + out, err := exec.Command("git", "--exec-path").Output() + if err != nil { + t.Skipf("git is not installed: %v", err) + } + return strings.TrimSpace(string(out)) +} + +func runGit(t *testing.T, dir string, env []string, name string, args ...string) string { + t.Helper() + out, err := runErr(dir, env, name, args...) + if err != nil { + t.Fatalf("%s %v: %v\n%s", name, args, err, out) + } + return out +} + +func runErr(dir string, env []string, name string, args ...string) (string, error) { + cmd := exec.Command(name, args...) + cmd.Dir = dir + cmd.Env = append(os.Environ(), "GIT_AUTHOR_NAME=t", "GIT_AUTHOR_EMAIL=t@t", "GIT_COMMITTER_NAME=t", + "GIT_COMMITTER_EMAIL=t@t", "GIT_TERMINAL_PROMPT=0", "GIT_ASKPASS=", "SSH_ASKPASS=") + cmd.Env = append(cmd.Env, env...) + out, err := cmd.CombinedOutput() + return string(out), err +} + +func commit(t *testing.T, dir, text string) { + t.Helper() + if err := os.WriteFile(filepath.Join(dir, "file"), []byte(text+"\n"), 0o644); err != nil { + t.Fatal(err) + } + runGit(t, dir, nil, "git", "add", "file") + runGit(t, dir, nil, "git", "commit", "-q", "-m", text) +} + +// sessionEnv is the environment a session starts with: the managed settings' `env`, in a home with no git +// configuration of its own. +func sessionEnv(t *testing.T, home, managedSettings string) []string { + t.Helper() + var settings struct { + Env map[string]string `json:"env"` + } + if err := json.Unmarshal([]byte(managedSettings), &settings); err != nil { + t.Fatal(err) + } + env := []string{"HOME=" + home, "XDG_CONFIG_HOME=" + filepath.Join(home, ".config"), "GIT_CONFIG_NOSYSTEM=1"} + for k, v := range settings.Env { + env = append(env, k+"="+v) + } + return env +} + +func forgePaths(t *testing.T) Paths { + t.Helper() + return Paths{State: t.TempDir(), Home: t.TempDir(), Node: "anchor"} +} + +func managedFor(p Paths) string { + facts := Facts{Node: p.Node, Console: "http://console"} + if ForgeCredentialHeld(p) { + facts.ForgeCredentials = p.forgeCredentials() + } + return Render(facts, Settings{}, nil, "", nil, Config{})["managed-settings.json"] +} + +// Issue 737's done-when, as far as a machine of its own proves it: an agent session clones a private repository and +// fetches it again, and no credential appears in a log, a command line or the session. +func TestAnAgentClonesAPrivateRepositoryAndFetchesItAgainWithTheCredentialPlaced(t *testing.T) { + origin, bare := privateForge(t) + p := forgePaths(t) + repo := origin + "/private.git" + + // Before the forge issued anything, the session's git has no credential: the clone is refused, as today. + before := sessionEnv(t, p.Home, managedFor(p)) + if out, err := runErr(t.TempDir(), before, "git", "clone", "-q", repo, "x"); err == nil { + t.Fatalf("a private repository was cloned with no credential:\n%s", out) + } + + // The vault hands this module what the forge issued for this machine's agent. + lines := strings.Replace(origin, "http://", "http://agent-anchor:"+agentToken+"@", 1) + "\n" + + "https://agent-anchor:" + agentToken + "@git.example.test\n" + if err := os.WriteFile(p.forgeSecret(), []byte(lines), 0o600); err != nil { + t.Fatal(err) + } + said, err := PlaceForgeCredential(p) + if err != nil { + t.Fatal(err) + } + info, err := os.Stat(p.forgeCredentials()) + if err != nil || info.Mode().Perm() != 0o600 { + t.Fatalf("the credential file: %v %v, want mode 0600", info, err) + } + if again, err := PlaceForgeCredential(p); err != nil || again != "" { + t.Fatalf("placed again with nothing changed: %q %v", again, err) + } + managed := managedFor(p) + env := sessionEnv(t, p.Home, managed) + + work := t.TempDir() + cloneOut := runGit(t, work, env, "git", "clone", repo, "clone") + clone := filepath.Join(work, "clone") + if got := runGit(t, clone, env, "git", "log", "--format=%s"); strings.TrimSpace(got) != "one" { + t.Fatalf("cloned %q", got) + } + + // The forge's repository moves on; the agent fetches it again. + other := t.TempDir() + runGit(t, other, nil, "git", "clone", "-q", bare, "w") + commit(t, filepath.Join(other, "w"), "two") + runGit(t, filepath.Join(other, "w"), nil, "git", "push", "-q", "origin", "main") + fetchOut := runGit(t, clone, env, "git", "fetch", "origin") + if got := runGit(t, clone, env, "git", "log", "--format=%s", "origin/main"); !strings.HasPrefix(got, "two") { + t.Fatalf("fetched %q", got) + } + + // A push is the forge's to refuse (ADR 0365 §5): the token reads, and nothing writes with it. + commit(t, clone, "three") + if out, err := runErr(clone, env, "git", "push", "origin", "main"); err == nil { + t.Fatalf("a push with the agent's credential went through:\n%s", out) + } + + // No credential in what the session saw, its command lines, its environment or the module's words. + remote := runGit(t, clone, env, "git", "remote", "get-url", "origin") + for what, text := range map[string]string{"the clone's output": cloneOut, "the fetch's output": fetchOut, + "the remote's url": remote, "the managed settings": managed, "the session's environment": strings.Join(env, "\n"), + "what the module said": said, "the clone's configuration": readOrEmpty(filepath.Join(clone, ".git", "config"))} { + if strings.Contains(text, agentToken) { + t.Errorf("the token appears in %s", what) + } + } +} + +func readOrEmpty(path string) string { + raw, _ := os.ReadFile(path) + return string(raw) +} + +func TestWhatGitChangedInTheFileIsPutBackAndAValueNotIssuedIsNeverPlaced(t *testing.T) { + p := forgePaths(t) + // A value the mesh made while the forge had not been asked: not credential lines, never placed. + if err := os.WriteFile(p.forgeSecret(), []byte("Zm9yZ2VkLWJ5LXRoZS1tZXNo\n"), 0o600); err != nil { + t.Fatal(err) + } + if said, err := PlaceForgeCredential(p); err != nil || said != "" { + t.Fatalf("%q %v", said, err) + } + if _, err := os.Stat(p.forgeCredentials()); err == nil { + t.Fatal("a value the forge did not issue was placed") + } + if ForgeCredentialHeld(p) || strings.Contains(managedFor(p), "GIT_CONFIG") { + t.Fatal("the sessions' git is pointed at a credential that is not held") + } + + issued := "https://agent-anchor:" + agentToken + "@git.example.test\n" + if err := os.WriteFile(p.forgeSecret(), []byte(issued), 0o600); err != nil { + t.Fatal(err) + } + if _, err := PlaceForgeCredential(p); err != nil { + t.Fatal(err) + } + // git's store helper erased the line after a rejection, and a person's password for another host was stored. + if err := os.WriteFile(p.forgeCredentials(), []byte("https://me:mine@example.org\n"), 0o644); err != nil { + t.Fatal(err) + } + if said, err := PlaceForgeCredential(p); err != nil || !strings.Contains(said, "placed") { + t.Fatalf("%q %v", said, err) + } + if got := readOrEmpty(p.forgeCredentials()); got != issued { + t.Fatalf("the file holds %q", strings.ReplaceAll(got, agentToken, "")) + } + if info, _ := os.Stat(p.forgeCredentials()); info.Mode().Perm() != 0o600 { + t.Fatalf("mode %v", info.Mode().Perm()) + } + + // The credential withdrawn: taken back, and the sessions' git no longer pointed at it. + if err := os.Remove(p.forgeSecret()); err != nil { + t.Fatal(err) + } + if said, err := PlaceForgeCredential(p); err != nil || !strings.Contains(said, "taken back") || strings.Contains(said, agentToken) { + t.Fatalf("%q %v", said, err) + } + if _, err := os.Stat(p.forgeCredentials()); err == nil { + t.Fatal("a withdrawn credential stayed in the agent's home") + } +} + +func TestTheSessionsGitIsPointedAtTheFileUnderTheOperatorsOwnEnvironment(t *testing.T) { + managed := map[string]any{"env": map[string]any{"FOO": "bar", "GIT_CONFIG_COUNT": "7"}} + forgeEnv(managed, "/home/agent/.config/mesh/forge.credentials") + env := managed["env"].(map[string]any) + if env["FOO"] != "bar" || env["GIT_CONFIG_COUNT"] != "1" || env["GIT_CONFIG_KEY_0"] != "credential.helper" || + env["GIT_CONFIG_VALUE_0"] != "store --file='/home/agent/.config/mesh/forge.credentials'" { + t.Fatalf("%v", env) + } + none := map[string]any{} + forgeEnv(none, "") + if _, set := none["env"]; set { + t.Fatal("an environment was set with no credential held") + } +} + +func TestForgeCredentialLinesAreReadStrictly(t *testing.T) { + for _, bad := range []string{"", "\n", "token", "https://git.example.test", "https://agent-anchor@git.example.test", + "https://agent-anchor:t@git.example.test/novox/hq.git", "ssh://agent-anchor:t@git.example.test", + "https://agent-anchor:t@git.example.test\nnot a line"} { + if _, err := ForgeCredentialLines(bad); err == nil { + t.Errorf("%q was taken", bad) + } else if strings.Contains(err.Error(), "t@git") { + t.Errorf("the refusal said the value: %v", err) + } + } +} diff --git a/modules/claude-code/cmd/claude-code/main.go b/modules/claude-code/cmd/claude-code/main.go index 618c8145..88250936 100644 --- a/modules/claude-code/cmd/claude-code/main.go +++ b/modules/claude-code/cmd/claude-code/main.go @@ -585,7 +585,25 @@ func run(p Paths, view *ServerView, configView *ConfigView) { report() var last string + // The forge's read credential (forge.go, ADR 0365 §4): placed as held, and put back when git or a person changed + // it; the sessions' environment names it only while one is held. + forgeHeld := ForgeCredentialHeld(p) + placeForge := func() { + if line, err := PlaceForgeCredential(p); err != nil { + say("%v", err) + } else if line != "" { + say("%s", line) + } + if held := ForgeCredentialHeld(p); held != forgeHeld { + forgeHeld = held + if _, err := RenderNow(p, writeManaged); err != nil { + say("rendering after the forge's read credential changed: %v", err) + } + } + } + placeForge() for range time.Tick(5 * time.Second) { + placeForge() info, err := fsys.Stat(p.credentials()) now := "absent" if err == nil { diff --git a/modules/claude-code/cmd/claude-code/node.go b/modules/claude-code/cmd/claude-code/node.go index 854e6061..12868291 100644 --- a/modules/claude-code/cmd/claude-code/node.go +++ b/modules/claude-code/cmd/claude-code/node.go @@ -216,6 +216,9 @@ func RenderNow(p Paths, write WriteManaged) ([]string, error) { if !readJSON(p.Facts, &facts) || facts.Console == "" { return nil, fmt.Errorf("the mesh has not rendered %s yet; nothing to write", p.Facts) } + if ForgeCredentialHeld(p) { + facts.ForgeCredentials = p.forgeCredentials() + } var settings Settings readJSON(p.Settings, &settings) var binding *Binding diff --git a/modules/claude-code/cmd/claude-code/render.go b/modules/claude-code/cmd/claude-code/render.go index 80fb011c..0ce3acde 100644 --- a/modules/claude-code/cmd/claude-code/render.go +++ b/modules/claude-code/cmd/claude-code/render.go @@ -34,6 +34,9 @@ const meshEntry = "mesh" type Facts struct { Node string `json:"node"` Console string `json:"console"` + // ForgeCredentials is the forge's read credential placed in the agent's home, when this machine holds one + // (forge.go): its path, which every session's git is pointed at. Never rendered by the mesh. + ForgeCredentials string `json:"-"` } // Settings are the operator's, for the mesh or this node. @@ -139,6 +142,7 @@ func RenderWithMesh(facts Facts, settings Settings, binding *Binding, helperPath } managed[key] = entries } + forgeEnv(managed, facts.ForgeCredentials) if binding != nil && binding.Kind == "api-key" { managed["apiKeyHelper"] = helperPath } diff --git a/modules/claude-code/cmd/claude-code/shell_test.go b/modules/claude-code/cmd/claude-code/shell_test.go index 9ce22880..3061a3b2 100644 --- a/modules/claude-code/cmd/claude-code/shell_test.go +++ b/modules/claude-code/cmd/claude-code/shell_test.go @@ -19,11 +19,12 @@ const agentAccountFile = "/etc/claude-code/agent-account" type manifestShell struct { Resources []map[string]any `json:"resources"` - Shell []struct { - For string `json:"for"` - Slot string `json:"slot"` - Code string `json:"code"` - } `json:"shell"` + // The zsh code is an item of the collection shell:zsh: (novox/hq ADR 0349 clause 13), its text the + // template file beside the manifest. + Items []struct { + Collection string `json:"collection"` + Template string `json:"template"` + } `json:"items"` } func readManifestShell(t *testing.T) manifestShell { @@ -54,9 +55,13 @@ func resourceByID(t *testing.T, m manifestShell, id string) map[string]any { func zshCode(t *testing.T, m manifestShell) string { t.Helper() var code []string - for _, s := range m.Shell { - if s.For == "zsh" { - code = append(code, s.Code) + for _, it := range m.Items { + if strings.HasPrefix(it.Collection, "shell:zsh:") { + text, err := os.ReadFile(filepath.Join("..", "..", it.Template)) + if err != nil { + t.Fatal(err) + } + code = append(code, string(text)) } } if len(code) != 1 { @@ -71,8 +76,8 @@ func TestTheAccountAgentsRunAsIsWrittenForTheShell(t *testing.T) { if r["type"] != "file" || r["path"] != agentAccountFile || r["mode"] != "0644" { t.Fatalf("the account's file is %v", r) } - if r["content"] != "${machine:agent-account}\n" { - t.Fatalf("the account's file holds %q", r["content"]) + if text := templateText(t, r); text != "${machine:agent-account}\n" { + t.Fatalf("the account's file holds %q", text) } if _, has := r["owner"]; has { t.Fatal("the account's file must be root's, as claude-agent is") @@ -163,7 +168,7 @@ func TestClaudeAgentRunsTheRealClaudeUnderTheAlias(t *testing.T) { if err != nil { t.Fatal(err) } - start, _ := resourceByID(t, readManifestShell(t), "start")["content"].(string) + start := templateText(t, resourceByID(t, readManifestShell(t), "start")) if !strings.Contains(start, "exec claude \"$@\"") { t.Fatalf("claude-agent no longer runs claude itself:\n%s", start) } diff --git a/modules/claude-code/cmd/claude-code/templates_test.go b/modules/claude-code/cmd/claude-code/templates_test.go new file mode 100644 index 00000000..7d0718b5 --- /dev/null +++ b/modules/claude-code/cmd/claude-code/templates_test.go @@ -0,0 +1,64 @@ +package main + +import ( + "encoding/json" + "os" + "path/filepath" + "testing" +) + +// The module's directory, where its manifest and its files/ are. +const moduleDir = "../.." + +// templateText is the text a file resource places on the machine: its template's, in the module's files/ +// (novox/hq ADR 0349 §2). A resource that names no template, or a template that is not there, fails the test. +func templateText(t *testing.T, r map[string]any) string { + t.Helper() + name, _ := r["template"].(string) + if name == "" { + t.Fatalf("resource %v names no template", r["id"]) + } + raw, err := os.ReadFile(filepath.Join(moduleDir, filepath.FromSlash(name))) + if err != nil { + t.Fatalf("resource %v names %s: %v", r["id"], name, err) + } + return string(raw) +} + +// No file resource carries its text inline: each names its template in files/, and the build packs files/ +// into the one archive the controller reads the templates back from (novox/hq ADR 0349 §2). +func TestEveryFileTakesItsTextFromATemplate(t *testing.T) { + raw, err := os.ReadFile(filepath.Join(moduleDir, "module.json")) + if err != nil { + t.Fatal(err) + } + var m struct { + Resources []map[string]any `json:"resources"` + Build struct { + Artifacts []map[string]any `json:"artifacts"` + } `json:"build"` + } + if err := json.Unmarshal(raw, &m); err != nil { + t.Fatal(err) + } + for _, r := range m.Resources { + if _, inline := r["content"]; inline { + t.Errorf("resource %v keeps its text inline: keep it in files/ and name it as the template", r["id"]) + } + if r["type"] == "file" { + templateText(t, r) + } + } + packs := 0 + for _, a := range m.Build.Artifacts { + if a["kind"] == "archive" && a["from"] == "files" { + packs++ + if a["name"] != "files" { + t.Errorf("the templates archive is named %v, not files", a["name"]) + } + } + } + if packs != 1 { + t.Errorf("the build packs files/ into %d archives, not one", packs) + } +} diff --git a/modules/claude-code/files/agent-account b/modules/claude-code/files/agent-account new file mode 100644 index 00000000..b081a6c3 --- /dev/null +++ b/modules/claude-code/files/agent-account @@ -0,0 +1 @@ +${machine:agent-account} diff --git a/modules/claude-code/files/claude-agent b/modules/claude-code/files/claude-agent new file mode 100755 index 00000000..e3d330f2 --- /dev/null +++ b/modules/claude-code/files/claude-agent @@ -0,0 +1,13 @@ +#!/bin/sh +# The mesh's (module claude-code, novox/hq ADR 0266): start the agent as the account agents run as on this +# machine. Written whole at every push: an edit here is overwritten. +agent='${machine:agent-account}' +if [ "$(id -un)" = "$agent" ]; then + exec claude "$@" +fi +# Another account, the operator's, becomes the agent's through its own sudo: the person is the authority. The +# operator's override of the guard travels only when it is set in this shell, as it would reach claude. +if [ -n "$MESH_GUARD_OVERRIDE" ]; then + exec sudo -iu "$agent" env MESH_GUARD_OVERRIDE="$MESH_GUARD_OVERRIDE" claude "$@" +fi +exec sudo -iu "$agent" claude "$@" diff --git a/modules/claude-code/files/facts.json b/modules/claude-code/files/facts.json new file mode 100644 index 00000000..7fd72bd4 --- /dev/null +++ b/modules/claude-code/files/facts.json @@ -0,0 +1,4 @@ +{ + "node": "${machine:name}", + "console": "http://127.0.0.1:${bound:mcp-endpoint:port}/mcp" +} diff --git a/modules/claude-code/files/settings.json b/modules/claude-code/files/settings.json new file mode 100644 index 00000000..8284e660 --- /dev/null +++ b/modules/claude-code/files/settings.json @@ -0,0 +1,10 @@ +{ + "role": "", + "mcp_servers": {}, + "managed_settings": {}, + "instructions": {}, + "output_styles": {}, + "skills": {}, + "commands": {}, + "agents": {} +} diff --git a/modules/claude-code/files/zsh-normal b/modules/claude-code/files/zsh-normal new file mode 100644 index 00000000..b90b85f9 --- /dev/null +++ b/modules/claude-code/files/zsh-normal @@ -0,0 +1,12 @@ +# The agent's session (module claude-code, novox/hq ADR 0266): where this machine names an account of the +# agents' own, claude in an interactive zsh is claude-agent, which starts the session as that account. Where +# agents run as the operator's account the file names that account, and nothing is added. claude-agent runs +# the real claude: exec, sudo and its sh see no alias. +if [[ -r /etc/claude-code/agent-account ]]; then + () { + local agent=$(:, in the order they are declared. +func (m manifest) shell(t *testing.T) []shellCode { + t.Helper() + var out []shellCode + for _, it := range m.Items { + parts := strings.Split(it.Collection, ":") + if len(parts) != 3 || parts[0] != "shell" { + continue + } + text, err := os.ReadFile(filepath.Join("..", "..", it.Template)) + if err != nil { + t.Fatal(err) + } + out = append(out, shellCode{For: parts[1], Slot: parts[2], Code: string(text)}) + } + return out +} + +// environmentOf is the variables the module sets, read from the templates of its items of the environment. +func (m manifest) environmentOf(t *testing.T) *environment { + t.Helper() + var env *environment + for _, it := range m.Items { + if it.Collection != "environment" || it.Attributes["kind"] != "variable" { + continue + } + text, err := os.ReadFile(filepath.Join("..", "..", it.Template)) + if err != nil { + t.Fatal(err) + } + if env == nil { + env = &environment{Variables: map[string]string{}} + } + env.Variables[it.Attributes["name"]] = strings.TrimSuffix(string(text), "\n") + } + return env +} + func readManifest(t *testing.T) manifest { t.Helper() raw, err := os.ReadFile(filepath.Join("..", "..", "module.json")) @@ -58,6 +107,7 @@ func readManifest(t *testing.T) manifest { if err := dec.Decode(&m); err != nil { t.Fatalf("module.json: %v", err) } + m.Environment = m.environmentOf(t) return m } @@ -85,26 +135,66 @@ func (m manifest) packages() (present, absent []string) { return present, absent } -// sameAsSource checks that a file resource's content is byte for byte the module's source file, so -// the readable file in the repository is what the machine gets. -func (m manifest) sameAsSource(t *testing.T, id, source string) { +// fromTemplate checks that a file resource takes its text from its template file in the module's +// repository (novox/hq ADR 0349 §2): it names the template, carries no text of its own beside it, and the +// template is there. The readable file in the repository is the one place the machine's text is written. +func (m manifest) fromTemplate(t *testing.T, id, template string) { t.Helper() - want, err := os.ReadFile(filepath.Join("..", "..", source)) - if err != nil { - t.Fatal(err) - } r := m.resource(t, id) if r["type"] != "file" { t.Fatalf("%s is a %v, not a file", id, r["type"]) } - if got, _ := r["content"].(string); got != string(want) { - t.Fatalf("resource %s's content is not %s: edit the source and copy it into module.json", id, source) + if r["template"] != template { + t.Fatalf("resource %s does not take its text from %s: template %v", id, template, r["template"]) + } + if _, inline := r["content"]; inline { + t.Fatalf("resource %s keeps its text inline beside its template: a file's text is in one place", id) + } + if _, err := os.Stat(filepath.Join("..", "..", template)); err != nil { + t.Fatalf("resource %s names %s: %v", id, template, err) } if r["owner"] != "${machine:account}" && !strings.HasPrefix(r["path"].(string), "/etc/") { t.Fatalf("%s under the home is the account's", id) } } +// everyFileIsATemplate checks that no file resource carries its text inline, and that the build packs +// files/ into the one archive the controller reads the templates back from (novox/hq ADR 0349 §2). +func (m manifest) everyFileIsATemplate(t *testing.T) { + t.Helper() + for _, r := range m.Resources { + if _, inline := r["content"]; inline { + t.Errorf("resource %v keeps its text inline: keep it in files/ and name it as the template", r["id"]) + } + } + packs := 0 + for _, a := range m.Build.Artifacts { + if a["kind"] == "archive" && a["from"] == "files" { + packs++ + if a["name"] != "files" { + t.Errorf("the templates archive is named %v, not files", a["name"]) + } + } + } + if packs != 1 { + t.Errorf("the build packs files/ into %d archives, not one", packs) + } +} + +// text is the text a file resource places on the machine: its template file's. +func (m manifest) text(t *testing.T, id string) string { + t.Helper() + name, _ := m.resource(t, id)["template"].(string) + if name == "" { + t.Fatalf("resource %s names no template", id) + } + raw, err := os.ReadFile(filepath.Join("..", "..", name)) + if err != nil { + t.Fatal(err) + } + return string(raw) +} + // checkTheToolsAgree checks that the manifest lists the module's own tools exactly, that the bundle // serves each seat verb the claims promise as ., and that the Go bundle is declared. func checkTheToolsAgree(t *testing.T, m manifest) { @@ -176,31 +266,42 @@ func checkNoSecretsOrInstallationNames(t *testing.T) { } } -type contribution struct { - Seat string `json:"seat"` - Kind string `json:"kind"` - Content string `json:"content"` -} +// displaySession is the collection the module's lines for the window manager are an item of. +const displaySession = "contribution:node-display-session:config" -// i3Lines is what the module contributes to the window manager. +// i3Lines is what the module contributes to the window manager: the text of its item of the collection +// contribution:node-display-session:config (design 57 L3, issue 848). func (m manifest) i3Lines() string { var out string - for _, c := range m.Contributions { - if c.Seat == "node-display-session" && c.Kind == "config" { - out += c.Content + for _, it := range m.Items { + if it.Collection == displaySession { + text, err := os.ReadFile(filepath.Join("..", "..", it.Template)) + if err != nil { + return "" + } + out += string(text) } } return out } -// i3LinesAreSource checks the window-manager contribution is the source file it is written from. +// i3LinesAreSource checks the window-manager lines are one item whose template is the source file, and that no +// old-form contribution to node-display-session is left. func (m manifest) i3LinesAreSource(t *testing.T, source string) { t.Helper() - want, err := os.ReadFile(filepath.Join("..", "..", source)) - if err != nil { - t.Fatal(err) + if len(m.Contributions) != 0 { + t.Fatalf("the module still holds contributions: the lines are an item of %s", displaySession) } - if m.i3Lines() != string(want) { - t.Fatalf("the contribution to node-display-session is not %s: edit the source and copy it into module.json", source) + var templates []string + for _, it := range m.Items { + if it.Collection == displaySession { + templates = append(templates, it.Template) + } + } + if len(templates) != 1 || templates[0] != source { + t.Fatalf("the items of %s are %v, want the one template %s", displaySession, templates, source) + } + if m.i3Lines() == "" { + t.Fatalf("the template %s holds no line", source) } } diff --git a/modules/clipmenu/cmd/clipmenu-tools/manifest_test.go b/modules/clipmenu/cmd/clipmenu-tools/manifest_test.go index ce2e40ba..4e769c4c 100644 --- a/modules/clipmenu/cmd/clipmenu-tools/manifest_test.go +++ b/modules/clipmenu/cmd/clipmenu-tools/manifest_test.go @@ -36,8 +36,8 @@ func TestItClaimsTheClipboardSeatServingHistoryAndCopy(t *testing.T) { func TestTheDaemonIsTheOperatorsUserUnitEnabledAndRunning(t *testing.T) { m := readManifest(t) - if len(m.Shell) != 0 { - t.Fatalf("a second start from the session: %+v", m.Shell) + if len(m.shell(t)) != 0 { + t.Fatalf("a second start from the session: %+v", m.shell(t)) } svc := m.resource(t, "daemon") want := map[string]any{"id": "daemon", "type": "service", "unit": "clipmenud.service", "scope": "user", @@ -53,14 +53,14 @@ func TestTheDaemonIsTheOperatorsUserUnitEnabledAndRunning(t *testing.T) { t.Fatalf("daemon health = %v, want kind unit", svc["health"]) } dropin := m.resource(t, "daemon-dropin") - content, _ := dropin["content"].(string) + content := m.text(t, "daemon-dropin") if dropin["path"] != "${machine:account-home}/.config/systemd/user/clipmenud.service.d/mesh.conf" || dropin["owner"] != "${machine:account}" || !strings.Contains(content, "ExecStart=/usr/local/lib/mesh-clipmenu/clipmenud-session") || strings.Contains(content, "greenclip") { t.Fatalf("drop-in: %v", dropin) } - launcher, _ := m.resource(t, "launcher")["content"].(string) + launcher := m.text(t, "launcher") if !strings.Contains(launcher, "DISPLAY") || !strings.Contains(launcher, "/tmp/.X11-unix") || !strings.Contains(launcher, `PATH="/usr/local/lib/mesh-clipmenu:$PATH"`) || !strings.Contains(launcher, "exec clipmenud") { t.Fatalf("launcher: %q", launcher) @@ -73,7 +73,7 @@ func TestTheDaemonIsTheOperatorsUserUnitEnabledAndRunning(t *testing.T) { if !strings.Contains(content, "StartLimitIntervalSec=0") || !strings.Contains(content, "RestartSec=2s") { t.Fatalf("drop-in lacks the safety net: %q", content) } - for k, v := range m.Environment.Variables { + for k, v := range m.environmentOf(t).Variables { if !strings.Contains(content, "Environment="+k+"="+v) { t.Fatalf("the unit lacks %s=%s", k, v) } @@ -100,10 +100,15 @@ func TestTheToolsAgreeWithTheManifest(t *testing.T) { checkNoSecretsOrInstallationNames(t) } +// No file resource carries its text inline (novox/hq ADR 0349 §2). +func TestEveryFileTakesItsTextFromATemplate(t *testing.T) { + readManifest(t).everyFileIsATemplate(t) +} + // Run for real: DISPLAY from the manager is used at once; several sockets and no DISPLAY wait. func TestTheLauncherUsesTheManagersDisplayAndWaitsOtherwise(t *testing.T) { m := readManifest(t) - script, _ := m.resource(t, "launcher")["content"].(string) + script := m.text(t, "launcher") dir := t.TempDir() file := dir + "/launcher" // clipmenud and the socket directory are faked: the script's own paths are rewritten. @@ -155,7 +160,7 @@ func TestTheLauncherWaitsQuietlyWhileAnotherClipmenudHoldsTheLock(t *testing.T) t.Skip("flock is missing here") } m := readManifest(t) - script, _ := m.resource(t, "launcher")["content"].(string) + script := m.text(t, "launcher") dir := t.TempDir() body := strings.Replace(script, "exec clipmenud", `echo "started"`, 1) file := dir + "/launcher" diff --git a/modules/clipmenu/files/bin/clipmenud-session b/modules/clipmenu/files/bin/clipmenud-session new file mode 100644 index 00000000..5bd40ee2 --- /dev/null +++ b/modules/clipmenu/files/bin/clipmenud-session @@ -0,0 +1,41 @@ +#!/bin/sh +# clipmenud as the operator's user unit starts it, written by the mesh (module clipmenu). Replaced at every push. +# +# The account's service manager starts before the X server does, and has a DISPLAY only once the session's +# xinitrc has handed it over (module xorg). So this waits, quietly, until the display is known, and exits +# only when the session ends with it: a quick exit would run the unit into its start limit. DISPLAY, when +# the manager has it, wins. Without it, exactly one X socket names the display; none or several (a +# second server beside the session) wait for the session's own to arrive. +# A running process does not see what the session hands the manager later, so each pass asks the manager. +until [ -n "$DISPLAY" ]; do + for pair in $(systemctl --user show-environment 2>/dev/null | grep -E '^(DISPLAY|XAUTHORITY)='); do + export "$pair" + done + [ -n "$DISPLAY" ] && break + set -- /tmp/.X11-unix/X* + if [ $# -eq 1 ] && [ -S "$1" ]; then + DISPLAY=":${1##*/X}" + export DISPLAY + else + sleep 1 + fi +done +if [ -z "$XAUTHORITY" ] && [ -r "$HOME/.Xauthority" ]; then + XAUTHORITY="$HOME/.Xauthority" + export XAUTHORITY +fi +# clipmenud allows one daemon per account: it takes an exclusive flock on its session_lock file and exits +# with status 2 when that is held. After a delivery without a new login, the session's older clipmenud (from +# its xinitrc) still holds it, and a start would run the unit round its restart loop. So this waits, quietly, +# until the lock is free: it looks at the same file clipmenud locks (the directory is clipmenud's own rule) +# and lets go at once. Exit status 1 is "held" for util-linux and BusyBox flock alike (no -E: BusyBox lacks it); any other outcome starts clipmenud, which says its own +# error. Another daemon winning the gap between look and start costs one restart, not a loop. +lock="${CM_DIR:-${XDG_RUNTIME_DIR-${TMPDIR-/tmp}}}/clipmenu.6.$USER/session_lock" +while [ -e "$lock" ]; do + flock -n "$lock" true + [ $? -eq 1 ] || break + sleep 2 +done +PATH="/usr/local/lib/mesh-clipmenu:$PATH" +export PATH +exec clipmenud diff --git a/modules/clipmenu/files/bin/xsel b/modules/clipmenu/files/bin/xsel new file mode 100755 index 00000000..10f9bbd9 --- /dev/null +++ b/modules/clipmenu/files/bin/xsel @@ -0,0 +1,20 @@ +#!/bin/sh +# xsel as clipmenud sees it, written by the mesh (module clipmenu). Replaced at every push. +# +# clipmenud records text, and reads a selection with `timeout 1 xsel -o`. A selection holding an +# image (a screenshot copied as image/png) is sent in pieces; one second is too short for megabytes, +# timeout kills xsel half way, and the program owning the image waits for ever for a reader that is +# gone. From then on nothing can ask the clipboard anything: pastes hang, and an Electron app that +# asks on its main thread freezes. So a read goes ahead only when the selection offers text. +case " $* " in +*" -o "* | *" --output "*) + selection=clipboard + case " $* " in + *" --primary "* | *" -p "*) selection=primary ;; + *" --secondary "* | *" -s "*) selection=secondary ;; + esac + targets=$(timeout 1 xclip -selection "$selection" -t TARGETS -o 2>/dev/null) || exit 1 + printf '%s\n' "$targets" | grep -qxE 'UTF8_STRING|STRING|TEXT|text/plain(;charset=utf-8)?' || exit 1 + ;; +esac +exec /usr/bin/xsel "$@" diff --git a/modules/clipmenu/files/env-cm-histlength b/modules/clipmenu/files/env-cm-histlength new file mode 100644 index 00000000..60d3b2f4 --- /dev/null +++ b/modules/clipmenu/files/env-cm-histlength @@ -0,0 +1 @@ +15 diff --git a/modules/clipmenu/files/env-cm-max-clips b/modules/clipmenu/files/env-cm-max-clips new file mode 100644 index 00000000..1b79f38e --- /dev/null +++ b/modules/clipmenu/files/env-cm-max-clips @@ -0,0 +1 @@ +500 diff --git a/modules/clipmenu/files/env-cm-selections b/modules/clipmenu/files/env-cm-selections new file mode 100644 index 00000000..613d81a1 --- /dev/null +++ b/modules/clipmenu/files/env-cm-selections @@ -0,0 +1 @@ +clipboard diff --git a/modules/clipmenu/files/systemd/clipmenud-mesh.conf b/modules/clipmenu/files/systemd/clipmenud-mesh.conf new file mode 100644 index 00000000..799e22d3 --- /dev/null +++ b/modules/clipmenu/files/systemd/clipmenud-mesh.conf @@ -0,0 +1,14 @@ +# clipmenud in the operator's account (module clipmenu, novox/hq ADR 0208). Owned by the mesh: replaced at every push. +# The package's unit starts clipmenud with no DISPLAY; the module's launcher finds the display and puts the +# text-only xsel first on the PATH. The settings repeat the module's environment, which the account's +# service manager does not read. +[Unit] +StartLimitIntervalSec=0 + +[Service] +RestartSec=2s +Environment=CM_SELECTIONS=clipboard +Environment=CM_MAX_CLIPS=500 +Environment=CM_HISTLENGTH=15 +ExecStart= +ExecStart=/usr/local/lib/mesh-clipmenu/clipmenud-session diff --git a/modules/clipmenu/module.json b/modules/clipmenu/module.json index 7343b982..f2d328f2 100644 --- a/modules/clipmenu/module.json +++ b/modules/clipmenu/module.json @@ -1,6 +1,7 @@ { "module": "clipmenu", "version": "1", + "moved": true, "capabilities": [ "package-manager" ], @@ -22,13 +23,40 @@ "clipmenu_clear", "clipmenu_delete" ], - "environment": { - "variables": { - "CM_SELECTIONS": "clipboard", - "CM_MAX_CLIPS": "500", - "CM_HISTLENGTH": "15" + "items": [ + { + "collection": "environment", + "name": "cm-selections", + "attributes": { + "kind": "variable", + "name": "CM_SELECTIONS" + }, + "template": "files/env-cm-selections" + }, + { + "collection": "environment", + "name": "cm-max-clips", + "attributes": { + "kind": "variable", + "name": "CM_MAX_CLIPS" + }, + "template": "files/env-cm-max-clips" + }, + { + "collection": "environment", + "name": "cm-histlength", + "attributes": { + "kind": "variable", + "name": "CM_HISTLENGTH" + }, + "template": "files/env-cm-histlength" + }, + { + "collection": "contribution:node-display-session:config", + "name": "display-session", + "template": "files/i3/50-clipmenu.conf" } - }, + ], "resources": [ { "id": "package", @@ -46,7 +74,7 @@ "type": "file", "path": "/usr/local/lib/mesh-clipmenu/xsel", "mode": "0755", - "content": "#!/bin/sh\n# xsel as clipmenud sees it, written by the mesh (module clipmenu). Replaced at every push.\n#\n# clipmenud records text, and reads a selection with `timeout 1 xsel -o`. A selection holding an\n# image (a screenshot copied as image/png) is sent in pieces; one second is too short for megabytes,\n# timeout kills xsel half way, and the program owning the image waits for ever for a reader that is\n# gone. From then on nothing can ask the clipboard anything: pastes hang, and an Electron app that\n# asks on its main thread freezes. So a read goes ahead only when the selection offers text.\ncase \" $* \" in\n*\" -o \"* | *\" --output \"*)\n\tselection=clipboard\n\tcase \" $* \" in\n\t*\" --primary \"* | *\" -p \"*) selection=primary ;;\n\t*\" --secondary \"* | *\" -s \"*) selection=secondary ;;\n\tesac\n\ttargets=$(timeout 1 xclip -selection \"$selection\" -t TARGETS -o 2>/dev/null) || exit 1\n\tprintf '%s\\n' \"$targets\" | grep -qxE 'UTF8_STRING|STRING|TEXT|text/plain(;charset=utf-8)?' || exit 1\n\t;;\nesac\nexec /usr/bin/xsel \"$@\"\n" + "template": "files/bin/xsel" }, { "id": "user-units-dir", @@ -68,14 +96,14 @@ "path": "${machine:account-home}/.config/systemd/user/clipmenud.service.d/mesh.conf", "owner": "${machine:account}", "mode": "0644", - "content": "# clipmenud in the operator's account (module clipmenu, novox/hq ADR 0208). Owned by the mesh: replaced at every push.\n# The package's unit starts clipmenud with no DISPLAY; the module's launcher finds the display and puts the\n# text-only xsel first on the PATH. The settings repeat the module's environment, which the account's\n# service manager does not read.\n[Unit]\nStartLimitIntervalSec=0\n\n[Service]\nRestartSec=2s\nEnvironment=CM_SELECTIONS=clipboard\nEnvironment=CM_MAX_CLIPS=500\nEnvironment=CM_HISTLENGTH=15\nExecStart=\nExecStart=/usr/local/lib/mesh-clipmenu/clipmenud-session\n" + "template": "files/systemd/clipmenud-mesh.conf" }, { "id": "launcher", "type": "file", "path": "/usr/local/lib/mesh-clipmenu/clipmenud-session", "mode": "0755", - "content": "#!/bin/sh\n# clipmenud as the operator's user unit starts it, written by the mesh (module clipmenu). Replaced at every push.\n#\n# The account's service manager starts before the X server does, and has a DISPLAY only once the session's\n# xinitrc has handed it over (module xorg). So this waits, quietly, until the display is known, and exits\n# only when the session ends with it: a quick exit would run the unit into its start limit. DISPLAY, when\n# the manager has it, wins. Without it, exactly one X socket names the display; none or several (a\n# second server beside the session) wait for the session's own to arrive.\n# A running process does not see what the session hands the manager later, so each pass asks the manager.\nuntil [ -n \"$DISPLAY\" ]; do\n\tfor pair in $(systemctl --user show-environment 2>/dev/null | grep -E '^(DISPLAY|XAUTHORITY)='); do\n\t\texport \"$pair\"\n\tdone\n\t[ -n \"$DISPLAY\" ] && break\n\tset -- /tmp/.X11-unix/X*\n\tif [ $# -eq 1 ] && [ -S \"$1\" ]; then\n\t\tDISPLAY=\":${1##*/X}\"\n\t\texport DISPLAY\n\telse\n\t\tsleep 1\n\tfi\ndone\nif [ -z \"$XAUTHORITY\" ] && [ -r \"$HOME/.Xauthority\" ]; then\n\tXAUTHORITY=\"$HOME/.Xauthority\"\n\texport XAUTHORITY\nfi\n# clipmenud allows one daemon per account: it takes an exclusive flock on its session_lock file and exits\n# with status 2 when that is held. After a delivery without a new login, the session's older clipmenud (from\n# its xinitrc) still holds it, and a start would run the unit round its restart loop. So this waits, quietly,\n# until the lock is free: it looks at the same file clipmenud locks (the directory is clipmenud's own rule)\n# and lets go at once. Exit status 1 is \"held\" for util-linux and BusyBox flock alike (no -E: BusyBox lacks it); any other outcome starts clipmenud, which says its own\n# error. Another daemon winning the gap between look and start costs one restart, not a loop.\nlock=\"${CM_DIR:-${XDG_RUNTIME_DIR-${TMPDIR-/tmp}}}/clipmenu.6.$USER/session_lock\"\nwhile [ -e \"$lock\" ]; do\n\tflock -n \"$lock\" true\n\t[ $? -eq 1 ] || break\n\tsleep 2\ndone\nPATH=\"/usr/local/lib/mesh-clipmenu:$PATH\"\nexport PATH\nexec clipmenud\n" + "template": "files/bin/clipmenud-session" }, { "id": "daemon", @@ -102,14 +130,12 @@ "loads": [ "clipmenu-tools" ] + }, + { + "name": "files", + "kind": "archive", + "from": "files" } ] - }, - "contributions": [ - { - "seat": "node-display-session", - "kind": "config", - "content": "# The clipboard's history key (module clipmenu, novox/hq ADR 0208). Owned by the mesh: replaced at\n# every push. clipmenu shows the history through `dmenu`, the node's dmenu-compatible command, which\n# the holder of node-launcher answers (rofi on the workstations); the chosen entry is put back on the\n# clipboard.\nbindsym $mod+period exec --no-startup-id clipmenu -p Clipboard\n" - } - ] + } } diff --git a/modules/confluence/files/config.json b/modules/confluence/files/config.json new file mode 100644 index 00000000..9e26dfee --- /dev/null +++ b/modules/confluence/files/config.json @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/modules/confluence/module.json b/modules/confluence/module.json index 18ff0ea6..18fb5e5e 100644 --- a/modules/confluence/module.json +++ b/modules/confluence/module.json @@ -5,6 +5,20 @@ "own-secrets": { "token": "${dir:state}/token" }, + "config": { + "atlassian-url": { + "type": "string", + "description": "The address of the operator's Atlassian site the confluence tools call, as a URL such as https://example.atlassian.net, laid into the file as ATLASSIAN_URL; unset, the code falls back to its environment and answers no calls until it is given.", + "trusted": true, + "default": "" + }, + "atlassian-email": { + "type": "string", + "description": "The email address of the Atlassian account the confluence tools sign in as with its API token, laid into the file as ATLASSIAN_EMAIL; unset, the code falls back to its environment and answers no calls until it is given.", + "trusted": true, + "default": "" + } + }, "resources": [ { "id": "state", @@ -17,8 +31,18 @@ "type": "file", "path": "${dir:state}/config.json", "merge": "json", - "content": "{}", - "mode": "0600" + "template": "files/config.json", + "mode": "0600", + "takes-config": [ + { + "key": "atlassian-url", + "as": "ATLASSIAN_URL" + }, + { + "key": "atlassian-email", + "as": "ATLASSIAN_EMAIL" + } + ] } ], "capabilities": [ @@ -40,6 +64,11 @@ "MESH_CONFLUENCE_TOKEN_FILE": "${dir:state}/token", "MESH_CONFLUENCE_CONFIG_FILE": "${dir:state}/config.json" } + }, + { + "name": "files", + "kind": "archive", + "from": "files" } ] } diff --git a/modules/de-spiegel/module.json b/modules/de-spiegel/module.json deleted file mode 100644 index 6ace95ec..00000000 --- a/modules/de-spiegel/module.json +++ /dev/null @@ -1,70 +0,0 @@ -{ - "module": "de-spiegel", - "version": "1", - "slug": "spiegel", - "capabilities": [ - "container-runtime" - ], - "requires": [ - "route" - ], - "contributes": { - "route": { - "label": "de-spiegel", - "endpoint": "web" - } - }, - "binds": { - "route": "${dir:state}/route.json" - }, - "own-secrets": { - "smtp-user": "${dir:state}/smtp-user.secret", - "smtp-pass": "${dir:state}/smtp-pass.secret" - }, - "listens": [ - { - "name": "web", - "port": 35621, - "protocol": "tcp", - "from": "mesh", - "why": "the de-spiegel site and its /contact endpoint over http; its public name is a route grant, and route-proxy reaches it on this published port" - } - ], - "resources": [ - { - "id": "state", - "type": "directory", - "mode": "0700", - "place": "." - }, - { - "id": "server-env", - "type": "file", - "path": "${dir:state}/server.env", - "mode": "0600", - "content": "SMTP_AUTH_USER=${secret:smtp-user}\nSMTP_AUTH_PASS=${secret:smtp-pass}\n" - }, - { - "id": "net", - "type": "network", - "name": "de-spiegel" - }, - { - "id": "server", - "type": "container", - "name": "de-spiegel", - "image": "registry-api.novox.be/novox/de-spiegel@sha256:e144b72ce9c145870470d765343549f2c60211728cd118b9ff0e4029f36342ba", - "network": "de-spiegel", - "env-file": [ - "${dir:state}/server.env" - ], - "ports": [ - "35621" - ], - "secrets-in-environment": "the application's own code reads SMTP_AUTH_USER/PASS from the environment (de-spiegel server/index.js); converting is that repository's change", - "names-on-purpose": { - "registry-api.novox.be": "built outside the mesh, from the application's own repository, and pulled from the registry that built it; moves when that repository is a build source on the git seat (novox/hq ADR 0155, issue 122)" - } - } - ] -} diff --git a/modules/desk-channel/cmd/desk-channel/templates_test.go b/modules/desk-channel/cmd/desk-channel/templates_test.go new file mode 100644 index 00000000..7d0718b5 --- /dev/null +++ b/modules/desk-channel/cmd/desk-channel/templates_test.go @@ -0,0 +1,64 @@ +package main + +import ( + "encoding/json" + "os" + "path/filepath" + "testing" +) + +// The module's directory, where its manifest and its files/ are. +const moduleDir = "../.." + +// templateText is the text a file resource places on the machine: its template's, in the module's files/ +// (novox/hq ADR 0349 §2). A resource that names no template, or a template that is not there, fails the test. +func templateText(t *testing.T, r map[string]any) string { + t.Helper() + name, _ := r["template"].(string) + if name == "" { + t.Fatalf("resource %v names no template", r["id"]) + } + raw, err := os.ReadFile(filepath.Join(moduleDir, filepath.FromSlash(name))) + if err != nil { + t.Fatalf("resource %v names %s: %v", r["id"], name, err) + } + return string(raw) +} + +// No file resource carries its text inline: each names its template in files/, and the build packs files/ +// into the one archive the controller reads the templates back from (novox/hq ADR 0349 §2). +func TestEveryFileTakesItsTextFromATemplate(t *testing.T) { + raw, err := os.ReadFile(filepath.Join(moduleDir, "module.json")) + if err != nil { + t.Fatal(err) + } + var m struct { + Resources []map[string]any `json:"resources"` + Build struct { + Artifacts []map[string]any `json:"artifacts"` + } `json:"build"` + } + if err := json.Unmarshal(raw, &m); err != nil { + t.Fatal(err) + } + for _, r := range m.Resources { + if _, inline := r["content"]; inline { + t.Errorf("resource %v keeps its text inline: keep it in files/ and name it as the template", r["id"]) + } + if r["type"] == "file" { + templateText(t, r) + } + } + packs := 0 + for _, a := range m.Build.Artifacts { + if a["kind"] == "archive" && a["from"] == "files" { + packs++ + if a["name"] != "files" { + t.Errorf("the templates archive is named %v, not files", a["name"]) + } + } + } + if packs != 1 { + t.Errorf("the build packs files/ into %d archives, not one", packs) + } +} diff --git a/modules/desk-channel/files/settings.json b/modules/desk-channel/files/settings.json new file mode 100644 index 00000000..1ef16fd8 --- /dev/null +++ b/modules/desk-channel/files/settings.json @@ -0,0 +1,3 @@ +{ + "desktop-machines": [] +} diff --git a/modules/desk-channel/module.json b/modules/desk-channel/module.json index 309efbb4..3ee2dcc2 100644 --- a/modules/desk-channel/module.json +++ b/modules/desk-channel/module.json @@ -45,6 +45,14 @@ "tools": [ "desk_channel_status" ], + "config": { + "desktop-machines": { + "type": "list", + "description": "The machines, by node name, whose desktop shows the operator's questions and notifications, in the order they are tried; empty shows them on none.", + "trusted": true, + "default": [] + } + }, "resources": [ { "id": "account", @@ -66,8 +74,9 @@ "path": "${dir:state}/settings.json", "mode": "0600", "owner": "desk-channel", + "takes-config": ["desktop-machines"], "merge": "json", - "content": "{\n \"desktop-machines\": []\n}\n" + "template": "files/settings.json" } ], "build": { @@ -85,6 +94,11 @@ "env": { "MESH_DESK_SETTINGS": "${dir:state}/settings.json" } + }, + { + "name": "files", + "kind": "archive", + "from": "files" } ] } diff --git a/modules/dnsmasq/cmd/dnsmasq-tools/templates_test.go b/modules/dnsmasq/cmd/dnsmasq-tools/templates_test.go new file mode 100644 index 00000000..7d0718b5 --- /dev/null +++ b/modules/dnsmasq/cmd/dnsmasq-tools/templates_test.go @@ -0,0 +1,64 @@ +package main + +import ( + "encoding/json" + "os" + "path/filepath" + "testing" +) + +// The module's directory, where its manifest and its files/ are. +const moduleDir = "../.." + +// templateText is the text a file resource places on the machine: its template's, in the module's files/ +// (novox/hq ADR 0349 §2). A resource that names no template, or a template that is not there, fails the test. +func templateText(t *testing.T, r map[string]any) string { + t.Helper() + name, _ := r["template"].(string) + if name == "" { + t.Fatalf("resource %v names no template", r["id"]) + } + raw, err := os.ReadFile(filepath.Join(moduleDir, filepath.FromSlash(name))) + if err != nil { + t.Fatalf("resource %v names %s: %v", r["id"], name, err) + } + return string(raw) +} + +// No file resource carries its text inline: each names its template in files/, and the build packs files/ +// into the one archive the controller reads the templates back from (novox/hq ADR 0349 §2). +func TestEveryFileTakesItsTextFromATemplate(t *testing.T) { + raw, err := os.ReadFile(filepath.Join(moduleDir, "module.json")) + if err != nil { + t.Fatal(err) + } + var m struct { + Resources []map[string]any `json:"resources"` + Build struct { + Artifacts []map[string]any `json:"artifacts"` + } `json:"build"` + } + if err := json.Unmarshal(raw, &m); err != nil { + t.Fatal(err) + } + for _, r := range m.Resources { + if _, inline := r["content"]; inline { + t.Errorf("resource %v keeps its text inline: keep it in files/ and name it as the template", r["id"]) + } + if r["type"] == "file" { + templateText(t, r) + } + } + packs := 0 + for _, a := range m.Build.Artifacts { + if a["kind"] == "archive" && a["from"] == "files" { + packs++ + if a["name"] != "files" { + t.Errorf("the templates archive is named %v, not files", a["name"]) + } + } + } + if packs != 1 { + t.Errorf("the build packs files/ into %d archives, not one", packs) + } +} diff --git a/modules/dnsmasq/files/dnsmasq.conf b/modules/dnsmasq/files/dnsmasq.conf new file mode 100644 index 00000000..10f3926a --- /dev/null +++ b/modules/dnsmasq/files/dnsmasq.conf @@ -0,0 +1,87 @@ +# Managed by the mesh. dnsmasq's own defaults are replaced whole rather than +# patched, because this module owns the file and a patch would leave whatever +# was there before to be discovered later. +# +# This is one of the mesh's resolvers (novox/hq ADR 0194, 0223): it holds the +# `mesh-dns-resolver` seat, which more than one machine may hold, each answering +# the same names from the same roster. Every node and every container lists every +# holder and no public resolver, so whichever answers first gives the one answer. +# It holds the mesh's own names and nothing else (ADR 0191) — the roster is the +# mesh's, rendered into each holder; no other copy lives on any machine. + +# What the mesh computed: one wildcard per machine — its name and everything +# under it — and the mesh's own suffix as a local domain, so a name under it is +# answered here or not at all and is never asked upstream. Rewritten whenever a +# machine joins or leaves, which is why the service below restarts on it: a +# reload makes dnsmasq re-read hosts files, not its configuration, and a +# wildcard is configuration. +conf-file=/etc/mesh-resolver/nodes.conf + +# Every zone a module answers itself (ADR 0199): one forwarding line per zone, to +# the private address of the machine its module runs on and the port its +# answerer is published on. Nothing in a zone is answered here; what names a zone +# holds is the module's. Rewritten whenever a zone is declared or moves, and the +# service restarts on it for the same reason as above. +conf-file=/etc/mesh-resolver/zones.conf + +# Where it answers: this machine's private address, so every node — and every +# container on every node, which copies its machine's resolvers — can ask; and +# loopback, for this machine's own use. Never a LAN address: a device that is +# not a member cannot reach what the mesh's names point at, and a LAN's resolver +# is its router's (ADR 0194). +# +# Named as an ADDRESS and not as the interface that carries it, on purpose. A +# container's query is addressed to this address but arrives on the runtime's +# bridge, and dnsmasq checks every query against what it was told to answer on: +# an `interface=` line admits queries by the interface they arrive on, a +# `listen-address=` line by the address they are sent to. With `interface=mesh0` +# alone, a query from a container was received and dropped without a word +# (novox/hq 04-ISSUES/110). The address admits it whatever bridge it comes in on. +# +# bind-interfaces, and never bind-dynamic (novox/hq issue 348). bind-dynamic +# re-reads the machine's addresses on every change of any link and closes each +# listener whose interface that reading did not find; a reading made while the +# container runtime deletes a bridge's virtual links fails ("index gone"), and +# then every listener is closed until a later change reads cleanly. On +# 2026-10-09 the private address was left unbound for twenty minutes, the daemon +# running and silent. bind-interfaces binds each address once, at start, and +# never closes it. It cannot bind an address that is not there yet, so the unit +# waits for it (the drop-in below); a new private address restarts the unit, as +# this file changing always did. +bind-interfaces +listen-address=${machine:address} +listen-address=127.0.0.1 + +# **It must never read resolv.conf to find out where to forward.** This +# machine's resolv.conf names this resolver — so a resolver that read it for +# upstreams would find itself, and every query it could not answer locally would +# loop until its receive queue filled. That is not theoretical: it filled with +# 15KB of queries and every lookup on the machine hung. no-resolv makes that loop +# impossible: the upstreams are the two lines below, and nothing on the machine +# can redirect them. The mesh's own names never reach them: the local= line in +# the file above stops them here, answered or refused. +no-resolv +server=1.1.1.1 +server=8.8.8.8 + +# Both upstreams validate DNSSEC and say so with the Authenticated Data bit, +# and this passes that bit down rather than dropping it, which dnsmasq does +# unless told. It does not validate itself: an answer's trust is the upstream's +# and the path to it, which is what a forwarding resolver's trust always was. +# Without it a program that refuses to run behind a resolver that does not +# validate — a mail system's admin does exactly that check at start — cannot +# use this resolver (novox/hq 04-ISSUES/171). +proxy-dnssec + +# A name without a dot is never forwarded, and reverse lookups of private ranges +# are answered here rather than asking the world who 10.x is. +domain-needed +bogus-priv + +# **No hosts file, and no operator's files.** This machine's /etc/hosts is its +# own — the hosts module's block and the operator's lines (ADR 0199) — and the +# mesh's resolver answers every node, so a line written for one machine's own +# programs must not become an answer for all of them. Every per-node resolver +# went wrong exactly here: a hosts file read once at start, an operator's old +# line beside the mesh's, a drop-in read from a directory nobody owned. +no-hosts diff --git a/modules/dnsmasq/files/mesh.conf b/modules/dnsmasq/files/mesh.conf new file mode 100644 index 00000000..96d8d7d6 --- /dev/null +++ b/modules/dnsmasq/files/mesh.conf @@ -0,0 +1,14 @@ +# Written by the mesh (module dnsmasq, novox/hq issue 348). Replaced on every push. +# +# The resolver binds its addresses once, at start (bind-interfaces), so it starts only once this +# machine's private address is there: it waits up to 80 seconds for it, and a start that found it +# missing is tried again every ten seconds rather than left failed. +[Unit] +After=network-online.target +Wants=network-online.target +StartLimitIntervalSec=0 + +[Service] +ExecStartPre=/usr/bin/sh -c 'for i in $(seq 1 80); do ip -4 -o addr show to ${machine:address} | grep -q . && exit 0; sleep 1; done; echo "${machine:address} is not on this machine" >&2; exit 1' +Restart=on-failure +RestartSec=10s diff --git a/modules/dnsmasq/module.json b/modules/dnsmasq/module.json index d868d771..1865d9b7 100644 --- a/modules/dnsmasq/module.json +++ b/modules/dnsmasq/module.json @@ -70,7 +70,7 @@ "type": "file", "path": "/etc/dnsmasq.conf", "mode": "0644", - "content": "# Managed by the mesh. dnsmasq's own defaults are replaced whole rather than\n# patched, because this module owns the file and a patch would leave whatever\n# was there before to be discovered later.\n#\n# This is one of the mesh's resolvers (novox/hq ADR 0194, 0223): it holds the\n# `mesh-dns-resolver` seat, which more than one machine may hold, each answering\n# the same names from the same roster. Every node and every container lists every\n# holder and no public resolver, so whichever answers first gives the one answer.\n# It holds the mesh's own names and nothing else (ADR 0191) — the roster is the\n# mesh's, rendered into each holder; no other copy lives on any machine.\n\n# What the mesh computed: one wildcard per machine — its name and everything\n# under it — and the mesh's own suffix as a local domain, so a name under it is\n# answered here or not at all and is never asked upstream. Rewritten whenever a\n# machine joins or leaves, which is why the service below restarts on it: a\n# reload makes dnsmasq re-read hosts files, not its configuration, and a\n# wildcard is configuration.\nconf-file=/etc/mesh-resolver/nodes.conf\n\n# Every zone a module answers itself (ADR 0199): one forwarding line per zone, to\n# the private address of the machine its module runs on and the port its\n# answerer is published on. Nothing in a zone is answered here; what names a zone\n# holds is the module's. Rewritten whenever a zone is declared or moves, and the\n# service restarts on it for the same reason as above.\nconf-file=/etc/mesh-resolver/zones.conf\n\n# Where it answers: this machine's private address, so every node — and every\n# container on every node, which copies its machine's resolvers — can ask; and\n# loopback, for this machine's own use. Never a LAN address: a device that is\n# not a member cannot reach what the mesh's names point at, and a LAN's resolver\n# is its router's (ADR 0194).\n#\n# Named as an ADDRESS and not as the interface that carries it, on purpose. A\n# container's query is addressed to this address but arrives on the runtime's\n# bridge, and dnsmasq checks every query against what it was told to answer on:\n# an `interface=` line admits queries by the interface they arrive on, a\n# `listen-address=` line by the address they are sent to. With `interface=mesh0`\n# alone, a query from a container was received and dropped without a word\n# (novox/hq 04-ISSUES/110). The address admits it whatever bridge it comes in on.\n#\n# bind-interfaces, and never bind-dynamic (novox/hq issue 348). bind-dynamic\n# re-reads the machine's addresses on every change of any link and closes each\n# listener whose interface that reading did not find; a reading made while the\n# container runtime deletes a bridge's virtual links fails (\"index gone\"), and\n# then every listener is closed until a later change reads cleanly. On\n# 2026-10-09 the private address was left unbound for twenty minutes, the daemon\n# running and silent. bind-interfaces binds each address once, at start, and\n# never closes it. It cannot bind an address that is not there yet, so the unit\n# waits for it (the drop-in below); a new private address restarts the unit, as\n# this file changing always did.\nbind-interfaces\nlisten-address=${machine:address}\nlisten-address=127.0.0.1\n\n# **It must never read resolv.conf to find out where to forward.** This\n# machine's resolv.conf names this resolver — so a resolver that read it for\n# upstreams would find itself, and every query it could not answer locally would\n# loop until its receive queue filled. That is not theoretical: it filled with\n# 15KB of queries and every lookup on the machine hung. no-resolv makes that loop\n# impossible: the upstreams are the two lines below, and nothing on the machine\n# can redirect them. The mesh's own names never reach them: the local= line in\n# the file above stops them here, answered or refused.\nno-resolv\nserver=1.1.1.1\nserver=8.8.8.8\n\n# Both upstreams validate DNSSEC and say so with the Authenticated Data bit,\n# and this passes that bit down rather than dropping it, which dnsmasq does\n# unless told. It does not validate itself: an answer's trust is the upstream's\n# and the path to it, which is what a forwarding resolver's trust always was.\n# Without it a program that refuses to run behind a resolver that does not\n# validate — a mail system's admin does exactly that check at start — cannot\n# use this resolver (novox/hq 04-ISSUES/171).\nproxy-dnssec\n\n# A name without a dot is never forwarded, and reverse lookups of private ranges\n# are answered here rather than asking the world who 10.x is.\ndomain-needed\nbogus-priv\n\n# **No hosts file, and no operator's files.** This machine's /etc/hosts is its\n# own — the hosts module's block and the operator's lines (ADR 0199) — and the\n# mesh's resolver answers every node, so a line written for one machine's own\n# programs must not become an answer for all of them. Every per-node resolver\n# went wrong exactly here: a hosts file read once at start, an operator's old\n# line beside the mesh's, a drop-in read from a directory nobody owned.\nno-hosts\n" + "template": "files/dnsmasq.conf" }, { "id": "drop-ins", @@ -83,7 +83,7 @@ "type": "file", "path": "/etc/systemd/system/dnsmasq.service.d/mesh.conf", "mode": "0644", - "content": "# Written by the mesh (module dnsmasq, novox/hq issue 348). Replaced on every push.\n#\n# The resolver binds its addresses once, at start (bind-interfaces), so it starts only once this\n# machine's private address is there: it waits up to 80 seconds for it, and a start that found it\n# missing is tried again every ten seconds rather than left failed.\n[Unit]\nAfter=network-online.target\nWants=network-online.target\nStartLimitIntervalSec=0\n\n[Service]\nExecStartPre=/usr/bin/sh -c 'for i in $(seq 1 80); do ip -4 -o addr show to ${machine:address} | grep -q . && exit 0; sleep 1; done; echo \"${machine:address} is not on this machine\" >&2; exit 1'\nRestart=on-failure\nRestartSec=10s\n" + "template": "files/mesh.conf" }, { "id": "service", @@ -148,6 +148,11 @@ "loads": [ "dnsmasq-tools" ] + }, + { + "name": "files", + "kind": "archive", + "from": "files" } ] } diff --git a/modules/docker/cmd/docker-tools/logrotation_test.go b/modules/docker/cmd/docker-tools/logrotation_test.go index e969730f..381df1e2 100644 --- a/modules/docker/cmd/docker-tools/logrotation_test.go +++ b/modules/docker/cmd/docker-tools/logrotation_test.go @@ -11,14 +11,15 @@ import ( "testing" ) -// The container runtime's log rotation is a preference (novox/hq issue 452). +// The container runtime's log rotation is two config values with defaults (novox/hq issue 452, ADR 0348 §4). type manifestForLogs struct { - Settings map[string]struct { - Kind string `json:"kind"` - Default any `json:"default"` - Why string `json:"why"` - } `json:"settings"` + Config map[string]struct { + Type string `json:"type"` + Description string `json:"description"` + Default any `json:"default"` + Trusted bool `json:"trusted"` + } `json:"config"` Resources []map[string]any `json:"resources"` } @@ -44,16 +45,16 @@ func resourceByID(m manifestForLogs, id string) (map[string]any, int) { return nil, -1 } -// filled is a file's content as the controller fills it from the settings' defaults alone: each -// ${setting:} replaced by the default as text, the way the controller writes a value into a file. +// filled is a file's content as the controller fills it from the config values' defaults alone: each +// ${config:} replaced by the default as text, the way the controller writes a value into a file. func filled(t *testing.T, m manifestForLogs, content string) string { t.Helper() - ref := regexp.MustCompile(`\$\{setting:([a-z0-9][a-z0-9_.-]*)\}`) + ref := regexp.MustCompile(`\$\{config:([a-z0-9][a-z0-9_.-]*)\}`) var missing []string out := ref.ReplaceAllStringFunc(content, func(s string) string { key := ref.FindStringSubmatch(s)[1] - d, ok := m.Settings[key] - if !ok || d.Kind != "preference" { + d, ok := m.Config[key] + if !ok || d.Default == nil || d.Description == "" || !d.Trusted { missing = append(missing, key) return s } @@ -77,7 +78,7 @@ func TestTheDefaultLogRotationIsTheOneTheHandSetMachineHas(t *testing.T) { t.Fatal("no daemon resource") } var keys map[string]any - if err := json.Unmarshal([]byte(filled(t, m, daemon["content"].(string))), &keys); err != nil { + if err := json.Unmarshal([]byte(filled(t, m, templateText(t, daemon))), &keys); err != nil { t.Fatalf("daemon.json's keys with the defaults are not JSON: %v", err) } if keys["log-driver"] != "json-file" { @@ -125,7 +126,7 @@ func TestTheLogRotationIsCheckedBeforeDaemonJSONIsWritten(t *testing.T) { t.Errorf("log-rotation says trusted %v, want true", conf["trusted"]) } // The step reads the very values daemon.json takes: the same keys, filled the same way. - values, err := parseLogRotation(filled(t, m, conf["content"].(string))) + values, err := parseLogRotation(filled(t, m, templateText(t, conf))) if err != nil { t.Fatalf("the defaults are refused: %v", err) } @@ -133,8 +134,8 @@ func TestTheLogRotationIsCheckedBeforeDaemonJSONIsWritten(t *testing.T) { t.Errorf("defaults read as %+v", values) } daemon, _ := resourceByID(m, "daemon") - for _, key := range []string{"${setting:log-max-size}", "${setting:log-max-file}"} { - if !strings.Contains(daemon["content"].(string), key) || !strings.Contains(conf["content"].(string), key) { + for _, key := range []string{"${config:log-max-size}", "${config:log-max-file}"} { + if !strings.Contains(templateText(t, daemon), key) || !strings.Contains(templateText(t, conf), key) { t.Errorf("%s is not in both daemon and log-rotation", key) } } diff --git a/modules/docker/cmd/docker-tools/templates_test.go b/modules/docker/cmd/docker-tools/templates_test.go new file mode 100644 index 00000000..7d0718b5 --- /dev/null +++ b/modules/docker/cmd/docker-tools/templates_test.go @@ -0,0 +1,64 @@ +package main + +import ( + "encoding/json" + "os" + "path/filepath" + "testing" +) + +// The module's directory, where its manifest and its files/ are. +const moduleDir = "../.." + +// templateText is the text a file resource places on the machine: its template's, in the module's files/ +// (novox/hq ADR 0349 §2). A resource that names no template, or a template that is not there, fails the test. +func templateText(t *testing.T, r map[string]any) string { + t.Helper() + name, _ := r["template"].(string) + if name == "" { + t.Fatalf("resource %v names no template", r["id"]) + } + raw, err := os.ReadFile(filepath.Join(moduleDir, filepath.FromSlash(name))) + if err != nil { + t.Fatalf("resource %v names %s: %v", r["id"], name, err) + } + return string(raw) +} + +// No file resource carries its text inline: each names its template in files/, and the build packs files/ +// into the one archive the controller reads the templates back from (novox/hq ADR 0349 §2). +func TestEveryFileTakesItsTextFromATemplate(t *testing.T) { + raw, err := os.ReadFile(filepath.Join(moduleDir, "module.json")) + if err != nil { + t.Fatal(err) + } + var m struct { + Resources []map[string]any `json:"resources"` + Build struct { + Artifacts []map[string]any `json:"artifacts"` + } `json:"build"` + } + if err := json.Unmarshal(raw, &m); err != nil { + t.Fatal(err) + } + for _, r := range m.Resources { + if _, inline := r["content"]; inline { + t.Errorf("resource %v keeps its text inline: keep it in files/ and name it as the template", r["id"]) + } + if r["type"] == "file" { + templateText(t, r) + } + } + packs := 0 + for _, a := range m.Build.Artifacts { + if a["kind"] == "archive" && a["from"] == "files" { + packs++ + if a["name"] != "files" { + t.Errorf("the templates archive is named %v, not files", a["name"]) + } + } + } + if packs != 1 { + t.Errorf("the build packs files/ into %d archives, not one", packs) + } +} diff --git a/modules/docker/files/daemon.json b/modules/docker/files/daemon.json new file mode 100644 index 00000000..83750f4b --- /dev/null +++ b/modules/docker/files/daemon.json @@ -0,0 +1 @@ +{"live-restore": true, "insecure-registries": ["${seat:mesh-artifact-store:reach}"], "log-driver": "json-file", "log-opts": {"max-size": "${config:log-max-size}", "max-file": "${config:log-max-file}"}} diff --git a/modules/docker/files/docker-prune.service b/modules/docker/files/docker-prune.service new file mode 100644 index 00000000..b9b92c1d --- /dev/null +++ b/modules/docker/files/docker-prune.service @@ -0,0 +1,15 @@ +# Generated by the mesh. Do not edit — module docker writes this file and replaces it at every push. +[Unit] +Description=Prune dangling images and unused build cache (the mesh's docker module) +# Never volumes, never a container, never an image a container uses: dangling +# images and build cache nothing refers to, unused for a week. What a person +# prunes beyond that is docker_prune's, by hand. +After=docker.service +ConditionPathExists=/run/docker.sock + +[Service] +Type=oneshot +Nice=19 +IOSchedulingClass=idle +ExecStart=/usr/bin/docker image prune --force --filter until=168h +ExecStart=/usr/bin/docker builder prune --force --filter until=168h diff --git a/modules/docker/files/docker-prune.timer b/modules/docker/files/docker-prune.timer new file mode 100644 index 00000000..82befd15 --- /dev/null +++ b/modules/docker/files/docker-prune.timer @@ -0,0 +1,11 @@ +# Generated by the mesh. Do not edit — module docker writes this file and replaces it at every push. +[Unit] +Description=Weekly prune of dangling images and unused build cache (the mesh's docker module) + +[Timer] +OnCalendar=weekly +RandomizedDelaySec=1h +Persistent=true + +[Install] +WantedBy=timers.target diff --git a/modules/docker/files/log-rotation.conf b/modules/docker/files/log-rotation.conf new file mode 100644 index 00000000..39190368 --- /dev/null +++ b/modules/docker/files/log-rotation.conf @@ -0,0 +1,5 @@ +# The log rotation daemon.json takes, written by the mesh from this machine's settings log-max-size and +# log-max-file, and judged by the step log-rotation-check before daemon.json is written (novox/hq issue 452). +# Replaced on every push; change the settings, never this file. +size=${config:log-max-size} +files=${config:log-max-file} diff --git a/modules/docker/module.json b/modules/docker/module.json index 3d267b9d..80f0bed3 100644 --- a/modules/docker/module.json +++ b/modules/docker/module.json @@ -44,18 +44,23 @@ "docker exec cat /etc/resolv.conf" ] }, - "settings": { - "log-max-size": { - "kind": "preference", - "default": "100m", - "why": "each container's log is cut at this size and the runtime keeps log-max-file of them, so 100m and 10 hold about 1 GB per container: what the one machine with rotation set by hand has (novox/hq issue 452). A whole number from 1 with k, m or g after it, refused before daemon.json is written. The runtime reads it only at its next start, which the mesh never forces, and a container created before that keeps the options it was created with until it is created again" + "config": { + "log-max-size": { + "type": "string", + "description": "The size each container's log is cut at, a whole number from 1 with k, m or g after it; the runtime keeps log-max-file of them, so 100m and 10 hold about 1 GB per container (novox/hq issue 452). The runtime reads it only at its next start, which the mesh never forces, and a container created before that keeps the options it was created with until it is created again.", + "pattern": "[1-9][0-9]*[kmg]", + "trusted": true, + "default": "100m" + }, + "log-max-file": { + "type": "number", + "description": "How many of a container's log files of log-max-size the runtime keeps, a count from 1 to 99, the oldest dropped first; taken at the runtime's next start and by containers created after it, like log-max-size (novox/hq issue 452).", + "minimum": 1, + "maximum": 99, + "trusted": true, + "default": 10 + } }, - "log-max-file": { - "kind": "preference", - "default": 10, - "why": "how many of a container's log files of log-max-size the runtime keeps, the oldest dropped first; a count from 1 to 99, refused before daemon.json is written, taken at the runtime's next start and by containers created after it, like log-max-size (novox/hq issue 452)" - } - }, "resources": [ { "id": "package", @@ -86,7 +91,7 @@ "path": "${dir:state}/log-rotation.conf", "mode": "0600", "trusted": true, - "content": "# The log rotation daemon.json takes, written by the mesh from this machine's settings log-max-size and\n# log-max-file, and judged by the step log-rotation-check before daemon.json is written (novox/hq issue 452).\n# Replaced on every push; change the settings, never this file.\nsize=${setting:log-max-size}\nfiles=${setting:log-max-file}\n" + "template": "files/log-rotation.conf" }, { "id": "log-rotation-check", @@ -112,7 +117,7 @@ "mode": "0644", "into": "json", "trusted": true, - "content": "{\"live-restore\": true, \"insecure-registries\": [\"${seat:mesh-artifact-store:reach}\"], \"log-driver\": \"json-file\", \"log-opts\": {\"max-size\": \"${setting:log-max-size}\", \"max-file\": \"${setting:log-max-file}\"}}\n" + "template": "files/daemon.json" }, { "id": "runtime", @@ -129,14 +134,14 @@ "type": "file", "path": "/etc/systemd/system/docker-prune.service", "mode": "0644", - "content": "# Generated by the mesh. Do not edit — module docker writes this file and replaces it at every push.\n[Unit]\nDescription=Prune dangling images and unused build cache (the mesh's docker module)\n# Never volumes, never a container, never an image a container uses: dangling\n# images and build cache nothing refers to, unused for a week. What a person\n# prunes beyond that is docker_prune's, by hand.\nAfter=docker.service\nConditionPathExists=/run/docker.sock\n\n[Service]\nType=oneshot\nNice=19\nIOSchedulingClass=idle\nExecStart=/usr/bin/docker image prune --force --filter until=168h\nExecStart=/usr/bin/docker builder prune --force --filter until=168h\n" + "template": "files/docker-prune.service" }, { "id": "prune-timer", "type": "file", "path": "/etc/systemd/system/docker-prune.timer", "mode": "0644", - "content": "# Generated by the mesh. Do not edit — module docker writes this file and replaces it at every push.\n[Unit]\nDescription=Weekly prune of dangling images and unused build cache (the mesh's docker module)\n\n[Timer]\nOnCalendar=weekly\nRandomizedDelaySec=1h\nPersistent=true\n\n[Install]\nWantedBy=timers.target\n" + "template": "files/docker-prune.timer" }, { "id": "prune", @@ -162,6 +167,11 @@ "loads": [ "docker-tools" ] + }, + { + "name": "files", + "kind": "archive", + "from": "files" } ] } diff --git a/modules/dunst/cmd/dunst-tools/manifest_helpers_test.go b/modules/dunst/cmd/dunst-tools/manifest_helpers_test.go index 55f2f09e..2959a8b6 100644 --- a/modules/dunst/cmd/dunst-tools/manifest_helpers_test.go +++ b/modules/dunst/cmd/dunst-tools/manifest_helpers_test.go @@ -12,29 +12,39 @@ import ( // every desktop module that carries it. type manifest struct { - Module string `json:"module"` - Version string `json:"version"` - Capabilities []string `json:"capabilities"` - Requires []string `json:"requires"` - Claims []claim `json:"claims"` - Seats []any `json:"seats"` - Tools []string `json:"tools"` - Settings map[string]setting `json:"settings"` - Emits []string `json:"emits"` - Contributions []contribution `json:"contributions"` - Environment *environment `json:"environment"` - Shell []shellCode `json:"shell"` - Resources []map[string]any `json:"resources"` + Module string `json:"module"` + Version string `json:"version"` + Moved bool `json:"moved"` + Capabilities []string `json:"capabilities"` + Requires []string `json:"requires"` + Claims []claim `json:"claims"` + Seats []any `json:"seats"` + Tools []string `json:"tools"` + Config map[string]config `json:"config"` + Emits []string `json:"emits"` + Items []item `json:"items"` + Contributions []any `json:"contributions"` + Environment *environment `json:"environment"` + Shell []shellCode `json:"shell"` + Resources []map[string]any `json:"resources"` Build struct { Artifacts []map[string]any `json:"artifacts"` } `json:"build"` } -// setting is one key's default, as the definition gives it (novox/hq ADR 0262). -type setting struct { - Kind string `json:"kind"` - Default any `json:"default"` - Why string `json:"why"` +// config is one config value as the definition declares it (novox/hq ADR 0348 §4). +type config struct { + Type string `json:"type"` + Description string `json:"description"` + Enum []any `json:"enum"` + Minimum *float64 `json:"minimum"` + Maximum *float64 `json:"maximum"` + Pattern string `json:"pattern"` + Trusted bool `json:"trusted"` + Predefined string `json:"predefined"` + Default any `json:"default"` + Required bool `json:"required"` + Runtime bool `json:"runtime"` } type claim struct { @@ -93,26 +103,66 @@ func (m manifest) packages() (present, absent []string) { return present, absent } -// sameAsSource checks that a file resource's content is byte for byte the module's source file, so -// the readable file in the repository is what the machine gets. -func (m manifest) sameAsSource(t *testing.T, id, source string) { +// fromTemplate checks that a file resource takes its text from its template file in the module's +// repository (novox/hq ADR 0349 §2): it names the template, carries no text of its own beside it, and the +// template is there. The readable file in the repository is the one place the machine's text is written. +func (m manifest) fromTemplate(t *testing.T, id, template string) { t.Helper() - want, err := os.ReadFile(filepath.Join("..", "..", source)) - if err != nil { - t.Fatal(err) - } r := m.resource(t, id) if r["type"] != "file" { t.Fatalf("%s is a %v, not a file", id, r["type"]) } - if got, _ := r["content"].(string); got != string(want) { - t.Fatalf("resource %s's content is not %s: edit the source and copy it into module.json", id, source) + if r["template"] != template { + t.Fatalf("resource %s does not take its text from %s: template %v", id, template, r["template"]) + } + if _, inline := r["content"]; inline { + t.Fatalf("resource %s keeps its text inline beside its template: a file's text is in one place", id) + } + if _, err := os.Stat(filepath.Join("..", "..", template)); err != nil { + t.Fatalf("resource %s names %s: %v", id, template, err) } if r["owner"] != "${machine:account}" && !strings.HasPrefix(r["path"].(string), "/etc/") { t.Fatalf("%s under the home is the account's", id) } } +// everyFileIsATemplate checks that no file resource carries its text inline, and that the build packs +// files/ into the one archive the controller reads the templates back from (novox/hq ADR 0349 §2). +func (m manifest) everyFileIsATemplate(t *testing.T) { + t.Helper() + for _, r := range m.Resources { + if _, inline := r["content"]; inline { + t.Errorf("resource %v keeps its text inline: keep it in files/ and name it as the template", r["id"]) + } + } + packs := 0 + for _, a := range m.Build.Artifacts { + if a["kind"] == "archive" && a["from"] == "files" { + packs++ + if a["name"] != "files" { + t.Errorf("the templates archive is named %v, not files", a["name"]) + } + } + } + if packs != 1 { + t.Errorf("the build packs files/ into %d archives, not one", packs) + } +} + +// text is the text a file resource places on the machine: its template file's. +func (m manifest) text(t *testing.T, id string) string { + t.Helper() + name, _ := m.resource(t, id)["template"].(string) + if name == "" { + t.Fatalf("resource %s names no template", id) + } + raw, err := os.ReadFile(filepath.Join("..", "..", name)) + if err != nil { + t.Fatal(err) + } + return string(raw) +} + // checkTheToolsAgree checks that the manifest lists the module's own tools exactly, that the bundle // serves each seat verb the claims promise as ., and that the Go bundle is declared. func checkTheToolsAgree(t *testing.T, m manifest) { @@ -184,31 +234,49 @@ func checkNoSecretsOrInstallationNames(t *testing.T) { } } -type contribution struct { - Seat string `json:"seat"` - Kind string `json:"kind"` - Content string `json:"content"` -} +// displaySession is the collection the module's lines for the window manager are an item of. +const displaySession = "contribution:node-display-session:config" -// i3Lines is what the module contributes to the window manager. +// i3Lines is what the module contributes to the window manager: the text of its item of the collection +// contribution:node-display-session:config (design 57 L3, issue 848). func (m manifest) i3Lines() string { var out string - for _, c := range m.Contributions { - if c.Seat == "node-display-session" && c.Kind == "config" { - out += c.Content + for _, it := range m.Items { + if it.Collection == displaySession { + text, err := os.ReadFile(filepath.Join("..", "..", it.Template)) + if err != nil { + return "" + } + out += string(text) } } return out } -// i3LinesAreSource checks the window-manager contribution is the source file it is written from. +// i3LinesAreSource checks the window-manager lines are one item whose template is the source file, and that no +// old-form contribution to node-display-session is left. func (m manifest) i3LinesAreSource(t *testing.T, source string) { t.Helper() - want, err := os.ReadFile(filepath.Join("..", "..", source)) - if err != nil { - t.Fatal(err) + if len(m.Contributions) != 0 { + t.Fatalf("the module still holds contributions: the lines are an item of %s", displaySession) } - if m.i3Lines() != string(want) { - t.Fatalf("the contribution to node-display-session is not %s: edit the source and copy it into module.json", source) + var templates []string + for _, it := range m.Items { + if it.Collection == displaySession { + templates = append(templates, it.Template) + } + } + if len(templates) != 1 || templates[0] != source { + t.Fatalf("the items of %s are %v, want the one template %s", displaySession, templates, source) + } + if m.i3Lines() == "" { + t.Fatalf("the template %s holds no line", source) } } + +type item struct { + Collection string `json:"collection"` + Name string `json:"name"` + Attributes map[string]string `json:"attributes"` + Template string `json:"template"` +} diff --git a/modules/dunst/cmd/dunst-tools/manifest_test.go b/modules/dunst/cmd/dunst-tools/manifest_test.go index 2712ea02..3f56b5fd 100644 --- a/modules/dunst/cmd/dunst-tools/manifest_test.go +++ b/modules/dunst/cmd/dunst-tools/manifest_test.go @@ -29,7 +29,7 @@ func TestItClaimsTheNotifierSeatServingSendAndHistory(t *testing.T) { func TestItOwnsItsFileAndTheDropInDirectory(t *testing.T) { m := readManifest(t) - m.sameAsSource(t, "configuration", "files/dunstrc") + m.fromTemplate(t, "configuration", "files/dunstrc") if p := m.resource(t, "configuration")["path"]; p != "${machine:account-home}/.config/dunst/dunstrc" { t.Fatalf("path: %v", p) } @@ -45,9 +45,9 @@ func TestItOwnsItsFileAndTheDropInDirectory(t *testing.T) { func TestTheFileIsTheDecidedOneAndCallsRofi(t *testing.T) { m := readManifest(t) - c := m.resource(t, "configuration")["content"].(string) - for _, want := range []string{"origin = top-right", "transparency = 0", "corner_radius = 0", "font = Inter ${setting:font-size}", - "width = ${setting:width}", "dmenu = rofi -dmenu -no-custom -p dunst"} { + c := m.text(t, "configuration") + for _, want := range []string{"origin = top-right", "transparency = 0", "corner_radius = 0", "font = Inter ${config:font-size}", + "width = ${config:width}", "dmenu = rofi -dmenu -no-custom -p dunst"} { if !strings.Contains(c, " "+want+"\n") { t.Errorf("lacks %q", want) } @@ -67,7 +67,7 @@ func TestTheFileIsTheDecidedOneAndCallsRofi(t *testing.T) { // actions closes. Every other click is as it was. func TestATapChoosesTheNotificationsAction(t *testing.T) { m := readManifest(t) - c := m.resource(t, "configuration")["content"].(string) + c := m.text(t, "configuration") for _, want := range []string{"mouse_left_click = do_action, close_current", "mouse_middle_click = do_action, close_current", "mouse_right_click = close_all"} { if !strings.Contains(c, " "+want+"\n") { @@ -95,31 +95,37 @@ func TestTheMenuKeyIsABindingI3Places(t *testing.T) { } } -// The font size and the width are preferences with today's values as their defaults (novox/hq ADR -// 0262): a machine with a denser screen sets its own, and every other machine's file is unchanged. -func TestTheFontSizeAndWidthAreSettingsWithTodaysDefaults(t *testing.T) { +// The font size and the width are config values with today's values as their defaults (novox/hq ADR +// 0348 §4): numbers with their unit said and their bounds, the font size falling back to the mesh's +// predefined font-size (ADR 0348 §2), so a machine with a denser screen sets its own and every other +// machine's file is unchanged. Neither is trusted: the file says "trusted": false. +func TestTheFontSizeAndWidthAreConfigValuesWithTodaysDefaults(t *testing.T) { m := readManifest(t) want := map[string]float64{"font-size": 10, "width": 250} - if len(m.Settings) != len(want) { - t.Fatalf("settings: %+v", m.Settings) + if len(m.Config) != len(want) { + t.Fatalf("config: %+v", m.Config) } for key, value := range want { - s, ok := m.Settings[key] - if !ok || s.Kind != "preference" || s.Default != value || strings.TrimSpace(s.Why) == "" { - t.Errorf("%s: %+v, want a preference defaulting to %v with why", key, s, value) + s, ok := m.Config[key] + if !ok || s.Type != "number" || s.Default != value || s.Required || s.Trusted || + strings.TrimSpace(s.Description) == "" || s.Minimum == nil || s.Maximum == nil { + t.Errorf("%s: %+v, want an untrusted number defaulting to %v, with a description and bounds", key, s, value) } } - // Filled with the defaults, the file is the one every machine had before it took settings. - c := m.resource(t, "configuration")["content"].(string) - c = strings.ReplaceAll(c, "${setting:font-size}", "10") - c = strings.ReplaceAll(c, "${setting:width}", "250") + if p := m.Config["font-size"].Predefined; p != "font-size" { + t.Errorf("the font size falls back to %q, want the predefined font-size", p) + } + // Filled with the defaults, the file is the one every machine had before it took config values. + c := m.text(t, "configuration") + c = strings.ReplaceAll(c, "${config:font-size}", "10") + c = strings.ReplaceAll(c, "${config:width}", "250") for _, want := range []string{"font = Inter 10", "width = 250"} { if !strings.Contains(c, " "+want+"\n") { t.Errorf("filled with the defaults, lacks %q", want) } } - if strings.Contains(c, "${setting:") { - t.Errorf("asks for a setting it gives no default: %s", c) + if strings.Contains(c, "${config:") || strings.Contains(c, "${setting:") { + t.Errorf("asks for a config value it gives no default: %s", c) } } @@ -145,3 +151,21 @@ func TestTheToolsAgreeWithTheManifest(t *testing.T) { checkTheToolsAgree(t, m) checkNoSecretsOrInstallationNames(t) } + +// No file resource carries its text inline (novox/hq ADR 0349 §2). +func TestEveryFileTakesItsTextFromATemplate(t *testing.T) { + readManifest(t).everyFileIsATemplate(t) +} + +// dunst moves first (design 57 P6, issue 672): marked moved, its dunstrc is filled on the node by the provisioner's +// client from its template, and reloaded as the file declares. +func TestItIsHandedToTheProvisioner(t *testing.T) { + m := readManifest(t) + if !m.Moved { + t.Fatal("dunst is not marked moved") + } + m.fromTemplate(t, "configuration", "files/dunstrc") + if on := m.resource(t, "notifier")["reload-on"]; !reflect.DeepEqual(on, []any{"configuration"}) { + t.Fatalf("the notifier is not reloaded on its dunstrc: %v", on) + } +} diff --git a/modules/dunst/files/dunstrc b/modules/dunst/files/dunstrc index 40d3eb71..afba676f 100644 --- a/modules/dunst/files/dunstrc +++ b/modules/dunst/files/dunstrc @@ -13,7 +13,7 @@ follow = none # Geometry - width = ${setting:width} + width = ${config:width} height = (0, 300) origin = top-right offset = (10, 50) @@ -39,7 +39,7 @@ corner_radius = 0 # Text: the interface face (research 026/04) - font = Inter ${setting:font-size} + font = Inter ${config:font-size} line_height = 0 markup = full format = "%s\n%b" diff --git a/modules/dunst/module.json b/modules/dunst/module.json index f94d5d96..fa4923db 100644 --- a/modules/dunst/module.json +++ b/modules/dunst/module.json @@ -1,6 +1,7 @@ { "module": "dunst", "version": "1", + "moved": true, "capabilities": [ "package-manager" ], @@ -22,18 +23,23 @@ "dunst_count", "dunst_reload" ], - "settings": { - "font-size": { - "kind": "preference", - "default": 10, - "why": "10 points of Inter reads well on a screen of about 100 DPI; a denser screen needs a larger size" + "config": { + "font-size": { + "type": "number", + "description": "The size of a notification's text, in points of Inter: 10 reads well on a screen of about 100 DPI, and a denser screen needs a larger size.", + "minimum": 6, + "maximum": 72, + "predefined": "font-size", + "default": 10 + }, + "width": { + "type": "number", + "description": "The width of a notification, in pixels: 250 fits a title of about forty characters at 10 points, and a larger font needs a wider notification.", + "minimum": 50, + "maximum": 4000, + "default": 250 + } }, - "width": { - "kind": "preference", - "default": 250, - "why": "250 pixels fits a title of about forty characters at 10 points; a larger font needs a wider notification" - } - }, "resources": [ { "id": "package", @@ -66,7 +72,7 @@ "path": "${machine:account-home}/.config/dunst/dunstrc", "owner": "${machine:account}", "mode": "0644", - "content": "# dunst, the notifier (module dunst, novox/hq ADR 0208). Owned by the mesh: this file is replaced\n# at every push. Adopted from the laptop's file of 2026-10-04 (the two workstations differed in\n# position, transparency and corner radius; the laptop's square, opaque, top-right one matches the\n# rest of the desktop). Only what differs from dunst's defaults, and what the desktop relies on.\n#\n# Other modules' rules go in ~/.config/dunst/dunstrc.d/*.conf, which dunst reads after this file,\n# so a drop-in outranks it. dunst is started by D-Bus on the first notification: nothing starts it.\n# When the mesh rewrites this file (a font-size or width setting changed), it tells a running dunst\n# to read it again (its unit's reload, dunstctl reload); what is on screen stays.\n\n[global]\n monitor = 0\n follow = none\n\n # Geometry\n width = ${setting:width}\n height = (0, 300)\n origin = top-right\n offset = (10, 50)\n notification_limit = 20\n\n progress_bar = true\n progress_bar_height = 10\n progress_bar_frame_width = 1\n progress_bar_min_width = 150\n progress_bar_max_width = 300\n\n indicate_hidden = yes\n transparency = 0\n separator_height = 2\n padding = 8\n horizontal_padding = 8\n text_icon_padding = 0\n frame_width = 3\n frame_color = \"#de5200\"\n gap_size = 0\n separator_color = frame\n sort = yes\n corner_radius = 0\n\n # Text: the interface face (research 026/04)\n font = Inter ${setting:font-size}\n line_height = 0\n markup = full\n format = \"%s\\n%b\"\n alignment = left\n vertical_alignment = center\n show_age_threshold = 60\n ellipsize = middle\n ignore_newline = no\n stack_duplicates = true\n hide_duplicate_count = false\n show_indicators = yes\n\n # Icons, from the desktop's icon theme\n enable_recursive_icon_lookup = true\n icon_theme = Adwaita\n icon_position = left\n min_icon_size = 32\n max_icon_size = 128\n\n # History\n sticky_history = yes\n history_length = 20\n\n # The context menu is rofi (installed by the rofi module); -no-custom keeps typed text from becoming a choice. Not `dmenu`, the node-launcher's command in\n # ~/.local/bin: dunst runs in the account's service manager, whose PATH does not hold that\n # directory (seen on the laptop on 2026-10-10), so the menu never opened. Links open in the\n # desktop's default browser.\n dmenu = rofi -dmenu -no-custom -p dunst\n browser = /usr/bin/xdg-open\n always_run_script = true\n\n title = Dunst\n class = Dunst\n ignore_dbusclose = false\n\n # A tap answers (the touchpads have no middle button). do_action runs the notification's only\n # action, or opens the context menu when it has several; without actions it opens its one link,\n # if it has one. The notification then closes; a menu dismissed with Escape leaves it shown.\n mouse_left_click = do_action, close_current\n mouse_middle_click = do_action, close_current\n mouse_right_click = close_all\n\n[urgency_low]\n background = \"#000000\"\n foreground = \"#ffffff\"\n timeout = 10\n\n[urgency_normal]\n background = \"#000000\"\n foreground = \"#ffffff\"\n timeout = 10\n\n[urgency_critical]\n background = \"#000000\"\n foreground = \"#ffffff\"\n frame_color = \"#ff0000\"\n timeout = 0\n\n# What is shown once is never kept (the review of 2026-10-09): a notification marked secret (a link's\n# code, category mesh.secret) and any transient one leave no entry in the history, which\n# node-notifier.history serves to every caller of the mesh's verbs.\n[mesh-secret]\n category = \"mesh.secret\"\n history_ignore = yes\n\n[transient-history-ignore]\n match_transient = yes\n history_ignore = yes\n" + "template": "files/dunstrc" }, { "id": "notifier", @@ -91,6 +97,11 @@ "loads": [ "dunst-tools" ] + }, + { + "name": "files", + "kind": "archive", + "from": "files" } ] }, @@ -99,11 +110,11 @@ "started", "paused" ], - "contributions": [ + "items": [ { - "seat": "node-display-session", - "kind": "config", - "content": "# The notifications' key (module dunst, novox/hq ADR 0208). Owned by the mesh: replaced at every push.\n# Super+N opens the menu (rofi) of the actions and links of every notification on screen, so one is\n# chosen from the keyboard as a tap chooses it on the notification.\nbindsym $mod+n exec --no-startup-id dunstctl context\n" + "collection": "contribution:node-display-session:config", + "name": "display-session", + "template": "files/i3/50-dunst.conf" } ] } diff --git a/modules/fail2ban/cmd/fail2ban-tools/own_test.go b/modules/fail2ban/cmd/fail2ban-tools/own_test.go index 293257b0..8a8699f2 100644 --- a/modules/fail2ban/cmd/fail2ban-tools/own_test.go +++ b/modules/fail2ban/cmd/fail2ban-tools/own_test.go @@ -267,7 +267,7 @@ func TestTheManifestGivesEveryJailTheListAndTheWatchItsHealth(t *testing.T) { for _, r := range m.Resources { byID[r["id"].(string)] = r } - base, _ := byID["jail-local"]["content"].(string) + base := templateText(t, byID["jail-local"]) var ignoreip []string for _, l := range strings.Split(base, "\n") { if strings.HasPrefix(l, "ignoreip") { @@ -278,17 +278,20 @@ func TestTheManifestGivesEveryJailTheListAndTheWatchItsHealth(t *testing.T) { t.Errorf("the base ignore list: %q", ignoreip) } for id, r := range byID { - if content, _ := r["content"].(string); id != "jail-local" && strings.Contains(content, "ignoreip") { + if r["type"] != "file" || id == "jail-local" { + continue + } + if strings.Contains(templateText(t, r), "ignoreip") { t.Errorf("%s sets an ignore list of its own, which leaves the operator's own addresses out", id) } } setting := byID["own-addresses"] if setting["path"] != SettingFile || setting["merge"] != "json" || setting["trusted"] != true || - !reflect.DeepEqual(setting["takes-settings"], []any{SettingKey}) { + !reflect.DeepEqual(setting["takes-config"], []any{SettingKey}) { t.Errorf("the setting's file: %v", setting) } var content map[string]any - if err := json.Unmarshal([]byte(setting["content"].(string)), &content); err != nil || + if err := json.Unmarshal([]byte(templateText(t, setting)), &content); err != nil || !reflect.DeepEqual(content, map[string]any{SettingKey: []any{}}) { t.Errorf("the setting starts from %v (%v)", content, err) } diff --git a/modules/fail2ban/cmd/fail2ban-tools/templates_test.go b/modules/fail2ban/cmd/fail2ban-tools/templates_test.go new file mode 100644 index 00000000..7d0718b5 --- /dev/null +++ b/modules/fail2ban/cmd/fail2ban-tools/templates_test.go @@ -0,0 +1,64 @@ +package main + +import ( + "encoding/json" + "os" + "path/filepath" + "testing" +) + +// The module's directory, where its manifest and its files/ are. +const moduleDir = "../.." + +// templateText is the text a file resource places on the machine: its template's, in the module's files/ +// (novox/hq ADR 0349 §2). A resource that names no template, or a template that is not there, fails the test. +func templateText(t *testing.T, r map[string]any) string { + t.Helper() + name, _ := r["template"].(string) + if name == "" { + t.Fatalf("resource %v names no template", r["id"]) + } + raw, err := os.ReadFile(filepath.Join(moduleDir, filepath.FromSlash(name))) + if err != nil { + t.Fatalf("resource %v names %s: %v", r["id"], name, err) + } + return string(raw) +} + +// No file resource carries its text inline: each names its template in files/, and the build packs files/ +// into the one archive the controller reads the templates back from (novox/hq ADR 0349 §2). +func TestEveryFileTakesItsTextFromATemplate(t *testing.T) { + raw, err := os.ReadFile(filepath.Join(moduleDir, "module.json")) + if err != nil { + t.Fatal(err) + } + var m struct { + Resources []map[string]any `json:"resources"` + Build struct { + Artifacts []map[string]any `json:"artifacts"` + } `json:"build"` + } + if err := json.Unmarshal(raw, &m); err != nil { + t.Fatal(err) + } + for _, r := range m.Resources { + if _, inline := r["content"]; inline { + t.Errorf("resource %v keeps its text inline: keep it in files/ and name it as the template", r["id"]) + } + if r["type"] == "file" { + templateText(t, r) + } + } + packs := 0 + for _, a := range m.Build.Artifacts { + if a["kind"] == "archive" && a["from"] == "files" { + packs++ + if a["name"] != "files" { + t.Errorf("the templates archive is named %v, not files", a["name"]) + } + } + } + if packs != 1 { + t.Errorf("the build packs files/ into %d archives, not one", packs) + } +} diff --git a/modules/fail2ban/files/fail2ban b/modules/fail2ban/files/fail2ban new file mode 100644 index 00000000..b5395db6 --- /dev/null +++ b/modules/fail2ban/files/fail2ban @@ -0,0 +1,7 @@ +/var/log/fail2ban.log { + missingok + notifempty + postrotate + /usr/bin/fail2ban-client flushlogs >/dev/null || true + endscript +} diff --git a/modules/fail2ban/files/iptables-allports-dualchain.conf b/modules/fail2ban/files/iptables-allports-dualchain.conf new file mode 100644 index 00000000..2c9fc73d --- /dev/null +++ b/modules/fail2ban/files/iptables-allports-dualchain.conf @@ -0,0 +1,40 @@ +# Fail2Ban action: ban in both INPUT and DOCKER-USER chains +# Used by recidive to block repeat offenders from both host and Docker services + +[INCLUDES] + +before = iptables.conf + +[Definition] + +type = allports + +actionstart = { -C f2b- -j >/dev/null 2>&1; } || { -N f2b- || true; -A f2b- -j ; } + { -C INPUT -p -j f2b- >/dev/null 2>&1; } || { -I INPUT -p -j f2b-; } + { -C DOCKER-USER -p -j f2b- >/dev/null 2>&1; } || { -I DOCKER-USER -p -j f2b-; } + +actionstop = -D INPUT -p -j f2b- 2>/dev/null || true + -D DOCKER-USER -p -j f2b- 2>/dev/null || true + -F f2b- + -X f2b- + +actioncheck = -n -L f2b- >/dev/null + +actionban = -I f2b- 1 -s -j + +actionunban = -D f2b- -s -j + +[Init] + +chain = INPUT +name = default +protocol = tcp +blocktype = REJECT --reject-with icmp-port-unreachable +returntype = RETURN +lockingopt = -w +iptables = iptables + +[Init?family=inet6] + +blocktype = REJECT --reject-with icmp6-port-unreachable +iptables = ip6tables diff --git a/modules/fail2ban/files/jail.local b/modules/fail2ban/files/jail.local new file mode 100644 index 00000000..1fa0ffe7 --- /dev/null +++ b/modules/fail2ban/files/jail.local @@ -0,0 +1,48 @@ +[INCLUDES] + +before = paths-arch.conf + +[DEFAULT] + +# Never act on the machine itself or on a tunnel peer: the mesh's private range is +# ${machine:mesh-range}, named here rather than written as a value the module cannot +# know (novox/hq ADR 0112). Without this, fail2ban could ban the mesh's own nodes. +# **A ban list never holds a neighbour.** The mesh's own range is named rather than written +# (novox/hq ADR 0112), and every private range beside it: a source on one is somebody's own +# network, not the internet. On a machine behind a router that reflects local traffic, every +# client in the house arrives as the gateway's address — so one mistyped local request banned +# 192.168.1.1 on the home server and would have cut the whole house off from it (ADR 0186). +# +# **Nor the operator's own addresses** (novox/hq ADR 0339, issue 238): the public addresses the operator names +# as theirs, in the mesh-wide setting own-addresses. An agent polling the forge from the operator's home banned +# that home for four weeks on 2026-10-04, and nothing in the house reached the control-node until it was found +# among 400 bans. The file is written by this module's watch from the setting, one entry per line; fail2ban +# reads it again within a second of a change, so a new value needs no restart. Empty until the operator +# fills the setting. A jail that sets its own ignoreip leaves all of this out; the watch says so. +ignoreip = 127.0.0.1/8 ::1 ${machine:mesh-range} 10.0.0.0/8 172.16.0.0/12 192.168.0.0/16 169.254.0.0/16 fc00::/7 fe80::/10 file:/var/lib/mesh-fail2ban/own-addresses + +# Three failures in a day ban for a day (novox/hq ADR 0179). The attackers this mesh sees pace +# themselves at one try every ten minutes, under any ten-minute window; a day's window counts +# them, and a day's ban costs a person who mistyped three times once, from one address, while +# the mesh's own range is never banned at all. +bantime = 1d +findtime = 1d +maxretry = 3 + +# Ban through iptables, not through a firewall front-end the machine may not have. ufw is +# installed on two of this mesh's machines and absent on the other two, and fail2ban finds out +# only at ban time: the service reports healthy, the jail counts the attempt, the ban command +# exits 127, and nothing is blocked. Proven on 2026-09-28 -- 'ufw: command not found' on a +# machine the mesh reported as protected. +# +# The action below is this module's own, already used by the recidive jail on every machine +# here, and it bans in DOCKER-USER as well as INPUT, so a container's published port is +# covered too. +banaction = iptables-allports-dualchain +banaction_allports = iptables-allports-dualchain + +[sshd] +enabled = true +port = ssh +logpath = %(sshd_log)s +backend = %(sshd_backend)s diff --git a/modules/fail2ban/files/mesh-own-addresses.json b/modules/fail2ban/files/mesh-own-addresses.json new file mode 100644 index 00000000..0eb96046 --- /dev/null +++ b/modules/fail2ban/files/mesh-own-addresses.json @@ -0,0 +1,3 @@ +{ + "own-addresses": [] +} diff --git a/modules/fail2ban/files/own-addresses b/modules/fail2ban/files/own-addresses new file mode 100644 index 00000000..e69de29b diff --git a/modules/fail2ban/files/recidive.conf b/modules/fail2ban/files/recidive.conf new file mode 100644 index 00000000..1f8786fc --- /dev/null +++ b/modules/fail2ban/files/recidive.conf @@ -0,0 +1,9 @@ +[recidive] +enabled = true +logpath = /var/log/fail2ban.log +# Ban in both INPUT (host services like SSH) and DOCKER-USER (container services) +banaction = iptables-allports-dualchain +# Banned twice in two weeks, by any jail, is banned for four (novox/hq ADR 0179). +bantime = 4w +findtime = 2w +maxretry = 2 diff --git a/modules/fail2ban/files/sshd.conf b/modules/fail2ban/files/sshd.conf new file mode 100644 index 00000000..c8a30f29 --- /dev/null +++ b/modules/fail2ban/files/sshd.conf @@ -0,0 +1,8 @@ +[sshd] +enabled = true +port = ssh +logpath = %(sshd_log)s +backend = %(sshd_backend)s +maxretry = 3 +findtime = 1d +bantime = 1d diff --git a/modules/fail2ban/module.json b/modules/fail2ban/module.json index 89158bbe..d2aa2a65 100644 --- a/modules/fail2ban/module.json +++ b/modules/fail2ban/module.json @@ -24,6 +24,14 @@ "into": "/etc/fail2ban/jail.d/mesh.conf", "filter-into": "/etc/fail2ban/filter.d" }, + "config": { + "own-addresses": { + "type": "list", + "description": "The addresses and networks outside the mesh that are the operator's own, each an IP address or a CIDR network: every jail ignores them beside the private ranges, the ban verb refuses them, and a failure a jail counted from one is said as a warning; empty is none.", + "trusted": true, + "default": [] + } + }, "resources": [ { "id": "package", @@ -42,7 +50,7 @@ "path": "/var/lib/mesh-fail2ban/own-addresses", "mode": "0644", "create-once": true, - "content": "" + "template": "files/own-addresses" }, { "id": "jail-d", @@ -73,14 +81,14 @@ "type": "file", "path": "/etc/fail2ban/jail.local", "mode": "0644", - "content": "[INCLUDES]\n\nbefore = paths-arch.conf\n\n[DEFAULT]\n\n# Never act on the machine itself or on a tunnel peer: the mesh's private range is\n# ${machine:mesh-range}, named here rather than written as a value the module cannot\n# know (novox/hq ADR 0112). Without this, fail2ban could ban the mesh's own nodes.\n# **A ban list never holds a neighbour.** The mesh's own range is named rather than written\n# (novox/hq ADR 0112), and every private range beside it: a source on one is somebody's own\n# network, not the internet. On a machine behind a router that reflects local traffic, every\n# client in the house arrives as the gateway's address — so one mistyped local request banned\n# 192.168.1.1 on the home server and would have cut the whole house off from it (ADR 0186).\n#\n# **Nor the operator's own addresses** (novox/hq ADR 0339, issue 238): the public addresses the operator names\n# as theirs, in the mesh-wide setting own-addresses. An agent polling the forge from the operator's home banned\n# that home for four weeks on 2026-10-04, and nothing in the house reached the control-node until it was found\n# among 400 bans. The file is written by this module's watch from the setting, one entry per line; fail2ban\n# reads it again within a second of a change, so a new value needs no restart. Empty until the operator\n# fills the setting. A jail that sets its own ignoreip leaves all of this out; the watch says so.\nignoreip = 127.0.0.1/8 ::1 ${machine:mesh-range} 10.0.0.0/8 172.16.0.0/12 192.168.0.0/16 169.254.0.0/16 fc00::/7 fe80::/10 file:/var/lib/mesh-fail2ban/own-addresses\n\n# Three failures in a day ban for a day (novox/hq ADR 0179). The attackers this mesh sees pace\n# themselves at one try every ten minutes, under any ten-minute window; a day's window counts\n# them, and a day's ban costs a person who mistyped three times once, from one address, while\n# the mesh's own range is never banned at all.\nbantime = 1d\nfindtime = 1d\nmaxretry = 3\n\n# Ban through iptables, not through a firewall front-end the machine may not have. ufw is\n# installed on two of this mesh's machines and absent on the other two, and fail2ban finds out\n# only at ban time: the service reports healthy, the jail counts the attempt, the ban command\n# exits 127, and nothing is blocked. Proven on 2026-09-28 -- 'ufw: command not found' on a\n# machine the mesh reported as protected.\n#\n# The action below is this module's own, already used by the recidive jail on every machine\n# here, and it bans in DOCKER-USER as well as INPUT, so a container's published port is\n# covered too.\nbanaction = iptables-allports-dualchain\nbanaction_allports = iptables-allports-dualchain\n\n[sshd]\nenabled = true\nport = ssh\nlogpath = %(sshd_log)s\nbackend = %(sshd_backend)s\n" + "template": "files/jail.local" }, { "id": "jail-sshd", "type": "file", "path": "/etc/fail2ban/jail.d/sshd.conf", "mode": "0644", - "content": "[sshd]\nenabled = true\nport = ssh\nlogpath = %(sshd_log)s\nbackend = %(sshd_backend)s\nmaxretry = 3\nfindtime = 1d\nbantime = 1d\n" + "template": "files/sshd.conf" }, { "id": "own-addresses", @@ -88,11 +96,11 @@ "path": "/etc/fail2ban/mesh-own-addresses.json", "mode": "0644", "merge": "json", - "takes-settings": [ + "takes-config": [ "own-addresses" ], "trusted": true, - "content": "{\n \"own-addresses\": []\n}\n" + "template": "files/mesh-own-addresses.json" }, { "id": "watch", @@ -118,28 +126,28 @@ "path": "/var/log/fail2ban.log", "mode": "0640", "create-once": true, - "content": "" + "template": "files/own-addresses" }, { "id": "jail-recidive", "type": "file", "path": "/etc/fail2ban/jail.d/recidive.conf", "mode": "0644", - "content": "[recidive]\nenabled = true\nlogpath = /var/log/fail2ban.log\n# Ban in both INPUT (host services like SSH) and DOCKER-USER (container services)\nbanaction = iptables-allports-dualchain\n# Banned twice in two weeks, by any jail, is banned for four (novox/hq ADR 0179).\nbantime = 4w\nfindtime = 2w\nmaxretry = 2\n" + "template": "files/recidive.conf" }, { "id": "action-dualchain", "type": "file", "path": "/etc/fail2ban/action.d/iptables-allports-dualchain.conf", "mode": "0644", - "content": "# Fail2Ban action: ban in both INPUT and DOCKER-USER chains\n# Used by recidive to block repeat offenders from both host and Docker services\n\n[INCLUDES]\n\nbefore = iptables.conf\n\n[Definition]\n\ntype = allports\n\nactionstart = { -C f2b- -j >/dev/null 2>&1; } || { -N f2b- || true; -A f2b- -j ; }\n { -C INPUT -p -j f2b- >/dev/null 2>&1; } || { -I INPUT -p -j f2b-; }\n { -C DOCKER-USER -p -j f2b- >/dev/null 2>&1; } || { -I DOCKER-USER -p -j f2b-; }\n\nactionstop = -D INPUT -p -j f2b- 2>/dev/null || true\n -D DOCKER-USER -p -j f2b- 2>/dev/null || true\n -F f2b-\n -X f2b-\n\nactioncheck = -n -L f2b- >/dev/null\n\nactionban = -I f2b- 1 -s -j \n\nactionunban = -D f2b- -s -j \n\n[Init]\n\nchain = INPUT\nname = default\nprotocol = tcp\nblocktype = REJECT --reject-with icmp-port-unreachable\nreturntype = RETURN\nlockingopt = -w\niptables = iptables \n\n[Init?family=inet6]\n\nblocktype = REJECT --reject-with icmp6-port-unreachable\niptables = ip6tables \n" + "template": "files/iptables-allports-dualchain.conf" }, { "id": "logrotate", "type": "file", "path": "/etc/logrotate.d/fail2ban", "mode": "0644", - "content": "/var/log/fail2ban.log {\n missingok\n notifempty\n postrotate\n /usr/bin/fail2ban-client flushlogs >/dev/null || true\n endscript\n}\n" + "template": "files/fail2ban" }, { "id": "run", @@ -171,6 +179,11 @@ "loads": [ "fail2ban-tools" ] + }, + { + "name": "files", + "kind": "archive", + "from": "files" } ] } diff --git a/modules/feh/cmd/feh-tools/manifest_helpers_test.go b/modules/feh/cmd/feh-tools/manifest_helpers_test.go index 441415f9..28adb660 100644 --- a/modules/feh/cmd/feh-tools/manifest_helpers_test.go +++ b/modules/feh/cmd/feh-tools/manifest_helpers_test.go @@ -14,6 +14,8 @@ import ( type manifest struct { Module string `json:"module"` Version string `json:"version"` + Moved bool `json:"moved"` + Items []item `json:"items"` Capabilities []string `json:"capabilities"` Requires []string `json:"requires"` Claims []claim `json:"claims"` @@ -23,7 +25,7 @@ type manifest struct { Shell []shellCode `json:"shell"` Resources []map[string]any `json:"resources"` // Lines for other modules' seats (novox/hq ADR 0212): the window manager's, here. - Contributions []contribution `json:"contributions"` + Contributions []any `json:"contributions"` Build struct { Artifacts []map[string]any `json:"artifacts"` } `json:"build"` @@ -46,6 +48,53 @@ type shellCode struct { Code string `json:"code"` } +// An item of a collection (novox/hq ADR 0349 clause 13): its text is the template file beside the manifest. +type item struct { + Collection string `json:"collection"` + Name string `json:"name"` + Attributes map[string]string `json:"attributes"` + Template string `json:"template"` +} + +// shell is the code the module adds to a shell's or the session's start, read from the templates of its items of +// the collections shell::, in the order they are declared. +func (m manifest) shell(t *testing.T) []shellCode { + t.Helper() + var out []shellCode + for _, it := range m.Items { + parts := strings.Split(it.Collection, ":") + if len(parts) != 3 || parts[0] != "shell" { + continue + } + text, err := os.ReadFile(filepath.Join("..", "..", it.Template)) + if err != nil { + t.Fatal(err) + } + out = append(out, shellCode{For: parts[1], Slot: parts[2], Code: string(text)}) + } + return out +} + +// environmentOf is the variables the module sets, read from the templates of its items of the environment. +func (m manifest) environmentOf(t *testing.T) *environment { + t.Helper() + var env *environment + for _, it := range m.Items { + if it.Collection != "environment" || it.Attributes["kind"] != "variable" { + continue + } + text, err := os.ReadFile(filepath.Join("..", "..", it.Template)) + if err != nil { + t.Fatal(err) + } + if env == nil { + env = &environment{Variables: map[string]string{}} + } + env.Variables[it.Attributes["name"]] = strings.TrimSuffix(string(text), "\n") + } + return env +} + func readManifest(t *testing.T) manifest { t.Helper() raw, err := os.ReadFile(filepath.Join("..", "..", "module.json")) @@ -58,6 +107,7 @@ func readManifest(t *testing.T) manifest { if err := dec.Decode(&m); err != nil { t.Fatalf("module.json: %v", err) } + m.Environment = m.environmentOf(t) return m } @@ -85,26 +135,66 @@ func (m manifest) packages() (present, absent []string) { return present, absent } -// sameAsSource checks that a file resource's content is byte for byte the module's source file, so -// the readable file in the repository is what the machine gets. -func (m manifest) sameAsSource(t *testing.T, id, source string) { +// fromTemplate checks that a file resource takes its text from its template file in the module's +// repository (novox/hq ADR 0349 §2): it names the template, carries no text of its own beside it, and the +// template is there. The readable file in the repository is the one place the machine's text is written. +func (m manifest) fromTemplate(t *testing.T, id, template string) { t.Helper() - want, err := os.ReadFile(filepath.Join("..", "..", source)) - if err != nil { - t.Fatal(err) - } r := m.resource(t, id) if r["type"] != "file" { t.Fatalf("%s is a %v, not a file", id, r["type"]) } - if got, _ := r["content"].(string); got != string(want) { - t.Fatalf("resource %s's content is not %s: edit the source and copy it into module.json", id, source) + if r["template"] != template { + t.Fatalf("resource %s does not take its text from %s: template %v", id, template, r["template"]) + } + if _, inline := r["content"]; inline { + t.Fatalf("resource %s keeps its text inline beside its template: a file's text is in one place", id) + } + if _, err := os.Stat(filepath.Join("..", "..", template)); err != nil { + t.Fatalf("resource %s names %s: %v", id, template, err) } if r["owner"] != "${machine:account}" && !strings.HasPrefix(r["path"].(string), "/etc/") { t.Fatalf("%s under the home is the account's", id) } } +// everyFileIsATemplate checks that no file resource carries its text inline, and that the build packs +// files/ into the one archive the controller reads the templates back from (novox/hq ADR 0349 §2). +func (m manifest) everyFileIsATemplate(t *testing.T) { + t.Helper() + for _, r := range m.Resources { + if _, inline := r["content"]; inline { + t.Errorf("resource %v keeps its text inline: keep it in files/ and name it as the template", r["id"]) + } + } + packs := 0 + for _, a := range m.Build.Artifacts { + if a["kind"] == "archive" && a["from"] == "files" { + packs++ + if a["name"] != "files" { + t.Errorf("the templates archive is named %v, not files", a["name"]) + } + } + } + if packs != 1 { + t.Errorf("the build packs files/ into %d archives, not one", packs) + } +} + +// text is the text a file resource places on the machine: its template file's. +func (m manifest) text(t *testing.T, id string) string { + t.Helper() + name, _ := m.resource(t, id)["template"].(string) + if name == "" { + t.Fatalf("resource %s names no template", id) + } + raw, err := os.ReadFile(filepath.Join("..", "..", name)) + if err != nil { + t.Fatal(err) + } + return string(raw) +} + // checkTheToolsAgree checks that the manifest lists the module's own tools exactly, that the bundle // serves each seat verb the claims promise as ., and that the Go bundle is declared. func checkTheToolsAgree(t *testing.T, m manifest) { @@ -176,31 +266,42 @@ func checkNoSecretsOrInstallationNames(t *testing.T) { } } -type contribution struct { - Seat string `json:"seat"` - Kind string `json:"kind"` - Content string `json:"content"` -} +// displaySession is the collection the module's lines for the window manager are an item of. +const displaySession = "contribution:node-display-session:config" -// i3Lines is what the module contributes to the window manager. +// i3Lines is what the module contributes to the window manager: the text of its item of the collection +// contribution:node-display-session:config (design 57 L3, issue 848). func (m manifest) i3Lines() string { var out string - for _, c := range m.Contributions { - if c.Seat == "node-display-session" && c.Kind == "config" { - out += c.Content + for _, it := range m.Items { + if it.Collection == displaySession { + text, err := os.ReadFile(filepath.Join("..", "..", it.Template)) + if err != nil { + return "" + } + out += string(text) } } return out } -// i3LinesAreSource checks the window-manager contribution is the source file it is written from. +// i3LinesAreSource checks the window-manager lines are one item whose template is the source file, and that no +// old-form contribution to node-display-session is left. func (m manifest) i3LinesAreSource(t *testing.T, source string) { t.Helper() - want, err := os.ReadFile(filepath.Join("..", "..", source)) - if err != nil { - t.Fatal(err) + if len(m.Contributions) != 0 { + t.Fatalf("the module still holds contributions: the lines are an item of %s", displaySession) } - if m.i3Lines() != string(want) { - t.Fatalf("the contribution to node-display-session is not %s: edit the source and copy it into module.json", source) + var templates []string + for _, it := range m.Items { + if it.Collection == displaySession { + templates = append(templates, it.Template) + } + } + if len(templates) != 1 || templates[0] != source { + t.Fatalf("the items of %s are %v, want the one template %s", displaySession, templates, source) + } + if m.i3Lines() == "" { + t.Fatalf("the template %s holds no line", source) } } diff --git a/modules/feh/cmd/feh-tools/manifest_test.go b/modules/feh/cmd/feh-tools/manifest_test.go index b02c1490..0dc0de86 100644 --- a/modules/feh/cmd/feh-tools/manifest_test.go +++ b/modules/feh/cmd/feh-tools/manifest_test.go @@ -48,16 +48,16 @@ func TestTheWallpaperIsTheModulesOwnArchive(t *testing.T) { func TestFehbgIsOwnedAndTheSessionStartRunsItOnce(t *testing.T) { m := readManifest(t) - m.sameAsSource(t, "fehbg", "files/fehbg") + m.fromTemplate(t, "fehbg", "files/fehbg") f := m.resource(t, "fehbg") if f["path"] != "${machine:account-home}/.fehbg" || f["mode"] != "0755" { t.Fatalf("%v", f) } - if c := f["content"].(string); !strings.Contains(c, "$HOME/.local/share/feh/wallpapers/default.jpg") || strings.Contains(c, ".hal") { + if c := m.text(t, "fehbg"); !strings.Contains(c, "$HOME/.local/share/feh/wallpapers/default.jpg") || strings.Contains(c, ".hal") { t.Fatalf("%s", c) } - if len(m.Shell) != 1 || m.Shell[0].For != "xinitrc" || m.Shell[0].Slot != "normal" || strings.Count(m.Shell[0].Code, `"$HOME/.fehbg"`) != 1 { - t.Fatalf("%+v", m.Shell) + if len(m.shell(t)) != 1 || m.shell(t)[0].For != "xinitrc" || m.shell(t)[0].Slot != "normal" || strings.Count(m.shell(t)[0].Code, `"$HOME/.fehbg"`) != 1 { + t.Fatalf("%+v", m.shell(t)) } } @@ -77,3 +77,8 @@ func TestTheToolsAgreeWithTheManifest(t *testing.T) { checkTheToolsAgree(t, m) checkNoSecretsOrInstallationNames(t) } + +// No file resource carries its text inline (novox/hq ADR 0349 §2). +func TestEveryFileTakesItsTextFromATemplate(t *testing.T) { + readManifest(t).everyFileIsATemplate(t) +} diff --git a/modules/feh/files/xinitrc-normal b/modules/feh/files/xinitrc-normal new file mode 100644 index 00000000..8e91e82e --- /dev/null +++ b/modules/feh/files/xinitrc-normal @@ -0,0 +1,2 @@ +# The wallpaper (module feh, novox/hq ADR 0208): the declared one, set once per session. +"$HOME/.fehbg" diff --git a/modules/feh/module.json b/modules/feh/module.json index 70a20714..eb2c69bd 100644 --- a/modules/feh/module.json +++ b/modules/feh/module.json @@ -1,6 +1,7 @@ { "module": "feh", "version": "1", + "moved": true, "capabilities": [ "package-manager" ], @@ -11,11 +12,16 @@ "feh_set", "feh_current" ], - "shell": [ + "items": [ { - "for": "xinitrc", - "slot": "normal", - "code": "# The wallpaper (module feh, novox/hq ADR 0208): the declared one, set once per session.\n\"$HOME/.fehbg\"\n" + "collection": "shell:xinitrc:normal", + "name": "xinitrc-normal", + "template": "files/xinitrc-normal" + }, + { + "collection": "contribution:node-display-session:config", + "name": "display-session", + "template": "files/i3/50-feh.conf" } ], "resources": [ @@ -37,7 +43,7 @@ "path": "${machine:account-home}/.fehbg", "owner": "${machine:account}", "mode": "0755", - "content": "#!/bin/sh\n# The wallpaper (module feh, novox/hq ADR 0208). Owned by the mesh: replaced at every push. The\n# session's start runs it, and so may anything that wants the declared wallpaper back. The image is\n# the module's own, in ~/.local/share/feh/wallpapers. feh_set changes the wallpaper for a session\n# without touching this file.\nfeh --no-fehbg --bg-fill \"$HOME/.local/share/feh/wallpapers/default.jpg\"\n" + "template": "files/fehbg" } ], "build": { @@ -57,14 +63,12 @@ "loads": [ "feh-tools" ] + }, + { + "name": "files", + "kind": "archive", + "from": "files" } ] - }, - "contributions": [ - { - "seat": "node-display-session", - "kind": "config", - "content": "# The wallpaper's key (module feh, novox/hq ADR 0208). Owned by the mesh: replaced at every push.\n# It puts the declared wallpaper back, after a monitor change or a wallpaper set for the session.\nbindsym $mod+Shift+b exec --no-startup-id ~/.fehbg\n" - } - ] + } } diff --git a/modules/fonts/cmd/fonts-tools/fonts_test.go b/modules/fonts/cmd/fonts-tools/fonts_test.go index 8fe9b19f..df4344c7 100644 --- a/modules/fonts/cmd/fonts-tools/fonts_test.go +++ b/modules/fonts/cmd/fonts-tools/fonts_test.go @@ -56,7 +56,7 @@ type fcDoc struct { func TestTheFontconfigFileMapsEachGenericToTheDecidedFaceStronglyAndFallsBackLast(t *testing.T) { m := readManifest(t) - content, _ := m.resource("defaults")["content"].(string) + content := templateText(t, m.resource("defaults")) if !strings.HasPrefix(content, " + + + + The mesh: the faces monospace, sans-serif, serif and emoji mean + + + Hack Nerd Fontmonospace + MesloLGS NFmonospace + Iosevka Nerd Fontmonospace + JetBrains Mono Nerd FontJetBrainsMono Nerd Font + + + monospaceJetBrainsMono Nerd Font + sans-serifInterNoto Sans + system-uiInter + serifNoto Serif + emojiNoto Color Emoji + + + + Symbols Nerd Font + Noto Color Emoji + + diff --git a/modules/fonts/module.json b/modules/fonts/module.json index e6684403..b2958da8 100644 --- a/modules/fonts/module.json +++ b/modules/fonts/module.json @@ -44,7 +44,7 @@ "path": "${machine:account-home}/.config/fontconfig/conf.d/50-mesh-fonts.conf", "owner": "${machine:account}", "mode": "0644", - "content": "\n\n\n\n The mesh: the faces monospace, sans-serif, serif and emoji mean\n\n \n Hack Nerd Fontmonospace\n MesloLGS NFmonospace\n Iosevka Nerd Fontmonospace\n JetBrains Mono Nerd FontJetBrainsMono Nerd Font\n\n \n monospaceJetBrainsMono Nerd Font\n sans-serifInterNoto Sans\n system-uiInter\n serifNoto Serif\n emojiNoto Color Emoji\n\n \n \n Symbols Nerd Font\n Noto Color Emoji\n \n\n" + "template": "files/50-mesh-fonts.conf" } ], "build": { @@ -59,6 +59,11 @@ "loads": [ "fonts-tools" ] + }, + { + "name": "files", + "kind": "archive", + "from": "files" } ] } diff --git a/modules/gitea/README.md b/modules/gitea/README.md index dca4ba53..d1a3397b 100644 --- a/modules/gitea/README.md +++ b/modules/gitea/README.md @@ -102,6 +102,42 @@ changed under the commit refused; a commit over its size limits refused; protect forge matches them; a code search stopped at its deadline; every malformed ask refused; and each read summarised. +## Each agent's read credential (novox/hq ADR 0365 §1, §3–§5, issue 737) + +An agent clones and fetches the repositories it may read as a forge account of its own, `agent-`. The +module offers the provision `gitea-read`, whose credential its software issues (ADR 0305) one per holder (issue +749): the Go bundle `cmd/agent-token` serves `gitea_agent_token`, which the vault alone asks (ADR 0349 clause 10; +any other caller is refused and nothing is minted). For each holder the vault names that is the claude-code module +on a machine, it + +- makes the account `agent-` with the administrator's login, when there is none: no administrator, a random + password sent once and kept nowhere, and **in no team** — it reads public repositories until the operator adds it + to a team that reads more (§3). An account found made an administrator, inactive or barred from logging in is + made a plain account again; +- deletes the account's token by the name `mesh-agent-read` (by its id, every one of them) and mints one whose one + scope is `read:repository`: whatever team it is put in, the forge refuses it a push, a comment and any token or + administration of its own (§5). Every ask mints anew; that is its rotation; +- answers git's credential-store lines for it — the forge's public origin, read from `origins` (the route's name), + and on the forge's own machine its address there too — sealed to the key the vault made for the call. A holder of + another module is not answered and said under `skipped`. + +The vault keeps each as `provision//gitea-read//claude-code` and hands it to that holder alone; +the claude-code module places it in the agent's home (ADR 0365 §4). The token is never logged, answered in the clear +or put in an error. `go test ./cmd/agent-token` runs it against a fake forge. + +## The agents' read teams (novox/hq ADR 0365 §3, issue 758) + +An agent's account reads more than public repositories only through a team, per repository, and only to read. Which +teams exist and who is in them is the operator's, and it is one trusted config value, `agent-read-teams`: +`{"": {"repos": ["/", ...], "members": ["", ...]}}`, a member `` being the account +`agent-`. The module renders it into `agent-teams.json` and runs `cmd/agent-teams` when it starts and +whenever that file changes. For each team it makes `agent-read-` in the organisation of the first repository's +owner, with permission read and no other right, its repositories exactly the listed ones and its members exactly the +listed accounts (an account that does not exist yet is skipped and said). A team named `agent-read-*` that has left +the value is emptied, never deleted. A team without that prefix is never touched. There are no verbs: a change is a +change of the value. It says each change in one line on stdout, and never a secret. `go test ./cmd/agent-teams` +runs it against a fake forge. + ## The package registry's retention (novox/hq ADR 0251 §4, to-be 51) Every publish adds a version to the registry, and until this nothing removed one. The Go bundle diff --git a/modules/gitea/cmd/agent-teams/main.go b/modules/gitea/cmd/agent-teams/main.go new file mode 100644 index 00000000..eb92207c --- /dev/null +++ b/modules/gitea/cmd/agent-teams/main.go @@ -0,0 +1,42 @@ +// agent-teams: the agents' read teams on the forge (novox/hq ADR 0365 clause 3, issue 758). A program the node's +// runtime runs once when the module starts and again whenever the module's config value `agent-read-teams` changes: +// it reads the file the mesh renders from that value, and makes the forge's teams named agent-read-* match it. Each +// team reads only; its repositories and its members (agent accounts, agent-) are exactly the value's. It +// touches nothing outside the value: no team without the prefix, no repository or account of another kind. +// +// It says each change on stdout in one line, and never a secret. +package main + +import ( + "context" + "fmt" + "os" + "time" + + "gitea/internal/forgehttp" +) + +func main() { + forge, missing := forgehttp.FromEnv() + file := os.Getenv("MESH_AGENT_TEAMS_FILE") + if file == "" { + missing = append(missing, "MESH_AGENT_TEAMS_FILE") + } + if len(missing) > 0 { + fmt.Fprintf(os.Stderr, "agent-teams: %v not set by the manifest; nothing was changed\n", missing) + os.Exit(1) + } + value, err := ReadValue(file) + if err != nil { + fmt.Fprintln(os.Stderr, "agent-teams:", err) + os.Exit(1) + } + s := &Syncer{Forge: forge, Log: func(format string, a ...any) { fmt.Printf(format+"\n", a...) }, + WaitFor: 3 * time.Minute, Poll: 5 * time.Second} + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute) + defer cancel() + if err := s.Sync(ctx, value); err != nil { + fmt.Fprintln(os.Stderr, "agent-teams:", err) + os.Exit(1) + } +} diff --git a/modules/gitea/cmd/agent-teams/teams.go b/modules/gitea/cmd/agent-teams/teams.go new file mode 100644 index 00000000..5f2a0b6e --- /dev/null +++ b/modules/gitea/cmd/agent-teams/teams.go @@ -0,0 +1,339 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net/http" + "net/url" + "os" + "regexp" + "sort" + "strings" + "time" + + "gitea/internal/forgehttp" +) + +const ( + // ConfigKey is the module's config value this program applies. + ConfigKey = "agent-read-teams" + // TeamPrefix names every team this program owns; a team without it is never touched (ADR 0365 §3). + TeamPrefix = "agent-read-" + // AccountPrefix names an agent's account for its node (cmd/agent-token AccountPrefix, ADR 0365 §1). + AccountPrefix = "agent-" + page = 50 +) + +var ( + teamName = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9_.-]{0,40}$`) + nodeName = regexp.MustCompile(`^[a-z0-9][a-z0-9-]{0,31}$`) + repoName = regexp.MustCompile(`^([A-Za-z0-9][A-Za-z0-9_.-]*)/([A-Za-z0-9_.-]+)$`) +) + +// readUnits are the units a read team has, each at read: the forge's rights of a team that reads and nothing more. +var readUnits = map[string]string{ + "repo.code": "read", "repo.issues": "read", "repo.pulls": "read", "repo.releases": "read", "repo.wiki": "read", +} + +// Team is one read team of the value: its repositories (owner/repo, read only) and its members (nodes whose agent +// account, agent-, is in the team). +type Team struct { + Repos []string `json:"repos"` + Members []string `json:"members"` +} + +// Value is the config value: each team by the name it is given without the prefix. +type Value map[string]Team + +// ReadValue reads the file the mesh renders from the config value. A file without the key is the empty value, +// which empties every read team. +func ReadValue(path string) (Value, error) { + raw, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("the teams file cannot be read (%s): %w", path, err) + } + var file map[string]json.RawMessage + if err := json.Unmarshal(raw, &file); err != nil { + return nil, fmt.Errorf("the teams file %s is not JSON: %w", path, err) + } + var v Value + if r, ok := file[ConfigKey]; ok { + dec := json.NewDecoder(strings.NewReader(string(r))) + dec.DisallowUnknownFields() + if err := dec.Decode(&v); err != nil { + return nil, fmt.Errorf("%s in %s is not {team: {repos, members}}: %w", ConfigKey, path, err) + } + } + return v, nil +} + +// Syncer makes the forge's agent-read-* teams the value's. +type Syncer struct { + Forge forgehttp.Client + Log func(format string, a ...any) + WaitFor time.Duration // how long to wait for the forge to answer; none when zero + Poll time.Duration +} + +type forgeTeam struct { + ID int64 `json:"id"` + Name string `json:"name"` + Permission string `json:"permission"` +} + +type forgeRepo struct { + FullName string `json:"full_name"` +} + +type forgeMember struct { + Login string `json:"login"` +} + +func (s *Syncer) say(format string, a ...any) { + if s.Log != nil { + s.Log(format, a...) + } +} + +// Sync applies the value. A team that cannot be applied is said and the others still are; the error names them. +func (s *Syncer) Sync(ctx context.Context, v Value) error { + if err := s.waitForForge(ctx); err != nil { + return err + } + orgs, err := s.orgs(ctx) + if err != nil { + return err + } + wanted := map[string]bool{} + var failed []string + names := make([]string, 0, len(v)) + for n := range v { + names = append(names, n) + } + sort.Strings(names) + for _, n := range names { + wanted[TeamPrefix+n] = true + if err := s.apply(ctx, n, v[n]); err != nil { + s.say("not applied: team %s%s: %v", TeamPrefix, n, err) + failed = append(failed, TeamPrefix+n) + } + } + for _, org := range orgs { + teams, err := s.teams(ctx, org) + if err != nil { + return err + } + for _, t := range teams { + if !strings.HasPrefix(t.Name, TeamPrefix) || wanted[t.Name] { + continue + } + if err := s.sync(ctx, org, t, nil, nil); err != nil { + s.say("not applied: team %s in %s: %v", t.Name, org, err) + failed = append(failed, t.Name) + } + } + } + if len(failed) > 0 { + return fmt.Errorf("not applied: %s", strings.Join(failed, ", ")) + } + return nil +} + +func (s *Syncer) waitForForge(ctx context.Context) error { + deadline := time.Now().Add(s.WaitFor) + for { + err := s.Forge.GetJSON(ctx, "/api/v1/version", nil) + if err == nil { + return nil + } + var se *forgehttp.StatusError + if errors.As(err, &se) || time.Now().After(deadline) || ctx.Err() != nil { + return fmt.Errorf("the forge is not reachable: %w", err) + } + time.Sleep(s.Poll) + } +} + +// apply makes one team of the value. +func (s *Syncer) apply(ctx context.Context, name string, t Team) error { + if !teamName.MatchString(name) { + return fmt.Errorf("%q is not a team name", name) + } + if len(t.Repos) == 0 { + return errors.New("it lists no repository, so the organisation to make it in is not known") + } + org := "" + repos := map[string]bool{} + for _, r := range t.Repos { + m := repoName.FindStringSubmatch(r) + if m == nil { + return fmt.Errorf("%q is not owner/repo", r) + } + if org == "" { + org = m[1] + } + if m[1] != org { + return fmt.Errorf("%q is not in %s: a team's repositories are of one owner, the first's", r, org) + } + repos[r] = true + } + members := map[string]bool{} + for _, n := range t.Members { + if !nodeName.MatchString(n) { + return fmt.Errorf("%q is not a node's name", n) + } + members[AccountPrefix+n] = true + } + full := TeamPrefix + name + teams, err := s.teams(ctx, org) + if err != nil { + return err + } + var team *forgeTeam + for i := range teams { + if teams[i].Name == full { + team = &teams[i] + } + } + if team == nil { + var made forgeTeam + err := s.Forge.SendJSON(ctx, http.MethodPost, "/api/v1/orgs/"+org+"/teams", map[string]any{ + "name": full, "description": "Read-only team of agents' forge accounts (ADR 0365 §3); set by the gitea config value " + ConfigKey, + "permission": "read", "includes_all_repositories": false, "can_create_org_repo": false, "units_map": readUnits, + }, &made) + if err != nil { + return fmt.Errorf("the team could not be made: %w", err) + } + s.say("made the team %s in %s, reading only", full, org) + team = &made + } else if team.Permission != "read" { + if err := s.Forge.SendJSON(ctx, http.MethodPatch, fmt.Sprintf("/api/v1/teams/%d", team.ID), map[string]any{ + "name": full, "permission": "read", "includes_all_repositories": false, "can_create_org_repo": false, "units_map": readUnits, + }, nil); err != nil { + return fmt.Errorf("the team could not be made read-only: %w", err) + } + s.say("made the team %s in %s read-only (it was %s)", full, org, team.Permission) + } + return s.sync(ctx, org, *team, repos, members) +} + +// sync makes the team's repositories and members exactly the sets (nil is none). +func (s *Syncer) sync(ctx context.Context, org string, t forgeTeam, repos, members map[string]bool) error { + var errs []string + var haveRepos []forgeRepo + if err := s.list(ctx, fmt.Sprintf("/api/v1/teams/%d/repos", t.ID), &haveRepos); err != nil { + return err + } + held := map[string]bool{} + for _, r := range haveRepos { + held[r.FullName] = true + if !repos[r.FullName] { + if err := s.Forge.SendJSON(ctx, http.MethodDelete, fmt.Sprintf("/api/v1/teams/%d/repos/%s", t.ID, r.FullName), nil, nil); err != nil { + errs = append(errs, err.Error()) + continue + } + s.say("took %s from the team %s", r.FullName, t.Name) + } + } + for _, r := range sortedKeys(repos) { + if held[r] { + continue + } + if err := s.Forge.SendJSON(ctx, http.MethodPut, fmt.Sprintf("/api/v1/teams/%d/repos/%s", t.ID, r), nil, nil); err != nil { + errs = append(errs, fmt.Sprintf("%s could not be given to the team: %v", r, err)) + continue + } + s.say("gave %s to the team %s, to read", r, t.Name) + } + var haveMembers []forgeMember + if err := s.list(ctx, fmt.Sprintf("/api/v1/teams/%d/members", t.ID), &haveMembers); err != nil { + return err + } + in := map[string]bool{} + for _, m := range haveMembers { + in[m.Login] = true + if !members[m.Login] { + if err := s.Forge.SendJSON(ctx, http.MethodDelete, fmt.Sprintf("/api/v1/teams/%d/members/%s", t.ID, url.PathEscape(m.Login)), nil, nil); err != nil { + errs = append(errs, err.Error()) + continue + } + s.say("took %s out of the team %s", m.Login, t.Name) + } + } + for _, m := range sortedKeys(members) { + if in[m] { + continue + } + if err := s.Forge.GetJSON(ctx, "/api/v1/users/"+url.PathEscape(m), nil); err != nil { + if errors.Is(err, forgehttp.ErrNotFound) { + s.say("skipped %s for the team %s: the account does not exist yet", m, t.Name) + continue + } + errs = append(errs, err.Error()) + continue + } + if err := s.Forge.SendJSON(ctx, http.MethodPut, fmt.Sprintf("/api/v1/teams/%d/members/%s", t.ID, url.PathEscape(m)), nil, nil); err != nil { + errs = append(errs, fmt.Sprintf("%s could not be put in the team: %v", m, err)) + continue + } + s.say("put %s in the team %s", m, t.Name) + } + if len(errs) > 0 { + return errors.New(strings.Join(errs, "; ")) + } + return nil +} + +func sortedKeys(m map[string]bool) []string { + out := make([]string, 0, len(m)) + for k := range m { + out = append(out, k) + } + sort.Strings(out) + return out +} + +// list reads every page of a list answer into into (a pointer to a slice). +func (s *Syncer) list(ctx context.Context, path string, into any) error { + var all []json.RawMessage + for p := 1; ; p++ { + var got []json.RawMessage + if err := s.Forge.GetJSON(ctx, fmt.Sprintf("%s?limit=%d&page=%d", path, page, p), &got); err != nil { + return err + } + all = append(all, got...) + if len(got) < page { + break + } + } + raw, _ := json.Marshal(all) + return json.Unmarshal(raw, into) +} + +func (s *Syncer) teams(ctx context.Context, org string) ([]forgeTeam, error) { + var out []forgeTeam + if err := s.list(ctx, "/api/v1/orgs/"+org+"/teams", &out); err != nil { + if errors.Is(err, forgehttp.ErrNotFound) { + return nil, fmt.Errorf("the organisation %s does not exist on the forge", org) + } + return nil, err + } + return out, nil +} + +// orgs are the organisations on the forge, where a read team no longer in the value is found. +func (s *Syncer) orgs(ctx context.Context) ([]string, error) { + var out []struct { + Name string `json:"name"` + } + if err := s.list(ctx, "/api/v1/admin/orgs", &out); err != nil { + return nil, fmt.Errorf("the forge's organisations could not be read: %w", err) + } + names := make([]string, len(out)) + for i, o := range out { + names[i] = o.Name + } + return names, nil +} diff --git a/modules/gitea/cmd/agent-teams/teams_test.go b/modules/gitea/cmd/agent-teams/teams_test.go new file mode 100644 index 00000000..0dfe617c --- /dev/null +++ b/modules/gitea/cmd/agent-teams/teams_test.go @@ -0,0 +1,333 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "regexp" + "sort" + "strconv" + "strings" + "sync" + "testing" + + "gitea/internal/forgehttp" +) + +type fTeam struct { + ID int64 + Org string + Name string + Perm string + Units map[string]string + Repos map[string]bool + Members map[string]bool +} + +type fakeForge struct { + mu sync.Mutex + orgs []string + users map[string]bool + teams []*fTeam + next int64 +} + +func newFake(orgs []string, users ...string) *fakeForge { + f := &fakeForge{orgs: orgs, users: map[string]bool{}, next: 1} + for _, u := range users { + f.users[u] = true + } + return f +} + +func (f *fakeForge) add(org, name, perm string, repos, members []string) *fTeam { + t := &fTeam{ID: f.next, Org: org, Name: name, Perm: perm, Repos: map[string]bool{}, Members: map[string]bool{}} + f.next++ + for _, r := range repos { + t.Repos[r] = true + } + for _, m := range members { + t.Members[m] = true + } + f.teams = append(f.teams, t) + return t +} + +func (f *fakeForge) byName(org, name string) *fTeam { + for _, t := range f.teams { + if t.Org == org && t.Name == name { + return t + } + } + return nil +} + +var ( + orgTeams = regexp.MustCompile(`^/api/v1/orgs/([^/]+)/teams$`) + teamPath = regexp.MustCompile(`^/api/v1/teams/(\d+)$`) + teamRepos = regexp.MustCompile(`^/api/v1/teams/(\d+)/repos(?:/(.+))?$`) + teamMems = regexp.MustCompile(`^/api/v1/teams/(\d+)/members(?:/(.+))?$`) +) + +func (f *fakeForge) find(id string) *fTeam { + n, _ := strconv.ParseInt(id, 10, 64) + for _, t := range f.teams { + if t.ID == n { + return t + } + } + return nil +} + +func keys(m map[string]bool) []string { + out := []string{} + for k := range m { + out = append(out, k) + } + sort.Strings(out) + return out +} + +func (f *fakeForge) ServeHTTP(w http.ResponseWriter, r *http.Request) { + f.mu.Lock() + defer f.mu.Unlock() + if u, p, ok := r.BasicAuth(); !ok || u != "mesh-admin" || p != "pw" { + http.Error(w, "no", http.StatusUnauthorized) + return + } + reply := func(status int, v any) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(status) + if v != nil { + _ = json.NewEncoder(w).Encode(v) + } + } + paged := r.URL.Query().Get("page") != "" && r.URL.Query().Get("page") != "1" + var in map[string]any + _ = json.NewDecoder(r.Body).Decode(&in) + p := r.URL.Path + switch { + case p == "/api/v1/version": + reply(200, map[string]string{"version": "1"}) + case p == "/api/v1/admin/orgs": + out := []map[string]string{} + for _, o := range f.orgs { + out = append(out, map[string]string{"name": o}) + } + if paged { + out = nil + } + reply(200, out) + case strings.HasPrefix(p, "/api/v1/users/"): + if !f.users[strings.TrimPrefix(p, "/api/v1/users/")] { + reply(404, nil) + return + } + reply(200, map[string]string{}) + case orgTeams.MatchString(p): + org := orgTeams.FindStringSubmatch(p)[1] + known := false + for _, o := range f.orgs { + known = known || o == org + } + if !known { + reply(404, nil) + return + } + if r.Method == http.MethodPost { + units, _ := in["units_map"].(map[string]any) + t := f.add(org, in["name"].(string), in["permission"].(string), nil, nil) + t.Units = map[string]string{} + for k, v := range units { + t.Units[k] = v.(string) + } + reply(201, map[string]any{"id": t.ID, "name": t.Name, "permission": t.Perm}) + return + } + out := []map[string]any{} + for _, t := range f.teams { + if t.Org == org && !paged { + out = append(out, map[string]any{"id": t.ID, "name": t.Name, "permission": t.Perm}) + } + } + reply(200, out) + case teamPath.MatchString(p): + t := f.find(teamPath.FindStringSubmatch(p)[1]) + t.Perm = in["permission"].(string) + reply(200, nil) + case teamRepos.MatchString(p): + m := teamRepos.FindStringSubmatch(p) + t := f.find(m[1]) + switch { + case m[2] == "" && r.Method == http.MethodGet: + out := []map[string]string{} + for _, k := range keys(t.Repos) { + out = append(out, map[string]string{"full_name": k}) + } + if paged { + out = nil + } + reply(200, out) + case r.Method == http.MethodPut: + t.Repos[m[2]] = true + reply(204, nil) + case r.Method == http.MethodDelete: + delete(t.Repos, m[2]) + reply(204, nil) + } + case teamMems.MatchString(p): + m := teamMems.FindStringSubmatch(p) + t := f.find(m[1]) + switch { + case m[2] == "" && r.Method == http.MethodGet: + out := []map[string]string{} + for _, k := range keys(t.Members) { + out = append(out, map[string]string{"login": k}) + } + if paged { + out = nil + } + reply(200, out) + case r.Method == http.MethodPut: + t.Members[m[2]] = true + reply(204, nil) + case r.Method == http.MethodDelete: + delete(t.Members, m[2]) + reply(204, nil) + } + default: + reply(404, nil) + } +} + +func syncer(t *testing.T, f *fakeForge) (*Syncer, *[]string) { + t.Helper() + srv := httptest.NewServer(f) + t.Cleanup(srv.Close) + pw := filepath.Join(t.TempDir(), "pw") + if err := os.WriteFile(pw, []byte("pw\n"), 0o600); err != nil { + t.Fatal(err) + } + var said []string + return &Syncer{Forge: forgehttp.Client{URL: srv.URL, User: "mesh-admin", PasswordFile: pw}, + Log: func(format string, a ...any) { said = append(said, sprintf(format, a...)) }}, &said +} + +func sprintf(format string, a ...any) string { return fmt.Sprintf(format, a...) } + +func equal(a, b []string) bool { return strings.Join(a, ",") == strings.Join(b, ",") } + +// The first test of issue 758: a value makes a read-only team with exactly its repositories and members; a changed +// value takes a repository and a member away; a team that is not named agent-read-* is never touched. +func TestValueMakesReadTeamAndChangeRemoves(t *testing.T) { + f := newFake([]string{"novox"}, "agent-g14", "agent-ace") + other := f.add("novox", "Owners", "owner", []string{"novox/hq"}, []string{"agent-ace"}) + s, said := syncer(t, f) + ctx := context.Background() + + err := s.Sync(ctx, Value{"all": {Repos: []string{"novox/hq", "novox/mesh-catalog"}, Members: []string{"g14"}}}) + if err != nil { + t.Fatal(err) + } + team := f.byName("novox", "agent-read-all") + if team == nil || team.Perm != "read" { + t.Fatalf("team = %+v, want a read team", team) + } + for _, u := range team.Units { + if u != "read" { + t.Errorf("unit %q, want read only", u) + } + } + if !equal(keys(team.Repos), []string{"novox/hq", "novox/mesh-catalog"}) || !equal(keys(team.Members), []string{"agent-g14"}) { + t.Fatalf("repos %v members %v", keys(team.Repos), keys(team.Members)) + } + + before := len(*said) + if err := s.Sync(ctx, Value{"all": {Repos: []string{"novox/hq", "novox/mesh-catalog"}, Members: []string{"g14"}}}); err != nil { + t.Fatal(err) + } + if len(*said) != before { + t.Errorf("a second run changed something: %v", (*said)[before:]) + } + + if err := s.Sync(ctx, Value{"all": {Repos: []string{"novox/hq"}, Members: nil}}); err != nil { + t.Fatal(err) + } + if !equal(keys(team.Repos), []string{"novox/hq"}) || len(team.Members) != 0 { + t.Fatalf("after change: repos %v members %v", keys(team.Repos), keys(team.Members)) + } + + // the team leaves the value: emptied, never deleted + if err := s.Sync(ctx, Value{}); err != nil { + t.Fatal(err) + } + if f.byName("novox", "agent-read-all") == nil || len(team.Repos) != 0 || len(team.Members) != 0 { + t.Fatalf("removed team not emptied: %+v", team) + } + if other.Perm != "owner" || !equal(keys(other.Repos), []string{"novox/hq"}) || !equal(keys(other.Members), []string{"agent-ace"}) { + t.Fatalf("a team without the prefix was touched: %+v", other) + } + for _, line := range *said { + if strings.Contains(line, "pw") { + t.Errorf("a line names the password: %q", line) + } + } +} + +func TestPermissionIsCorrectedAndMissingAccountSkipped(t *testing.T) { + f := newFake([]string{"novox"}, "agent-g14") + team := f.add("novox", "agent-read-all", "write", nil, nil) + s, said := syncer(t, f) + err := s.Sync(context.Background(), Value{"all": {Repos: []string{"novox/hq"}, Members: []string{"g14", "later"}}}) + if err != nil { + t.Fatal(err) + } + if team.Perm != "read" || !equal(keys(team.Members), []string{"agent-g14"}) { + t.Fatalf("team = %+v", team) + } + if !strings.Contains(strings.Join(*said, "\n"), "agent-later") { + t.Errorf("the missing account was not said: %v", *said) + } +} + +func TestRefusesWhatIsNotAReadTeam(t *testing.T) { + f := newFake([]string{"novox", "other"}) + s, _ := syncer(t, f) + for name, v := range map[string]Value{ + "no repo": {"a": {Members: []string{"g14"}}}, + "two owners": {"a": {Repos: []string{"novox/hq", "other/x"}}}, + "bad repo": {"a": {Repos: []string{"hq"}}}, + "bad member": {"a": {Repos: []string{"novox/hq"}, Members: []string{"../x"}}}, + "unknown org": {"a": {Repos: []string{"nowhere/hq"}}}, + "bad teamname": {"a b": {Repos: []string{"novox/hq"}}}, + } { + if err := s.Sync(context.Background(), v); err == nil { + t.Errorf("%s: no error", name) + } + } + if len(f.teams) != 0 { + t.Errorf("teams made: %+v", f.teams) + } +} + +func TestReadValue(t *testing.T) { + dir := t.TempDir() + p := filepath.Join(dir, "t.json") + _ = os.WriteFile(p, []byte(`{"agent-read-teams":{"a":{"repos":["o/r"],"members":["g14"]}}}`), 0o600) + v, err := ReadValue(p) + if err != nil || len(v["a"].Repos) != 1 || v["a"].Members[0] != "g14" { + t.Fatalf("%v %v", v, err) + } + _ = os.WriteFile(p, []byte(`{}`), 0o600) + if v, err := ReadValue(p); err != nil || len(v) != 0 { + t.Fatalf("%v %v", v, err) + } + _ = os.WriteFile(p, []byte(`{"agent-read-teams":{"a":{"repos":["o/r"],"write":true}}}`), 0o600) + if _, err := ReadValue(p); err == nil { + t.Fatal("an unknown field was accepted") + } +} diff --git a/modules/gitea/cmd/agent-token/main.go b/modules/gitea/cmd/agent-token/main.go new file mode 100644 index 00000000..92f5a6d8 --- /dev/null +++ b/modules/gitea/cmd/agent-token/main.go @@ -0,0 +1,72 @@ +// agent-token: the forge's read credential for each node's agent (novox/hq ADR 0365 clauses 1, 3, 4 and 5; issue +// 737). A process the node's runtime launches and speaks MCP over stdio to, beside the module's other bundles. +// +// It serves one tool, gitea_agent_token, which the vault alone asks (ADR 0349 clause 10, issue 749): for each holder +// of the provision `gitea-read` the vault names — the claude-code module on one machine — it makes that machine's +// agent a forge account of its own, `agent-`, no administrator and in no team, mints it a token whose one +// scope is `read:repository`, and answers the git credential lines for it, each holder's sealed to the key the vault +// made for the call. Asked again, it mints again and deletes the account's token from before: that is its rotation. +// +// stdout is the protocol; it says what it says on stderr, and never a token or the admin's password. +package main + +import ( + "fmt" + "os" + + stdio "git.novox.be/novox/mesh-sdk/go" +) + +// ToolNames are the tools this bundle serves. The module's manifest lists no `tools` (see the npm-registry bundle +// for why); the README names this one, and a test holds the three together. +var ToolNames = []string{ToolName} + +func main() { + if err := stdio.Serve("", Tools(MinterFromEnv)); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } +} + +// Tools are the bundle's tools over the minter the function gives, asked per call, so a manifest that does not set +// the environment is a refusal naming what is missing, not a bundle that will not start. +func Tools(minter func() (*Minter, error)) []stdio.Tool { + return []stdio.Tool{{ + Name: ToolName, + Description: "The forge's read credential for each node's agent (novox/hq ADR 0365), answered to the vault " + + "alone: for each holder named, a forge account of the node's agent's own (agent-), no administrator " + + "and in no team, and a token of scope read:repository minted for it anew, its git credential lines " + + "sealed to seal_to. Any other caller is refused and nothing is minted.", + Input: map[string]any{ + "type": "object", + "properties": map[string]any{ + "provision": map[string]any{"type": "string", "description": "the provision, " + Provision}, + "seal_to": map[string]any{"type": "string", "description": "the key made for this call, base64"}, + "holders": map[string]any{ + "type": "array", + "description": "the provision's holders: each a consuming module on one machine", + "items": map[string]any{"type": "object", "properties": map[string]any{ + "node": map[string]any{"type": "string"}, + "module": map[string]any{"type": "string"}, + }}, + }, + }, + "required": []string{"provision", "seal_to", "holders"}, + }, + Run: func(args map[string]any) (any, error) { + caller := stdio.CallerOf(args) + if err := fromTheVault(caller); err != nil { + return nil, err + } + ask, err := readAsk(args) + if err != nil { + return nil, err + } + m, err := minter() + if err != nil { + return nil, err + } + return m.Answer(ask) + }, + }} +} diff --git a/modules/gitea/cmd/agent-token/token.go b/modules/gitea/cmd/agent-token/token.go new file mode 100644 index 00000000..33e8de9e --- /dev/null +++ b/modules/gitea/cmd/agent-token/token.go @@ -0,0 +1,348 @@ +package main + +import ( + "context" + "crypto/rand" + "encoding/base64" + "errors" + "fmt" + "net/http" + "net/url" + "os" + "regexp" + "sort" + "strings" + "sync" + "time" + + "golang.org/x/crypto/nacl/box" + + "gitea/internal/forgehttp" +) + +const ( + // ToolName is the issuing tool the vault asks (the offer's credential tool in module.json). + ToolName = "gitea_agent_token" + // Provision is what the tool issues: the forge's read credential, one per agent. + Provision = "gitea-read" + // Holder is the one consuming module answered: the agent's own module, which places the credential in the + // agent's home (ADR 0365 §4, ADR 0266). A holder of another module is not answered and keeps nothing. + Holder = "claude-code" + // TokenName is the token's name in the agent account's list, found and deleted by it at the next mint. + TokenName = "mesh-agent-read" + // AccountPrefix names an agent's account for its node (ADR 0365 §1). + AccountPrefix = "agent-" + // vaultModule is the vault's module: its account is `.mesh-vault` (ADR 0259 §8's runs-as). + vaultModule = "mesh-vault" + // deadline bounds one call, inside the vault's ask. + deadline = 45 * time.Second +) + +// TokenScopes are the only scopes an agent's token carries (ADR 0365 §4, §5): it reads repositories and nothing +// else, so the forge refuses it a push, a comment, a token of its own and any administration — whatever team the +// account is in. +var TokenScopes = []string{"read:repository"} + +var nodeName = regexp.MustCompile(`^[a-z0-9][a-z0-9-]{0,31}$`) + +// fromTheVault refuses every caller but the vault's own account (ADR 0349 clause 10): whoever else could call this +// tool would hold every agent's credential. +func fromTheVault(caller string) error { + node, module, ok := strings.Cut(caller, ".") + if ok && module == vaultModule && nodeName.MatchString(node) { + return nil + } + return fmt.Errorf("%s answers the vault alone (novox/hq ADR 0349 clause 10, ADR 0365); %q is not the vault, and "+ + "nothing was minted", ToolName, caller) +} + +// HolderAsk is one holder the vault named. +type HolderAsk struct { + Node string `json:"node"` + Module string `json:"module"` +} + +// Ask is one call: the holders, and the key to seal each answer to. +type Ask struct { + SealTo [32]byte + Holders []HolderAsk +} + +// readAsk reads the call's arguments, refusing anything that is not the vault's per-holder ask. +func readAsk(args map[string]any) (Ask, error) { + if p, _ := args["provision"].(string); p != Provision { + return Ask{}, fmt.Errorf("%s issues %s alone, not %q; nothing was minted", ToolName, Provision, p) + } + key, _ := args["seal_to"].(string) + raw, err := base64.StdEncoding.DecodeString(strings.TrimSpace(key)) + if err != nil || len(raw) != 32 { + return Ask{}, errors.New("seal_to is not the key of a call; nothing was minted") + } + var out Ask + copy(out.SealTo[:], raw) + list, ok := args["holders"].([]any) + if !ok || len(list) == 0 { + return Ask{}, fmt.Errorf("%s answers a value for each holder, and none was named; nothing was minted", ToolName) + } + seen := map[HolderAsk]bool{} + for _, item := range list { + m, _ := item.(map[string]any) + node, _ := m["node"].(string) + module, _ := m["module"].(string) + h := HolderAsk{Node: strings.TrimSpace(node), Module: strings.TrimSpace(module)} + if h.Node == "" || h.Module == "" || seen[h] { + return Ask{}, errors.New("each holder is a node and a module, named once; nothing was minted") + } + seen[h] = true + out.Holders = append(out.Holders, h) + } + return out, nil +} + +// Minter mints the agents' tokens with the administrator's login, which sets the forge up and nothing else (ADR +// 0365 §2). +type Minter struct { + Forge forgehttp.Client + // Node is the forge's own machine: a holder there also reaches the forge at Forge.URL. + Node string + // OriginsFile holds the forge's public origins, one per line (https://): the origins every + // holder's credential is for. + OriginsFile string + // Password makes the password an agent's account is created with: random, sent once and kept nowhere. + Password func() string + Log func(format string, a ...any) +} + +// MinterFromEnv is the minter as the manifest hands it to this bundle. It builds the administrator's client itself: +// the module's words reach all its bundles, and a word naming another bundle's token must not make this one act as +// another account. +func MinterFromEnv() (*Minter, error) { + m := &Minter{ + Forge: forgehttp.Client{URL: strings.TrimRight(os.Getenv("MESH_GITEA_URL"), "/"), + User: os.Getenv("MESH_GITEA_ADMIN_USER"), PasswordFile: os.Getenv("MESH_GITEA_ADMIN_PASSWORD_FILE"), + Timeout: 20 * time.Second}, + Node: os.Getenv("MESH_NODE"), + OriginsFile: os.Getenv("MESH_GITEA_ORIGINS_FILE"), + } + var missing []string + for _, kv := range [][2]string{{"MESH_GITEA_URL", m.Forge.URL}, {"MESH_GITEA_ADMIN_USER", m.Forge.User}, + {"MESH_GITEA_ADMIN_PASSWORD_FILE", m.Forge.PasswordFile}, {"MESH_GITEA_ORIGINS_FILE", m.OriginsFile}} { + if kv[1] == "" { + missing = append(missing, kv[0]) + } + } + if len(missing) > 0 { + return nil, fmt.Errorf("the forge cannot be reached: %s not set by the manifest; nothing was minted", strings.Join(missing, ", ")) + } + return m, nil +} + +// Sealed is one holder's credential as the vault takes it. +type Sealed struct { + Node string `json:"node"` + Module string `json:"module"` + Sealed string `json:"sealed"` +} + +// Answer is what the tool answers: a sealed credential for each holder answered, and why each other was not. Never +// a token. +type Answer struct { + Holders []Sealed `json:"holders"` + Skipped map[string]string `json:"skipped,omitempty"` +} + +// one call at a time: a mint deletes the account's token from before, and two at once would delete each other's. +var minting sync.Mutex + +// Answer mints each holder's token and seals its credential lines. A holder that is not an agent's module is not +// answered; a failure for one holder fails the call, so the vault keeps nothing rather than a part. +func (m *Minter) Answer(ask Ask) (Answer, error) { + minting.Lock() + defer minting.Unlock() + ctx, cancel := context.WithTimeout(context.Background(), deadline) + defer cancel() + origins, err := m.origins() + if err != nil { + return Answer{}, err + } + out := Answer{Holders: []Sealed{}} + skip := func(h HolderAsk, why string) { + if out.Skipped == nil { + out.Skipped = map[string]string{} + } + out.Skipped[h.Module+" on "+h.Node] = why + } + for _, h := range ask.Holders { + if h.Module != Holder { + skip(h, fmt.Sprintf("only %s places the forge's read credential, in its agent's home (ADR 0365 §4)", Holder)) + continue + } + if !nodeName.MatchString(h.Node) { + skip(h, "not a machine's name an account can be named for") + continue + } + reach := origins + if h.Node == m.Node { + reach = append(append([]string{}, origins...), m.Forge.URL) + } + if len(reach) == 0 { + skip(h, "the forge names no origin it is reached at from that machine") + continue + } + account := AccountPrefix + h.Node + if err := m.ensureAccount(ctx, account); err != nil { + return Answer{}, err + } + token, err := m.mint(ctx, account) + if err != nil { + return Answer{}, err + } + lines, err := credentialLines(account, token, reach) + token = "" + if err != nil { + return Answer{}, err + } + sealed, err := box.SealAnonymous(nil, []byte(lines), &ask.SealTo, rand.Reader) + lines = "" + if err != nil { + return Answer{}, fmt.Errorf("the credential for %s could not be sealed: %w", account, err) + } + out.Holders = append(out.Holders, Sealed{Node: h.Node, Module: h.Module, Sealed: base64.StdEncoding.EncodeToString(sealed)}) + m.say("minted a read token for %s (%s on %s); its token from before is deleted", account, h.Module, h.Node) + } + return out, nil +} + +// origins are the forge's public origins, as its route names them. +func (m *Minter) origins() ([]string, error) { + raw, err := os.ReadFile(m.OriginsFile) + if err != nil { + return nil, fmt.Errorf("the forge's origins cannot be read (%s); nothing was minted: %w", m.OriginsFile, err) + } + var out []string + for _, line := range strings.Split(string(raw), "\n") { + line = strings.TrimRight(strings.TrimSpace(line), "/") + if line == "" { + continue + } + u, err := url.Parse(line) + if err != nil || (u.Scheme != "https" && u.Scheme != "http") || u.Host == "" || u.User != nil || u.Path != "" { + return nil, fmt.Errorf("%s names %q, which is not an origin (scheme://host); nothing was minted", m.OriginsFile, line) + } + out = append(out, line) + } + return out, nil +} + +// credentialLines are the lines of git's credential store for one account at each origin: what the agent's module +// places, and git reads by protocol and host. +func credentialLines(account, token string, origins []string) (string, error) { + seen := map[string]bool{} + var lines []string + for _, o := range origins { + u, err := url.Parse(strings.TrimRight(o, "/")) + if err != nil || u.Host == "" { + return "", fmt.Errorf("%q is not an origin", o) + } + u.Path, u.RawQuery, u.Fragment = "", "", "" + u.User = url.UserPassword(account, token) + if line := u.String(); !seen[line] { + seen[line] = true + lines = append(lines, line) + } + } + sort.Strings(lines) + return strings.Join(lines, "\n") + "\n", nil +} + +// gitea's account as the admin reads it. +type forgeUser struct { + IsAdmin bool `json:"is_admin"` + Active bool `json:"active"` + ProhibitLogin bool `json:"prohibit_login"` + Restricted bool `json:"restricted"` +} + +// ensureAccount makes the agent's account, plain, when there is none, and puts back what was changed by hand: never +// an administrator, active, able to use its token. It adds it to no team: what it may read beyond public +// repositories is the operator's to grant (ADR 0365 §3). +func (m *Minter) ensureAccount(ctx context.Context, account string) error { + var u forgeUser + err := m.Forge.GetJSON(ctx, "/api/v1/users/"+account, &u) + switch { + case errors.Is(err, forgehttp.ErrNotFound): + password := m.password() + err := m.Forge.SendJSON(ctx, http.MethodPost, "/api/v1/admin/users", map[string]any{ + "username": account, "email": account + "@noreply.localhost", "password": password, + "must_change_password": false, "visibility": "private", + }, nil) + password = "" + if err != nil { + return fmt.Errorf("the account %s could not be made: %w", account, err) + } + m.say("made the forge account %s for that machine's agent, in no team", account) + return nil + case err != nil: + return fmt.Errorf("the account %s could not be read: %w", account, err) + } + if u.IsAdmin || !u.Active || u.ProhibitLogin { + if err := m.Forge.SendJSON(ctx, http.MethodPatch, "/api/v1/admin/users/"+account, map[string]any{ + "login_name": account, "source_id": 0, "admin": false, "active": true, "prohibit_login": false, + }, nil); err != nil { + return fmt.Errorf("the account %s could not be made a plain account again: %w", account, err) + } + m.say("the account %s was changed by hand (administrator %t, active %t, login prohibited %t); made a plain account again", + account, u.IsAdmin, u.Active, u.ProhibitLogin) + } + return nil +} + +type forgeToken struct { + ID int64 `json:"id"` + Name string `json:"name"` + SHA1 string `json:"sha1"` +} + +// mint deletes every token of the account by this module's name, by its id (a name held twice is cleared, not +// refused), and mints one with the read scope alone. +func (m *Minter) mint(ctx context.Context, account string) (string, error) { + var held []forgeToken + if err := m.Forge.GetJSON(ctx, "/api/v1/users/"+account+"/tokens?limit=50", &held); err != nil { + return "", fmt.Errorf("the tokens of %s could not be read: %w", account, err) + } + for _, t := range held { + if t.Name != TokenName { + continue + } + if err := m.Forge.SendJSON(ctx, http.MethodDelete, fmt.Sprintf("/api/v1/users/%s/tokens/%d", account, t.ID), nil, nil); err != nil && + !errors.Is(err, forgehttp.ErrNotFound) { + return "", fmt.Errorf("the token of %s from before could not be deleted: %w", account, err) + } + } + var made forgeToken + if err := m.Forge.SendJSON(ctx, http.MethodPost, "/api/v1/users/"+account+"/tokens", + map[string]any{"name": TokenName, "scopes": TokenScopes}, &made); err != nil { + return "", fmt.Errorf("no token could be minted for %s: %w", account, err) + } + if strings.TrimSpace(made.SHA1) == "" { + return "", fmt.Errorf("the forge answered no token for %s", account) + } + return strings.TrimSpace(made.SHA1), nil +} + +func (m *Minter) password() string { + if m.Password != nil { + return m.Password() + } + b := make([]byte, 24) + _, _ = rand.Read(b) + return base64.RawURLEncoding.EncodeToString(b) +} + +func (m *Minter) say(format string, a ...any) { + if m.Log != nil { + m.Log(format, a...) + return + } + fmt.Fprintf(os.Stderr, format+"\n", a...) +} diff --git a/modules/gitea/cmd/agent-token/token_test.go b/modules/gitea/cmd/agent-token/token_test.go new file mode 100644 index 00000000..3f12dc4d --- /dev/null +++ b/modules/gitea/cmd/agent-token/token_test.go @@ -0,0 +1,376 @@ +package main + +import ( + "crypto/rand" + "encoding/base64" + "encoding/json" + "fmt" + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "regexp" + "slices" + "strings" + "sync" + "testing" + + "golang.org/x/crypto/nacl/box" + + "gitea/internal/forgehttp" +) + +const adminPassword = "admin-password-never-said" + +// fakeForge is gitea's API as far as this bundle uses it, as the administrator's login reaches it. +type fakeForge struct { + mu sync.Mutex + users map[string]*forgeUser + tokens map[string][]forgeToken + next int64 + calls []string + made map[string]map[string]any + minted map[string][]string // account -> scopes of its last mint + failMint bool +} + +func newFakeForge() *fakeForge { + return &fakeForge{users: map[string]*forgeUser{}, tokens: map[string][]forgeToken{}, made: map[string]map[string]any{}, + minted: map[string][]string{}, next: 1} +} + +var tokenPath = regexp.MustCompile(`^/api/v1/users/([^/]+)/tokens(?:/(\d+))?$`) + +func (f *fakeForge) ServeHTTP(w http.ResponseWriter, r *http.Request) { + f.mu.Lock() + defer f.mu.Unlock() + user, pass, ok := r.BasicAuth() + if !ok || user != "mesh-admin" || pass != adminPassword { + http.Error(w, "unauthorized", http.StatusUnauthorized) + return + } + f.calls = append(f.calls, r.Method+" "+r.URL.Path) + body, _ := io.ReadAll(r.Body) + var in map[string]any + _ = json.Unmarshal(body, &in) + reply := func(status int, v any) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(status) + if v != nil { + _ = json.NewEncoder(w).Encode(v) + } + } + switch { + case r.Method == http.MethodGet && strings.HasPrefix(r.URL.Path, "/api/v1/users/") && strings.Count(r.URL.Path, "/") == 4: + u, ok := f.users[strings.TrimPrefix(r.URL.Path, "/api/v1/users/")] + if !ok { + reply(404, map[string]any{"message": "user does not exist"}) + return + } + reply(200, map[string]any{"is_admin": u.IsAdmin, "active": u.Active, "prohibit_login": u.ProhibitLogin}) + case r.Method == http.MethodPost && r.URL.Path == "/api/v1/admin/users": + name, _ := in["username"].(string) + f.users[name] = &forgeUser{Active: true} + f.made[name] = in + reply(201, map[string]any{"login": name}) + case r.Method == http.MethodPatch && strings.HasPrefix(r.URL.Path, "/api/v1/admin/users/"): + u := f.users[strings.TrimPrefix(r.URL.Path, "/api/v1/admin/users/")] + u.IsAdmin, _ = in["admin"].(bool) + u.Active, _ = in["active"].(bool) + u.ProhibitLogin, _ = in["prohibit_login"].(bool) + reply(200, map[string]any{}) + case tokenPath.MatchString(r.URL.Path): + m := tokenPath.FindStringSubmatch(r.URL.Path) + account := m[1] + switch r.Method { + case http.MethodGet: + reply(200, f.tokens[account]) + case http.MethodDelete: + kept := f.tokens[account][:0] + for _, t := range f.tokens[account] { + if fmt.Sprint(t.ID) != m[2] { + kept = append(kept, t) + } + } + f.tokens[account] = kept + reply(204, nil) + case http.MethodPost: + if f.failMint { + reply(500, map[string]any{"message": "the forge is unwell"}) + return + } + name, _ := in["name"].(string) + for _, t := range f.tokens[account] { + if t.Name == name { + reply(400, map[string]any{"message": "access token name has been used already"}) + return + } + } + var scopes []string + for _, s := range in["scopes"].([]any) { + scopes = append(scopes, s.(string)) + } + f.minted[account] = scopes + b := make([]byte, 20) + _, _ = rand.Read(b) + t := forgeToken{ID: f.next, Name: name, SHA1: fmt.Sprintf("%x", b)} + f.next++ + f.tokens[account] = append(f.tokens[account], forgeToken{ID: t.ID, Name: t.Name}) + reply(201, t) + } + default: + reply(404, map[string]any{"message": "not here: " + r.URL.Path}) + } +} + +type harness struct { + forge *fakeForge + server *httptest.Server + minter *Minter + public *[32]byte + private *[32]byte + logged *[]string +} + +func newHarness(t *testing.T) harness { + t.Helper() + f := newFakeForge() + srv := httptest.NewServer(f) + t.Cleanup(srv.Close) + dir := t.TempDir() + pw := filepath.Join(dir, "admin.secret") + origins := filepath.Join(dir, "origins") + if err := os.WriteFile(pw, []byte(adminPassword+"\n"), 0o600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(origins, []byte("https://git.example.test\n"), 0o644); err != nil { + t.Fatal(err) + } + pub, priv, err := box.GenerateKey(rand.Reader) + if err != nil { + t.Fatal(err) + } + var logged []string + m := &Minter{Forge: forgehttp.Client{URL: srv.URL, User: "mesh-admin", PasswordFile: pw}, Node: "anchor", + OriginsFile: origins, Log: func(format string, a ...any) { logged = append(logged, fmt.Sprintf(format, a...)) }} + return harness{forge: f, server: srv, minter: m, public: pub, private: priv, logged: &logged} +} + +func (h harness) open(t *testing.T, sealed string) string { + t.Helper() + raw, err := base64.StdEncoding.DecodeString(sealed) + if err != nil { + t.Fatal(err) + } + out, ok := box.OpenAnonymous(nil, raw, h.public, h.private) + if !ok { + t.Fatal("a holder's value is not sealed to the call's key") + } + return string(out) +} + +func (h harness) args(holders ...HolderAsk) map[string]any { + list := []any{} + for _, x := range holders { + list = append(list, map[string]any{"node": x.Node, "module": x.Module}) + } + return map[string]any{"provision": Provision, "seal_to": base64.StdEncoding.EncodeToString(h.public[:]), "holders": list} +} + +func TestOnlyTheVaultIsAnsweredAndNothingIsMintedForAnyoneElse(t *testing.T) { + h := newHarness(t) + for _, caller := range []string{"controller", "novox.node-tools", "provisioner.anchor", "anchor.claude-code", + "mesh-vault", "anchor.mesh-vault.x", "", "unproven"} { + if err := fromTheVault(caller); err == nil { + t.Errorf("%q was taken for the vault", caller) + } + } + if err := fromTheVault("novox.mesh-vault"); err != nil { + t.Errorf("the vault was refused: %v", err) + } + // Through the tool as the runtime serves it: a call no runtime named a caller for is the vault's to nobody. + tools := Tools(func() (*Minter, error) { return h.minter, nil }) + if _, err := tools[0].Run(h.args(HolderAsk{"anchor", "claude-code"})); err == nil || !strings.Contains(err.Error(), "vault alone") { + t.Fatalf("an unproven caller was answered: %v", err) + } + if len(h.forge.calls) != 0 { + t.Fatalf("the forge was reached for a caller that is not the vault: %v", h.forge.calls) + } +} + +func TestEachAgentGetsAnAccountOfItsOwnAndAReadTokenSealedForItAlone(t *testing.T) { + h := newHarness(t) + ask, err := readAsk(h.args(HolderAsk{"anchor", "claude-code"}, HolderAsk{"laptop", "claude-code"}, HolderAsk{"laptop", "lab"})) + if err != nil { + t.Fatal(err) + } + got, err := h.minter.Answer(ask) + if err != nil { + t.Fatal(err) + } + if len(got.Holders) != 2 { + t.Fatalf("answered %d holders, want the two agents: %+v", len(got.Holders), got) + } + if !strings.Contains(got.Skipped["lab on laptop"], Holder) { + t.Errorf("a holder that is not an agent's module was not said: %v", got.Skipped) + } + lines := map[string]string{} + for _, s := range got.Holders { + lines[s.Node] = h.open(t, s.Sealed) + } + anchorTok := tokenIn(t, lines["anchor"], "agent-anchor") + laptopTok := tokenIn(t, lines["laptop"], "agent-laptop") + if anchorTok == laptopTok { + t.Fatal("two agents were handed one token") + } + // The forge's own machine reaches it at its own address too; another machine only at its public origin. + if !strings.Contains(lines["anchor"], "https://agent-anchor:"+anchorTok+"@git.example.test\n") || + !strings.Contains(lines["anchor"], strings.Replace(h.server.URL, "http://", "http://agent-anchor:"+anchorTok+"@", 1)) { + t.Errorf("anchor's credential lines: %q", strings.ReplaceAll(lines["anchor"], anchorTok, "")) + } + if lines["laptop"] != "https://agent-laptop:"+laptopTok+"@git.example.test\n" { + t.Errorf("laptop's credential lines: %q", strings.ReplaceAll(lines["laptop"], laptopTok, "")) + } + for _, account := range []string{"agent-anchor", "agent-laptop"} { + u := h.forge.users[account] + if u == nil || u.IsAdmin { + t.Errorf("%s: %+v, want a plain account", account, u) + } + if made := h.forge.made[account]; made["must_change_password"] != false || made["password"] == "" { + t.Errorf("%s was made with %v", account, made) + } + if !slices.Equal(h.forge.minted[account], []string{"read:repository"}) { + t.Errorf("%s's token carries %v, want read:repository alone", account, h.forge.minted[account]) + } + } + for _, c := range h.forge.calls { + if strings.Contains(c, "/teams") || strings.Contains(c, "/orgs/") || strings.Contains(c, "collaborators") { + t.Errorf("an agent's account was given access (%s): it starts in no team, the operator grants (ADR 0365 §3)", c) + } + } + if _, made := h.forge.users["agent-laptop-lab"]; made || len(h.forge.users) != 2 { + t.Errorf("accounts made: %v", h.forge.users) + } + said := strings.Join(*h.logged, "\n") + if strings.Contains(said, anchorTok) || strings.Contains(said, laptopTok) { + t.Fatal("a token was said in the log") + } +} + +func TestAskedAgainTheTokenIsMintedAnewAndTheOneFromBeforeDeleted(t *testing.T) { + h := newHarness(t) + ask, _ := readAsk(h.args(HolderAsk{"laptop", "claude-code"})) + first, err := h.minter.Answer(ask) + if err != nil { + t.Fatal(err) + } + h.forge.tokens["agent-laptop"] = append(h.forge.tokens["agent-laptop"], forgeToken{ID: 99, Name: "a-persons-own"}) + again, err := h.minter.Answer(ask) + if err != nil { + t.Fatal(err) + } + if h.open(t, first.Holders[0].Sealed) == h.open(t, again.Holders[0].Sealed) { + t.Fatal("asked again, the same token was answered: it was not rotated") + } + var names []string + for _, tok := range h.forge.tokens["agent-laptop"] { + names = append(names, tok.Name) + } + slices.Sort(names) + if !slices.Equal(names, []string{"a-persons-own", TokenName}) { + t.Fatalf("the account holds %v: the token from before is not deleted, or another was", names) + } +} + +func TestAnAccountMadeAnAdministratorByHandIsMadePlainAgain(t *testing.T) { + h := newHarness(t) + h.forge.users["agent-laptop"] = &forgeUser{IsAdmin: true, Active: true} + ask, _ := readAsk(h.args(HolderAsk{"laptop", "claude-code"})) + if _, err := h.minter.Answer(ask); err != nil { + t.Fatal(err) + } + if h.forge.users["agent-laptop"].IsAdmin { + t.Fatal("the agent's account is still an administrator") + } +} + +func TestAFailureSaysNeitherTheAdminsPasswordNorATokenAndAnswersNothing(t *testing.T) { + h := newHarness(t) + h.forge.failMint = true + ask, _ := readAsk(h.args(HolderAsk{"anchor", "claude-code"}, HolderAsk{"laptop", "claude-code"})) + got, err := h.minter.Answer(ask) + if err == nil || len(got.Holders) != 0 { + t.Fatalf("a failed mint answered %+v, %v", got, err) + } + if strings.Contains(err.Error(), adminPassword) { + t.Fatal("the admin's password was said in an error") + } + // A wrong password is a refusal, said without it. + if err := os.WriteFile(h.minter.Forge.PasswordFile, []byte("wrong"), 0o600); err != nil { + t.Fatal(err) + } + h.forge.failMint = false + if _, err := h.minter.Answer(ask); err == nil || strings.Contains(err.Error(), "wrong") { + t.Fatalf("a refused login: %v", err) + } +} + +func TestTheAskIsReadStrictly(t *testing.T) { + h := newHarness(t) + for name, args := range map[string]map[string]any{ + "another provision": {"provision": "git", "seal_to": base64.StdEncoding.EncodeToString(h.public[:]), + "holders": []any{map[string]any{"node": "a", "module": "claude-code"}}}, + "no key": {"provision": Provision, "holders": []any{map[string]any{"node": "a", "module": "claude-code"}}}, + "short key": {"provision": Provision, "seal_to": "AAAA", "holders": []any{map[string]any{"node": "a", "module": "claude-code"}}}, + "no holders": {"provision": Provision, "seal_to": base64.StdEncoding.EncodeToString(h.public[:])}, + "a holder twice": {"provision": Provision, "seal_to": base64.StdEncoding.EncodeToString(h.public[:]), + "holders": []any{map[string]any{"node": "a", "module": "claude-code"}, map[string]any{"node": "a", "module": "claude-code"}}}, + } { + if _, err := readAsk(args); err == nil { + t.Errorf("%s was taken", name) + } + } +} + +func TestTheReadmeNamesWhatTheBundleServes(t *testing.T) { + var served []string + for _, tool := range Tools(MinterFromEnv) { + served = append(served, tool.Name) + } + if !slices.Equal(served, ToolNames) { + t.Fatalf("served %v, named %v", served, ToolNames) + } + readme, err := os.ReadFile("../../README.md") + if err != nil { + t.Fatal(err) + } + for _, n := range ToolNames { + if !strings.Contains(string(readme), "`"+n+"`") { + t.Errorf("the README does not name %s", n) + } + } + manifest, err := os.ReadFile("../../module.json") + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(manifest), `"tool": "`+ToolName+`"`) || !strings.Contains(string(manifest), `"name": "`+Provision+`"`) { + t.Error("the manifest does not offer the provision with this tool as its issuer") + } +} + +// tokenIn is the token of account's credential lines, all one. +func tokenIn(t *testing.T, lines, account string) string { + t.Helper() + re := regexp.MustCompile(regexp.QuoteMeta(account) + `:([0-9a-f]{40})@`) + all := re.FindAllStringSubmatch(lines, -1) + if len(all) == 0 { + t.Fatalf("no credential for %s in the lines", account) + } + for _, m := range all { + if m[1] != all[0][1] { + t.Fatalf("%s's lines carry two tokens", account) + } + } + return all[0][1] +} diff --git a/modules/gitea/cmd/forge/templates_test.go b/modules/gitea/cmd/forge/templates_test.go new file mode 100644 index 00000000..7d0718b5 --- /dev/null +++ b/modules/gitea/cmd/forge/templates_test.go @@ -0,0 +1,64 @@ +package main + +import ( + "encoding/json" + "os" + "path/filepath" + "testing" +) + +// The module's directory, where its manifest and its files/ are. +const moduleDir = "../.." + +// templateText is the text a file resource places on the machine: its template's, in the module's files/ +// (novox/hq ADR 0349 §2). A resource that names no template, or a template that is not there, fails the test. +func templateText(t *testing.T, r map[string]any) string { + t.Helper() + name, _ := r["template"].(string) + if name == "" { + t.Fatalf("resource %v names no template", r["id"]) + } + raw, err := os.ReadFile(filepath.Join(moduleDir, filepath.FromSlash(name))) + if err != nil { + t.Fatalf("resource %v names %s: %v", r["id"], name, err) + } + return string(raw) +} + +// No file resource carries its text inline: each names its template in files/, and the build packs files/ +// into the one archive the controller reads the templates back from (novox/hq ADR 0349 §2). +func TestEveryFileTakesItsTextFromATemplate(t *testing.T) { + raw, err := os.ReadFile(filepath.Join(moduleDir, "module.json")) + if err != nil { + t.Fatal(err) + } + var m struct { + Resources []map[string]any `json:"resources"` + Build struct { + Artifacts []map[string]any `json:"artifacts"` + } `json:"build"` + } + if err := json.Unmarshal(raw, &m); err != nil { + t.Fatal(err) + } + for _, r := range m.Resources { + if _, inline := r["content"]; inline { + t.Errorf("resource %v keeps its text inline: keep it in files/ and name it as the template", r["id"]) + } + if r["type"] == "file" { + templateText(t, r) + } + } + packs := 0 + for _, a := range m.Build.Artifacts { + if a["kind"] == "archive" && a["from"] == "files" { + packs++ + if a["name"] != "files" { + t.Errorf("the templates archive is named %v, not files", a["name"]) + } + } + } + if packs != 1 { + t.Errorf("the build packs files/ into %d archives, not one", packs) + } +} diff --git a/modules/gitea/files/agent-teams.json b/modules/gitea/files/agent-teams.json new file mode 100644 index 00000000..0967ef42 --- /dev/null +++ b/modules/gitea/files/agent-teams.json @@ -0,0 +1 @@ +{} diff --git a/modules/gitea/files/config.json b/modules/gitea/files/config.json new file mode 100644 index 00000000..0967ef42 --- /dev/null +++ b/modules/gitea/files/config.json @@ -0,0 +1 @@ +{} diff --git a/modules/gitea/files/origins b/modules/gitea/files/origins new file mode 100644 index 00000000..6bbfb357 --- /dev/null +++ b/modules/gitea/files/origins @@ -0,0 +1 @@ +https://${bound:route:name-web} diff --git a/modules/gitea/files/ready-authors b/modules/gitea/files/ready-authors new file mode 100644 index 00000000..794440bf --- /dev/null +++ b/modules/gitea/files/ready-authors @@ -0,0 +1,3 @@ +# The accounts whose "ready at " comment marks a pull request ready (novox/hq issue 722), from the setting +# ready-authors, written by the mesh. Replaced on every push; change the setting, never this file. +authors=${config:ready-authors} diff --git a/modules/gitea/files/server.env b/modules/gitea/files/server.env new file mode 100644 index 00000000..8cb88e22 --- /dev/null +++ b/modules/gitea/files/server.env @@ -0,0 +1,7 @@ +GITEA__security__INTERNAL_TOKEN=${secret:internal-token} +GITEA__database__DB_TYPE=postgres +GITEA__database__HOST=${bound:postgres-database:at}:${bound:postgres-database:port} +GITEA__database__NAME=${bound:postgres-database:as} +GITEA__database__USER=${bound:postgres-database:as} +GITEA__database__PASSWD=${secret:postgres-database} +GITEA__webhook__ALLOWED_HOST_LIST=172.17.0.1 diff --git a/modules/gitea/go.mod b/modules/gitea/go.mod index c05b9bd6..ea8089d6 100644 --- a/modules/gitea/go.mod +++ b/modules/gitea/go.mod @@ -1,5 +1,10 @@ module gitea -go 1.22 +go 1.25.0 -require git.novox.be/novox/mesh-sdk/go v0.1.7 +require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261013235212-c8b07242a009 + +require ( + golang.org/x/crypto v0.50.0 + golang.org/x/sys v0.43.0 // indirect +) diff --git a/modules/gitea/go.sum b/modules/gitea/go.sum index b474419e..afe914a8 100644 --- a/modules/gitea/go.sum +++ b/modules/gitea/go.sum @@ -1,2 +1,6 @@ -git.novox.be/novox/mesh-sdk/go v0.1.7 h1:C0sTQmtTiyYH7bnqZb7PusXnqA37gKuT7Nqjn9gG47w= -git.novox.be/novox/mesh-sdk/go v0.1.7/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261013235212-c8b07242a009 h1:I9rm8HhYRJ0+mzQQz+VAH0kgv2YJWkvctEc67Yd0PX8= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261013235212-c8b07242a009/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY= +golang.org/x/crypto v0.50.0 h1:zO47/JPrL6vsNkINmLoo/PH1gcxpls50DNogFvB5ZGI= +golang.org/x/crypto v0.50.0/go.mod h1:3muZ7vA7PBCE6xgPX7nkzzjiUq87kRItoJQM1Yo8S+Q= +golang.org/x/sys v0.43.0 h1:Rlag2XtaFTxp19wS8MXlJwTvoh8ArU6ezoyFsMyCTNI= +golang.org/x/sys v0.43.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= diff --git a/modules/gitea/module.json b/modules/gitea/module.json index 1908795c..a086f1d3 100644 --- a/modules/gitea/module.json +++ b/modules/gitea/module.json @@ -1,6 +1,7 @@ { "module": "gitea", "version": "1", + "moved": true, "requires": [ "postgres-database", "route", @@ -156,11 +157,24 @@ } } }, - "settings": { + "config": { + "url": { + "type": "string", + "description": "The address the module's tools reach the forge at, as a URL; unset, the code falls back to its environment, else this machine's gitea on its port.", + "trusted": true, + "default": "" + }, + "agent-read-teams": { + "type": "object", + "description": "The read teams of the agents' forge accounts (ADR 0365 clause 3): each team by name, {\"\": {\"repos\": [\"/\", ...], \"members\": [\"\", ...]}}. The forge gets a team agent-read- that reads only, with exactly those repositories and the accounts agent-; a team taken out of the value is emptied. Nothing else on the forge is touched; an agent account is in no team until it is listed here.", + "trusted": true, + "default": {} + }, "ready-authors": { - "kind": "preference", - "default": "mesh-tools mesh-admin jschoubben", - "why": "the forge accounts whose comment \"ready at \" marks a pull request ready for mesh-delivery to merge, separated by spaces: the account the forge's tools act as (novox/hq ADR 0365), the admin account for marks made before the tools acted as their own, and the operator's (novox/hq issue 722). A comment by any other account is ignored and said in the forge's log. Trusted: who may have a change merged is the operator's word" + "type": "string", + "description": "the forge accounts whose comment \"ready at \" marks a pull request ready for mesh-delivery to merge, separated by spaces: the account the forge's tools act as (novox/hq ADR 0365), the admin account for marks made before the tools acted as their own, and the operator's (novox/hq issue 722). A comment by any other account is ignored and said in the forge's log. Trusted: who may have a change merged is the operator's word", + "trusted": true, + "default": "mesh-tools mesh-admin jschoubben" } }, "resources": [ @@ -182,7 +196,7 @@ "path": "${dir:mesh-state}/ready-authors", "mode": "0600", "trusted": true, - "content": "# The accounts whose \"ready at \" comment marks a pull request ready (novox/hq issue 722), from the setting\n# ready-authors, written by the mesh. Replaced on every push; change the setting, never this file.\nauthors=${setting:ready-authors}\n" + "template": "files/ready-authors" }, { "id": "forge-patches", @@ -207,7 +221,7 @@ "type": "file", "path": "${dir:state}/server.env", "mode": "0600", - "content": "GITEA__security__INTERNAL_TOKEN=${secret:internal-token}\nGITEA__database__DB_TYPE=postgres\nGITEA__database__HOST=${bound:postgres-database:at}:${bound:postgres-database:port}\nGITEA__database__NAME=${bound:postgres-database:as}\nGITEA__database__USER=${bound:postgres-database:as}\nGITEA__database__PASSWD=${secret:postgres-database}\nGITEA__webhook__ALLOWED_HOST_LIST=172.17.0.1\n" + "template": "files/server.env" }, { "id": "data", @@ -270,14 +284,53 @@ ], "secrets-in-environment": "gitea honours GITEA__database__PASSWD__FILE and GITEA__security__INTERNAL_TOKEN__FILE; convertible, awaiting a bed that proves it" }, + { + "id": "origins", + "type": "file", + "path": "${dir:mesh-state}/origins", + "mode": "0644", + "template": "files/origins" + }, { "id": "runtime-config", "type": "file", "path": "${dir:mesh-state}/config.json", "mode": "0600", - "content": "{}\n", + "template": "files/config.json", "merge": "json", - "takes-settings": [] + "takes-config": [ + "url" + ] + }, + { + "id": "agent-teams-file", + "type": "file", + "path": "${dir:mesh-state}/agent-teams.json", + "mode": "0600", + "template": "files/agent-teams.json", + "merge": "json", + "takes-config": [ + "agent-read-teams" + ] + }, + { + "id": "agent-teams", + "type": "process", + "name": "gitea-agent-teams", + "artifact": "agent-teams", + "run": [ + "./agent-teams" + ], + "run-once": true, + "env": { + "MESH_GITEA_URL": "http://127.0.0.1:${port:3000}", + "MESH_GITEA_ADMIN_USER": "mesh-admin", + "MESH_GITEA_ADMIN_PASSWORD_FILE": "${dir:state}/admin.secret", + "MESH_AGENT_TEAMS_FILE": "${dir:mesh-state}/agent-teams.json" + }, + "restart-on": [ + "agent-teams-file" + ] } ], "provides": [ @@ -292,6 +345,14 @@ { "name": "git", "scope": "mesh" + }, + { + "name": "gitea-read", + "scope": "mesh", + "credential": { + "issued-by": "software", + "tool": "gitea_agent_token" + } } ], "build": { @@ -357,6 +418,39 @@ "MESH_GITEA_TOKEN_FILE": "${dir:runtime-state}/token", "MESH_GITEA_PATCH_DIR": "${dir:forge-patches}" } + }, + { + "name": "files", + "kind": "archive", + "from": "files" + }, + { + "name": "agent-token", + "kind": "bundle", + "language": "go", + "system": "arch", + "from": "cmd/agent-token", + "binary": "agent-token", + "loads": [ + "agent-token" + ], + "env": { + "MESH_GITEA_URL": "http://127.0.0.1:${port:3000}", + "MESH_GITEA_ADMIN_USER": "mesh-admin", + "MESH_GITEA_ADMIN_PASSWORD_FILE": "${dir:state}/admin.secret", + "MESH_GITEA_ORIGINS_FILE": "${dir:mesh-state}/origins" + } + }, + { + "name": "agent-teams", + "kind": "bundle", + "language": "go", + "system": "arch", + "from": "cmd/agent-teams", + "binary": "agent-teams", + "loads": [ + "agent-teams" + ] } ] }, diff --git a/modules/gitea/test/tools-account.test.ts b/modules/gitea/test/tools-account.test.ts index efc6dcfa..6941a13a 100644 --- a/modules/gitea/test/tools-account.test.ts +++ b/modules/gitea/test/tools-account.test.ts @@ -262,7 +262,7 @@ test("the manifest hands the tools their account, and the merge-ready authors na const manifest = JSON.parse(await readFile(new URL("../module.json", import.meta.url), "utf8")); const code = manifest.build.artifacts.find((a: any) => a.name === "code"); assert.equal(code.env.MESH_GITEA_TOOLS_USER, TOOLS_ACCOUNT); - const authors = String(manifest.settings["ready-authors"].default).split(/\s+/); + const authors = String(manifest.config["ready-authors"].default).split(/\s+/); assert.ok(authors.includes(TOOLS_ACCOUNT)); // Kept for now (review of issue 469): a pull request the admin account marked ready before the tools acted as // their own keeps counting as ready after the delivery. diff --git a/modules/gitlab/files/config.json b/modules/gitlab/files/config.json new file mode 100644 index 00000000..9e26dfee --- /dev/null +++ b/modules/gitlab/files/config.json @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/modules/gitlab/module.json b/modules/gitlab/module.json index a925e1bb..38be2014 100644 --- a/modules/gitlab/module.json +++ b/modules/gitlab/module.json @@ -4,6 +4,14 @@ "own-secrets": { "token": "${dir:state}/token" }, + "config": { + "gitlab-url": { + "type": "string", + "description": "The address of the GitLab instance the gitlab tools call, as a URL such as https://gitlab.com, laid into the file as GITLAB_URL; unset, the code falls back to its environment and answers no calls until it is given.", + "trusted": true, + "default": "" + } + }, "resources": [ { "id": "state", @@ -16,8 +24,14 @@ "type": "file", "path": "${dir:state}/config.json", "merge": "json", - "content": "{}", - "mode": "0600" + "template": "files/config.json", + "mode": "0600", + "takes-config": [ + { + "key": "gitlab-url", + "as": "GITLAB_URL" + } + ] } ], "capabilities": [ @@ -39,6 +53,11 @@ "MESH_GITLAB_TOKEN_FILE": "${dir:state}/token", "MESH_GITLAB_CONFIG_FILE": "${dir:state}/config.json" } + }, + { + "name": "files", + "kind": "archive", + "from": "files" } ] } diff --git a/modules/gnome-keyring/cmd/gnome-keyring-tools/manifest_helpers_test.go b/modules/gnome-keyring/cmd/gnome-keyring-tools/manifest_helpers_test.go index d4fb76d9..fb992951 100644 --- a/modules/gnome-keyring/cmd/gnome-keyring-tools/manifest_helpers_test.go +++ b/modules/gnome-keyring/cmd/gnome-keyring-tools/manifest_helpers_test.go @@ -14,6 +14,8 @@ import ( type manifest struct { Module string `json:"module"` Version string `json:"version"` + Moved bool `json:"moved"` + Items []item `json:"items"` Capabilities []string `json:"capabilities"` Requires []string `json:"requires"` Claims []claim `json:"claims"` @@ -44,6 +46,53 @@ type shellCode struct { Code string `json:"code"` } +// An item of a collection (novox/hq ADR 0349 clause 13): its text is the template file beside the manifest. +type item struct { + Collection string `json:"collection"` + Name string `json:"name"` + Attributes map[string]string `json:"attributes"` + Template string `json:"template"` +} + +// shell is the code the module adds to a shell's or the session's start, read from the templates of its items of +// the collections shell::, in the order they are declared. +func (m manifest) shell(t *testing.T) []shellCode { + t.Helper() + var out []shellCode + for _, it := range m.Items { + parts := strings.Split(it.Collection, ":") + if len(parts) != 3 || parts[0] != "shell" { + continue + } + text, err := os.ReadFile(filepath.Join("..", "..", it.Template)) + if err != nil { + t.Fatal(err) + } + out = append(out, shellCode{For: parts[1], Slot: parts[2], Code: string(text)}) + } + return out +} + +// environmentOf is the variables the module sets, read from the templates of its items of the environment. +func (m manifest) environmentOf(t *testing.T) *environment { + t.Helper() + var env *environment + for _, it := range m.Items { + if it.Collection != "environment" || it.Attributes["kind"] != "variable" { + continue + } + text, err := os.ReadFile(filepath.Join("..", "..", it.Template)) + if err != nil { + t.Fatal(err) + } + if env == nil { + env = &environment{Variables: map[string]string{}} + } + env.Variables[it.Attributes["name"]] = strings.TrimSuffix(string(text), "\n") + } + return env +} + func readManifest(t *testing.T) manifest { t.Helper() raw, err := os.ReadFile(filepath.Join("..", "..", "module.json")) @@ -56,6 +105,7 @@ func readManifest(t *testing.T) manifest { if err := dec.Decode(&m); err != nil { t.Fatalf("module.json: %v", err) } + m.Environment = m.environmentOf(t) return m } @@ -83,26 +133,66 @@ func (m manifest) packages() (present, absent []string) { return present, absent } -// sameAsSource checks that a file resource's content is byte for byte the module's source file, so -// the readable file in the repository is what the machine gets. -func (m manifest) sameAsSource(t *testing.T, id, source string) { +// fromTemplate checks that a file resource takes its text from its template file in the module's +// repository (novox/hq ADR 0349 §2): it names the template, carries no text of its own beside it, and the +// template is there. The readable file in the repository is the one place the machine's text is written. +func (m manifest) fromTemplate(t *testing.T, id, template string) { t.Helper() - want, err := os.ReadFile(filepath.Join("..", "..", source)) - if err != nil { - t.Fatal(err) - } r := m.resource(t, id) if r["type"] != "file" { t.Fatalf("%s is a %v, not a file", id, r["type"]) } - if got, _ := r["content"].(string); got != string(want) { - t.Fatalf("resource %s's content is not %s: edit the source and copy it into module.json", id, source) + if r["template"] != template { + t.Fatalf("resource %s does not take its text from %s: template %v", id, template, r["template"]) + } + if _, inline := r["content"]; inline { + t.Fatalf("resource %s keeps its text inline beside its template: a file's text is in one place", id) + } + if _, err := os.Stat(filepath.Join("..", "..", template)); err != nil { + t.Fatalf("resource %s names %s: %v", id, template, err) } if r["owner"] != "${machine:account}" && !strings.HasPrefix(r["path"].(string), "/etc/") { t.Fatalf("%s under the home is the account's", id) } } +// everyFileIsATemplate checks that no file resource carries its text inline, and that the build packs +// files/ into the one archive the controller reads the templates back from (novox/hq ADR 0349 §2). +func (m manifest) everyFileIsATemplate(t *testing.T) { + t.Helper() + for _, r := range m.Resources { + if _, inline := r["content"]; inline { + t.Errorf("resource %v keeps its text inline: keep it in files/ and name it as the template", r["id"]) + } + } + packs := 0 + for _, a := range m.Build.Artifacts { + if a["kind"] == "archive" && a["from"] == "files" { + packs++ + if a["name"] != "files" { + t.Errorf("the templates archive is named %v, not files", a["name"]) + } + } + } + if packs != 1 { + t.Errorf("the build packs files/ into %d archives, not one", packs) + } +} + +// text is the text a file resource places on the machine: its template file's. +func (m manifest) text(t *testing.T, id string) string { + t.Helper() + name, _ := m.resource(t, id)["template"].(string) + if name == "" { + t.Fatalf("resource %s names no template", id) + } + raw, err := os.ReadFile(filepath.Join("..", "..", name)) + if err != nil { + t.Fatal(err) + } + return string(raw) +} + // checkTheToolsAgree checks that the manifest lists the module's own tools exactly, that the bundle // serves each seat verb the claims promise as ., and that the Go bundle is declared. func checkTheToolsAgree(t *testing.T, m manifest) { diff --git a/modules/gnome-keyring/cmd/gnome-keyring-tools/manifest_test.go b/modules/gnome-keyring/cmd/gnome-keyring-tools/manifest_test.go index 67754e06..25f7b3f6 100644 --- a/modules/gnome-keyring/cmd/gnome-keyring-tools/manifest_test.go +++ b/modules/gnome-keyring/cmd/gnome-keyring-tools/manifest_test.go @@ -29,13 +29,13 @@ func TestThePAMLinesAreBlocksWrittenIntoTheStacks(t *testing.T) { "pam-login": {"/etc/pam.d/login", "files/pam/login"}, "pam-passwd": {"/etc/pam.d/passwd", "files/pam/passwd"}, } { - m.sameAsSource(t, id, want.source) + m.fromTemplate(t, id, want.source) r := m.resource(t, id) if r["path"] != want.path || r["into"] != "block" || r["at"] != "end" || r["owner"] != nil { t.Errorf("%s: %v", id, r) } } - login := m.resource(t, "pam-login")["content"].(string) + login := m.text(t, "pam-login") if !strings.Contains(login, "\nauth optional pam_gnome_keyring.so\n") || !strings.Contains(login, "\nsession optional pam_gnome_keyring.so auto_start\n") { t.Fatalf("%s", login) @@ -44,10 +44,10 @@ func TestThePAMLinesAreBlocksWrittenIntoTheStacks(t *testing.T) { func TestItStartsNoDaemonAndOnlyNamesTheAgentsSocket(t *testing.T) { m := readManifest(t) - if len(m.Shell) != 1 || m.Shell[0].For != "xinitrc" || m.Shell[0].Slot != "first" { - t.Fatalf("%+v", m.Shell) + if len(m.shell(t)) != 1 || m.shell(t)[0].For != "xinitrc" || m.shell(t)[0].Slot != "first" { + t.Fatalf("%+v", m.shell(t)) } - code := m.Shell[0].Code + code := m.shell(t)[0].Code if strings.Contains(code, "gnome-keyring-daemon") || strings.Contains(code, "--unlock") || !strings.Contains(code, `SSH_AUTH_SOCK="${XDG_RUNTIME_DIR:-/run/user/$(id -u)}/gcr/ssh"`) { t.Fatalf("%q", code) @@ -62,3 +62,8 @@ func TestTheToolsAgreeWithTheManifest(t *testing.T) { checkTheToolsAgree(t, m) checkNoSecretsOrInstallationNames(t) } + +// No file resource carries its text inline (novox/hq ADR 0349 §2). +func TestEveryFileTakesItsTextFromATemplate(t *testing.T) { + readManifest(t).everyFileIsATemplate(t) +} diff --git a/modules/gnome-keyring/files/xinitrc-first b/modules/gnome-keyring/files/xinitrc-first new file mode 100644 index 00000000..5fb92b81 --- /dev/null +++ b/modules/gnome-keyring/files/xinitrc-first @@ -0,0 +1,5 @@ +# The ssh agent (module gnome-keyring, novox/hq ADR 0208): gcr's, which the account's service manager +# starts on first use from its socket. Named here, for the session and every terminal it starts, until +# the account's environment can say a path under the runtime directory (see the module's README). +SSH_AUTH_SOCK="${XDG_RUNTIME_DIR:-/run/user/$(id -u)}/gcr/ssh" +export SSH_AUTH_SOCK diff --git a/modules/gnome-keyring/module.json b/modules/gnome-keyring/module.json index 0c796fc2..9f75b9ff 100644 --- a/modules/gnome-keyring/module.json +++ b/modules/gnome-keyring/module.json @@ -1,6 +1,7 @@ { "module": "gnome-keyring", "version": "1", + "moved": true, "capabilities": [ "package-manager" ], @@ -16,11 +17,11 @@ "gnome_keyring_collections", "gnome_keyring_ssh_keys" ], - "shell": [ + "items": [ { - "for": "xinitrc", - "slot": "first", - "code": "# The ssh agent (module gnome-keyring, novox/hq ADR 0208): gcr's, which the account's service manager\n# starts on first use from its socket. Named here, for the session and every terminal it starts, until\n# the account's environment can say a path under the runtime directory (see the module's README).\nSSH_AUTH_SOCK=\"${XDG_RUNTIME_DIR:-/run/user/$(id -u)}/gcr/ssh\"\nexport SSH_AUTH_SOCK\n" + "collection": "shell:xinitrc:first", + "name": "xinitrc-first", + "template": "files/xinitrc-first" } ], "resources": [ @@ -46,7 +47,7 @@ "mode": "0644", "into": "block", "at": "end", - "content": "# The login keyring (module gnome-keyring, novox/hq ADR 0208, ADR 0102): the password typed at the\n# login screen unlocks the keyring, and the login session starts the keyring daemon with it.\nauth optional pam_gnome_keyring.so\nsession optional pam_gnome_keyring.so auto_start\n" + "template": "files/pam/login" }, { "id": "pam-passwd", @@ -55,7 +56,7 @@ "mode": "0644", "into": "block", "at": "end", - "content": "# The login keyring (module gnome-keyring, novox/hq ADR 0208, ADR 0102): changing the account's\n# password changes the login keyring's with it, so the next login still unlocks it.\npassword optional pam_gnome_keyring.so\n" + "template": "files/pam/passwd" } ], "build": { @@ -70,6 +71,11 @@ "loads": [ "gnome-keyring-tools" ] + }, + { + "name": "files", + "kind": "archive", + "from": "files" } ] } diff --git a/modules/grafana/files/admin.secret b/modules/grafana/files/admin.secret new file mode 100644 index 00000000..51bf424b --- /dev/null +++ b/modules/grafana/files/admin.secret @@ -0,0 +1 @@ +${secret:admin} \ No newline at end of file diff --git a/modules/grafana/files/config.json b/modules/grafana/files/config.json new file mode 100644 index 00000000..567df9a7 --- /dev/null +++ b/modules/grafana/files/config.json @@ -0,0 +1,4 @@ +{ + "user": "admin", + "password": "${secret:admin}" +} diff --git a/modules/grafana/files/influxdb-api b/modules/grafana/files/influxdb-api new file mode 100644 index 00000000..8329ab78 --- /dev/null +++ b/modules/grafana/files/influxdb-api @@ -0,0 +1 @@ +${secret:influxdb-api} \ No newline at end of file diff --git a/modules/grafana/files/mesh-influxdb.yaml b/modules/grafana/files/mesh-influxdb.yaml new file mode 100644 index 00000000..e3410057 --- /dev/null +++ b/modules/grafana/files/mesh-influxdb.yaml @@ -0,0 +1,19 @@ +apiVersion: 1 +# Written by the mesh from grafana's influxdb-api binding, only while it is bound (novox/hq ADR 0330): +# without influxdb, grafana runs with no mesh data source. grafana reads it at start. Its own name and +# uid, so a data source somebody made in the UI is never overwritten, and read-only in the UI because +# the mesh resets it. The password is read from the file the mesh delivers, never written here. +datasources: + - name: InfluxDB (mesh) + uid: mesh-influxdb-api + type: influxdb + access: proxy + url: ${bound:influxdb-api:scheme}://${bound:influxdb-api:at}:${bound:influxdb-api:port} + user: ${bound:influxdb-api:as} + isDefault: false + editable: false + jsonData: + dbName: ${bound:influxdb-api:bucket} + httpMode: POST + secureJsonData: + password: $__file{/run/secrets/influxdb/influxdb-api} diff --git a/modules/grafana/files/oidc-client.secret b/modules/grafana/files/oidc-client.secret new file mode 100644 index 00000000..9146dc18 --- /dev/null +++ b/modules/grafana/files/oidc-client.secret @@ -0,0 +1 @@ +${secret:oidc-client} \ No newline at end of file diff --git a/modules/grafana/files/oidc.env b/modules/grafana/files/oidc.env new file mode 100644 index 00000000..bcc1e9c2 --- /dev/null +++ b/modules/grafana/files/oidc.env @@ -0,0 +1,13 @@ +GF_SERVER_ROOT_URL=https://${bound:route:name} +GF_AUTH_GENERIC_OAUTH_ENABLED=true +GF_AUTH_GENERIC_OAUTH_NAME=Keycloak +GF_AUTH_GENERIC_OAUTH_CLIENT_ID=${bound:oidc-client:as} +GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET__FILE=/run/secrets/oidc-client +GF_AUTH_GENERIC_OAUTH_SCOPES=openid email profile roles +GF_AUTH_GENERIC_OAUTH_AUTH_URL=${bound:oidc-client:issuer}${bound:oidc-client:authorization-path} +GF_AUTH_GENERIC_OAUTH_TOKEN_URL=${bound:oidc-client:issuer}${bound:oidc-client:token-path} +GF_AUTH_GENERIC_OAUTH_API_URL=${bound:oidc-client:issuer}${bound:oidc-client:userinfo-path} +GF_AUTH_GENERIC_OAUTH_ROLE_ATTRIBUTE_PATH=contains(roles[*], 'admin') && 'Admin' || contains(realm_access.roles[*], 'admin') && 'Admin' || 'Viewer' +GF_AUTH_GENERIC_OAUTH_USE_PKCE=true +GF_AUTH_GENERIC_OAUTH_ALLOW_SIGN_UP=true +GF_AUTH_GENERIC_OAUTH_ALLOW_ASSIGN_GRAFANA_ADMIN=true diff --git a/modules/grafana/module.json b/modules/grafana/module.json index 8c16177f..9d8ceebf 100644 --- a/modules/grafana/module.json +++ b/modules/grafana/module.json @@ -29,6 +29,20 @@ } ] }, + "config": { + "user": { + "type": "string", + "description": "The Grafana account the module's tools sign in as with the vault's password; the file's default is admin.", + "trusted": true, + "default": "admin" + }, + "url": { + "type": "string", + "description": "The address the module's tools reach Grafana at, as a URL; unset, the code falls back to its environment, else this machine's Grafana on its port.", + "trusted": true, + "default": "" + } + }, "resources": [ { "id": "mesh-state", @@ -54,7 +68,7 @@ "path": "${dir:state}/admin.secret", "mode": "0400", "owner": "472:472", - "content": "${secret:admin}" + "template": "files/admin.secret" }, { "id": "oidc-secret", @@ -62,14 +76,14 @@ "path": "${dir:state}/oidc-client.secret", "mode": "0400", "owner": "472:472", - "content": "${secret:oidc-client}" + "template": "files/oidc-client.secret" }, { "id": "oidc-env", "type": "file", "path": "${dir:state}/oidc.env", "mode": "0644", - "content": "GF_SERVER_ROOT_URL=https://${bound:route:name}\nGF_AUTH_GENERIC_OAUTH_ENABLED=true\nGF_AUTH_GENERIC_OAUTH_NAME=Keycloak\nGF_AUTH_GENERIC_OAUTH_CLIENT_ID=${bound:oidc-client:as}\nGF_AUTH_GENERIC_OAUTH_CLIENT_SECRET__FILE=/run/secrets/oidc-client\nGF_AUTH_GENERIC_OAUTH_SCOPES=openid email profile roles\nGF_AUTH_GENERIC_OAUTH_AUTH_URL=${bound:oidc-client:issuer}${bound:oidc-client:authorization-path}\nGF_AUTH_GENERIC_OAUTH_TOKEN_URL=${bound:oidc-client:issuer}${bound:oidc-client:token-path}\nGF_AUTH_GENERIC_OAUTH_API_URL=${bound:oidc-client:issuer}${bound:oidc-client:userinfo-path}\nGF_AUTH_GENERIC_OAUTH_ROLE_ATTRIBUTE_PATH=contains(roles[*], 'admin') && 'Admin' || contains(realm_access.roles[*], 'admin') && 'Admin' || 'Viewer'\nGF_AUTH_GENERIC_OAUTH_USE_PKCE=true\nGF_AUTH_GENERIC_OAUTH_ALLOW_SIGN_UP=true\nGF_AUTH_GENERIC_OAUTH_ALLOW_ASSIGN_GRAFANA_ADMIN=true\n" + "template": "files/oidc.env" }, { "id": "influxdb-datasources", @@ -91,7 +105,7 @@ "path": "${dir:influxdb-credential}/influxdb-api", "mode": "0400", "owner": "472:472", - "content": "${secret:influxdb-api}", + "template": "files/influxdb-api", "only-when-bound": "influxdb-api" }, { @@ -99,7 +113,7 @@ "type": "file", "path": "${dir:influxdb-datasources}/mesh-influxdb.yaml", "mode": "0644", - "content": "apiVersion: 1\n# Written by the mesh from grafana's influxdb-api binding, only while it is bound (novox/hq ADR 0330):\n# without influxdb, grafana runs with no mesh data source. grafana reads it at start. Its own name and\n# uid, so a data source somebody made in the UI is never overwritten, and read-only in the UI because\n# the mesh resets it. The password is read from the file the mesh delivers, never written here.\ndatasources:\n - name: InfluxDB (mesh)\n uid: mesh-influxdb-api\n type: influxdb\n access: proxy\n url: ${bound:influxdb-api:scheme}://${bound:influxdb-api:at}:${bound:influxdb-api:port}\n user: ${bound:influxdb-api:as}\n isDefault: false\n editable: false\n jsonData:\n dbName: ${bound:influxdb-api:bucket}\n httpMode: POST\n secureJsonData:\n password: $__file{/run/secrets/influxdb/influxdb-api}\n", + "template": "files/mesh-influxdb.yaml", "only-when-bound": "influxdb-api" }, { @@ -140,8 +154,9 @@ "type": "file", "path": "${dir:mesh-state}/config.json", "mode": "0600", - "content": "{\n \"user\": \"admin\",\n \"password\": \"${secret:admin}\"\n}\n", + "template": "files/config.json", "merge": "json", + "takes-config": ["user", "url"], "protected": ["password", "token"] } ], @@ -194,6 +209,11 @@ "MESH_GRAFANA_URL": "http://127.0.0.1:${port:3000}", "MESH_GRAFANA_CONFIG_FILE": "${dir:mesh-state}/config.json" } + }, + { + "name": "files", + "kind": "archive", + "from": "files" } ] } diff --git a/modules/grafana/test/manifest.test.ts b/modules/grafana/test/manifest.test.ts index b0516070..a6d362ce 100644 --- a/modules/grafana/test/manifest.test.ts +++ b/modules/grafana/test/manifest.test.ts @@ -7,11 +7,26 @@ import assert from "node:assert/strict"; import { readFileSync } from "node:fs"; type Requirement = string | { name: string; optional?: boolean }; -type Resource = { id: string; type: string; path?: string; content?: string; volumes?: string[]; "only-when-bound"?: string }; +type Resource = { id: string; type: string; path?: string; content?: string; template?: string; volumes?: string[]; "only-when-bound"?: string }; const manifest = JSON.parse(readFileSync(new URL("../module.json", import.meta.url), "utf8")) as { requires: Requirement[]; resources: Resource[]; + build: { artifacts: { name: string; kind: string; from?: string }[] }; }; +// A file's text is its template's, in the module's files/ (novox/hq ADR 0349 §2). +const text = (r: Resource): string => { + assert.ok(r.template, `${r.id} names no template`); + return readFileSync(new URL(`../${r.template}`, import.meta.url), "utf8"); +}; + +test("no file carries its text inline: each names its template, and the build packs files/ into one archive", () => { + for (const r of manifest.resources) { + assert.equal(r.content, undefined, `${r.id} keeps its text inline`); + if (r.type === "file") text(r); + } + const packs = manifest.build.artifacts.filter((a) => a.kind === "archive" && a.from === "files"); + assert.deepEqual(packs.map((a) => a.name), ["files"]); +}); const optional = (name: string) => manifest.requires.some((r) => typeof r !== "string" && r.name === name && r.optional === true); @@ -24,7 +39,7 @@ test("influxdb-api is optional, and route and oidc-client stay hard", () => { test("every file naming influxdb-api is delivered only while it is bound", () => { for (const r of manifest.resources) { - if (r.type === "file" && /\$\{(bound:influxdb-api:|secret:influxdb-api\})/.test(r.content ?? "")) { + if (r.type === "file" && /\$\{(bound:influxdb-api:|secret:influxdb-api\})/.test(text(r))) { assert.equal(r["only-when-bound"], "influxdb-api", `${r.id} names influxdb-api and is delivered while it is absent`); } } diff --git a/modules/hello-web/files/index.html b/modules/hello-web/files/index.html new file mode 100644 index 00000000..0d513fd5 --- /dev/null +++ b/modules/hello-web/files/index.html @@ -0,0 +1 @@ +hello from hello-web, routed by the mesh diff --git a/modules/hello-web/module.json b/modules/hello-web/module.json index 1811a1f1..44dfc471 100644 --- a/modules/hello-web/module.json +++ b/modules/hello-web/module.json @@ -38,7 +38,7 @@ "type": "file", "path": "${dir:state}/index.html", "mode": "0644", - "content": "hello from hello-web, routed by the mesh\n" + "template": "files/index.html" }, { "id": "net", @@ -70,6 +70,11 @@ "name": "server", "kind": "upstream", "from": "alpine@sha256:28bd5fe8b56d1bd048e5babf5b10710ebe0bae67db86916198a6eec434943f8b" + }, + { + "name": "files", + "kind": "archive", + "from": "files" } ] } diff --git a/modules/home-assistant/files/config.json b/modules/home-assistant/files/config.json new file mode 100644 index 00000000..0967ef42 --- /dev/null +++ b/modules/home-assistant/files/config.json @@ -0,0 +1 @@ +{} diff --git a/modules/home-assistant/files/provisions.env b/modules/home-assistant/files/provisions.env new file mode 100644 index 00000000..93aba0ca --- /dev/null +++ b/modules/home-assistant/files/provisions.env @@ -0,0 +1,4 @@ +MESH_HOMEASSISTANT_URL=http://127.0.0.1:${port:8123} +MESH_HOMEASSISTANT_TOKEN_FILE=${dir:mesh-state}/token +MESH_PROVISIONS_DIR=${dir:state} +MESH_WRITTEN_DIR=${dir:written} diff --git a/modules/home-assistant/module.json b/modules/home-assistant/module.json index d35ae18e..8aff932f 100644 --- a/modules/home-assistant/module.json +++ b/modules/home-assistant/module.json @@ -45,6 +45,14 @@ } ] }, + "config": { + "url": { + "type": "string", + "description": "The address the module's tools reach Home Assistant at, as a URL; unset, the code falls back to its environment, else this machine's Home Assistant on its port.", + "trusted": true, + "default": "" + } + }, "resources": [ { "id": "mesh-state", @@ -86,15 +94,16 @@ "type": "file", "path": "${dir:mesh-state}/config.json", "mode": "0600", - "content": "{}\n", - "merge": "json" + "template": "files/config.json", + "merge": "json", + "takes-config": ["url"] }, { "id": "provisions-env", "type": "file", "path": "${dir:state}/provisions.env", "mode": "0600", - "content": "MESH_HOMEASSISTANT_URL=http://127.0.0.1:${port:8123}\nMESH_HOMEASSISTANT_TOKEN_FILE=${dir:mesh-state}/token\nMESH_PROVISIONS_DIR=${dir:state}\nMESH_WRITTEN_DIR=${dir:written}\n" + "template": "files/provisions.env" }, { "id": "provisions", @@ -185,6 +194,11 @@ "MESH_HOMEASSISTANT_TOKEN_FILE": "${dir:mesh-state}/token", "MESH_HOMEASSISTANT_CONFIG_FILE": "${dir:mesh-state}/config.json" } + }, + { + "name": "files", + "kind": "archive", + "from": "files" } ] } diff --git a/modules/hostname/cmd/hostname-tools/hostsfile_test.go b/modules/hostname/cmd/hostname-tools/hostsfile_test.go index b8562b6f..f2811b0a 100644 --- a/modules/hostname/cmd/hostname-tools/hostsfile_test.go +++ b/modules/hostname/cmd/hostname-tools/hostsfile_test.go @@ -304,7 +304,7 @@ func TestTheMachinesNameIsTheOperatorsSetting(t *testing.T) { if r["path"] != "/etc/hostname" { continue } - if r["id"] != "name" || r["into"] != nil || r["content"] != "${setting:hostname}\n" { + if r["id"] != "name" || r["into"] != nil || templateText(t, r) != "${config:hostname}\n" { t.Errorf("/etc/hostname is not written whole from the hostname setting: %v", r) } return diff --git a/modules/hostname/cmd/hostname-tools/templates_test.go b/modules/hostname/cmd/hostname-tools/templates_test.go new file mode 100644 index 00000000..7d0718b5 --- /dev/null +++ b/modules/hostname/cmd/hostname-tools/templates_test.go @@ -0,0 +1,64 @@ +package main + +import ( + "encoding/json" + "os" + "path/filepath" + "testing" +) + +// The module's directory, where its manifest and its files/ are. +const moduleDir = "../.." + +// templateText is the text a file resource places on the machine: its template's, in the module's files/ +// (novox/hq ADR 0349 §2). A resource that names no template, or a template that is not there, fails the test. +func templateText(t *testing.T, r map[string]any) string { + t.Helper() + name, _ := r["template"].(string) + if name == "" { + t.Fatalf("resource %v names no template", r["id"]) + } + raw, err := os.ReadFile(filepath.Join(moduleDir, filepath.FromSlash(name))) + if err != nil { + t.Fatalf("resource %v names %s: %v", r["id"], name, err) + } + return string(raw) +} + +// No file resource carries its text inline: each names its template in files/, and the build packs files/ +// into the one archive the controller reads the templates back from (novox/hq ADR 0349 §2). +func TestEveryFileTakesItsTextFromATemplate(t *testing.T) { + raw, err := os.ReadFile(filepath.Join(moduleDir, "module.json")) + if err != nil { + t.Fatal(err) + } + var m struct { + Resources []map[string]any `json:"resources"` + Build struct { + Artifacts []map[string]any `json:"artifacts"` + } `json:"build"` + } + if err := json.Unmarshal(raw, &m); err != nil { + t.Fatal(err) + } + for _, r := range m.Resources { + if _, inline := r["content"]; inline { + t.Errorf("resource %v keeps its text inline: keep it in files/ and name it as the template", r["id"]) + } + if r["type"] == "file" { + templateText(t, r) + } + } + packs := 0 + for _, a := range m.Build.Artifacts { + if a["kind"] == "archive" && a["from"] == "files" { + packs++ + if a["name"] != "files" { + t.Errorf("the templates archive is named %v, not files", a["name"]) + } + } + } + if packs != 1 { + t.Errorf("the build packs files/ into %d archives, not one", packs) + } +} diff --git a/modules/hostname/files/hostname b/modules/hostname/files/hostname new file mode 100644 index 00000000..34c12a43 --- /dev/null +++ b/modules/hostname/files/hostname @@ -0,0 +1 @@ +${config:hostname} diff --git a/modules/hostname/files/hosts b/modules/hostname/files/hosts new file mode 100644 index 00000000..7c9c0735 --- /dev/null +++ b/modules/hostname/files/hosts @@ -0,0 +1,7 @@ +# The machine's own names (module hostname, novox/hq ADR 0199, ADR 0223). Every line outside this +# block is the operator's: kept across every push, changed through the node-hostname verbs add and +# remove, and given back when this module goes. The mesh's names are not here: the mesh's resolver +# answers them. +127.0.0.1 localhost +::1 localhost +127.0.1.1 ${machine:name} diff --git a/modules/hostname/module.json b/modules/hostname/module.json index bd605df0..e84e7be2 100644 --- a/modules/hostname/module.json +++ b/modules/hostname/module.json @@ -12,6 +12,14 @@ ] } ], + "config": { + "hostname": { + "type": "string", + "description": "The machine's host name, written to /etc/hostname: the operator's name for this machine, with no default (novox/hq ADR 0112).", + "trusted": true, + "required": true + } + }, "resources": [ { "id": "own", @@ -20,14 +28,14 @@ "mode": "0644", "into": "block", "at": "start", - "content": "# The machine's own names (module hostname, novox/hq ADR 0199, ADR 0223). Every line outside this\n# block is the operator's: kept across every push, changed through the node-hostname verbs add and\n# remove, and given back when this module goes. The mesh's names are not here: the mesh's resolver\n# answers them.\n127.0.0.1\tlocalhost\n::1\tlocalhost\n127.0.1.1\t${machine:name}\n" + "template": "files/hosts" }, { "id": "name", "type": "file", "path": "/etc/hostname", "mode": "0644", - "content": "${setting:hostname}\n" + "template": "files/hostname" } ], "build": { @@ -42,6 +50,11 @@ "loads": [ "hostname-tools" ] + }, + { + "name": "files", + "kind": "archive", + "from": "files" } ] } diff --git a/modules/i3/README.md b/modules/i3/README.md index b1657c82..5903d847 100644 --- a/modules/i3/README.md +++ b/modules/i3/README.md @@ -18,7 +18,7 @@ The window manager as a module (novox/hq ADR 0208, research 026, to-be 42 phase |---|---|---| | `~/.config/i3/` | owned directory | its contents other than `config` are found and kept | | `~/.config/i3/config.d/` | owned directory | other modules' drop-ins, and yours | -| `~/.config/i3/config` | owned (the found file kept once) | the main configuration, from [`config/config`](config/config) | +| `~/.config/i3/config` | owned (the found file kept once) | the main configuration, from [`files/config`](files/config) | | `/etc/lemurs/wms/i3` | owned | the login manager's session entry: `exec systemd-cat -t x-session /bin/sh "$HOME/.xinitrc"`, the session's output in the journal (`journalctl -t x-session`) because the login manager's pipe has no reader | **Drop-ins.** Another module adds to i3 with its own `~/.config/i3/config.d/-.conf`. The diff --git a/modules/i3/cmd/i3-tools/manifest_test.go b/modules/i3/cmd/i3-tools/manifest_test.go index d0f8afc3..366a6723 100644 --- a/modules/i3/cmd/i3-tools/manifest_test.go +++ b/modules/i3/cmd/i3-tools/manifest_test.go @@ -28,6 +28,11 @@ type manifest struct { Environment struct { Variables map[string]string `json:"variables"` } `json:"environment"` + Items []struct { + Collection string `json:"collection"` + Attributes map[string]string `json:"attributes"` + Template string `json:"template"` + } `json:"items"` Shell []struct { For, Slot, Code string } `json:"shell"` @@ -46,6 +51,23 @@ func readManifest(t *testing.T) manifest { if err := json.Unmarshal(raw, &m); err != nil { t.Fatal(err) } + // The module's code and variables are the templates of its items (design 57 L3, issues 795 and 807). + for _, it := range m.Items { + text, err := os.ReadFile(filepath.Join("..", "..", it.Template)) + if err != nil { + t.Fatal(err) + } + parts := strings.Split(it.Collection, ":") + switch { + case len(parts) == 3 && parts[0] == "shell": + m.Shell = append(m.Shell, struct{ For, Slot, Code string }{parts[1], parts[2], string(text)}) + case it.Collection == "environment" && it.Attributes["kind"] == "variable": + if m.Environment.Variables == nil { + m.Environment.Variables = map[string]string{} + } + m.Environment.Variables[it.Attributes["name"]] = strings.TrimSuffix(string(text), "\n") + } + } return m } @@ -108,9 +130,10 @@ func TestItContributesTheSessionsExecLastAndTheDesktopsIdentity(t *testing.T) { func TestTheConfigurationIsTheModulesFileImprovedAndEndsWithTheDropIns(t *testing.T) { m := readManifest(t) r := resource(t, m, "config") - raw, _ := os.ReadFile(filepath.Join("..", "..", "config", "config")) - if r["content"] != string(raw) || r["path"] != "${machine:account-home}/.config/i3/config" || r["into"] != nil { - t.Fatal("the manifest carries config/config whole, as an owned file") + raw, _ := os.ReadFile(filepath.Join("..", "..", "files", "config")) + if _, inline := r["content"]; inline || r["template"] != "files/config" || len(raw) == 0 || + r["path"] != "${machine:account-home}/.config/i3/config" || r["into"] != nil { + t.Fatal("the manifest takes files/config whole as its template, as an owned file") } c := string(raw) lines := strings.Split(strings.TrimSpace(c), "\n") @@ -159,7 +182,11 @@ func TestTheLoginManagersEntryRunsTheSessionsStart(t *testing.T) { if r["path"] != "/etc/lemurs/wms/i3" || r["mode"] != "0755" { t.Fatalf("%v", r) } - if !strings.Contains(r["content"].(string), `exec systemd-cat -t x-session /bin/sh "$HOME/.xinitrc"`) { + if r["template"] != "files/wms/i3" { + t.Fatalf("the entry's text is not its template files/wms/i3: %v", r["template"]) + } + entry, _ := os.ReadFile(filepath.Join("..", "..", "files", "wms", "i3")) + if !strings.Contains(string(entry), `exec systemd-cat -t x-session /bin/sh "$HOME/.xinitrc"`) { t.Fatal("the entry runs the session's start, never i3 bare") } pkgs := map[string]bool{} @@ -219,7 +246,7 @@ func TestTheMainFileAndEveryModulesContributionLoadTogether(t *testing.T) { if len(found) < 5 { t.Fatalf("expected the launcher, clipboard, wallpaper, bars and the laptop to contribute; found %v", found) } - main, _ := os.ReadFile(filepath.Join("..", "..", "config", "config")) + main, _ := os.ReadFile(filepath.Join("..", "..", "files", "config")) text := strings.Replace(string(main), "${contribution:node-display-session:config}", placed.String(), 1) // The operator's own files are not the catalogue's: the include line itself goes (a contribution may // name it in a comment, so only the line that is exactly it). @@ -230,3 +257,31 @@ func TestTheMainFileAndEveryModulesContributionLoadTogether(t *testing.T) { t.Fatalf("with the contributions of %v: %v\n%s", found, err, out) } } + +// No file resource carries its text inline: each names its template in files/, and the build packs files/ +// into the one archive the controller reads the templates back from (novox/hq ADR 0349 §2). +func TestEveryFileTakesItsTextFromATemplate(t *testing.T) { + m := readManifest(t) + for _, r := range m.Resources { + if _, inline := r["content"]; inline { + t.Errorf("resource %v keeps its text inline: keep it in files/ and name it as the template", r["id"]) + } + if name, ok := r["template"].(string); ok { + if _, err := os.Stat(filepath.Join("..", "..", name)); err != nil { + t.Errorf("resource %v names %s: %v", r["id"], name, err) + } + } + } + packs := 0 + for _, a := range m.Build.Artifacts { + if a["kind"] == "archive" && a["from"] == "files" { + packs++ + if a["name"] != "files" { + t.Errorf("the templates archive is named %v, not files", a["name"]) + } + } + } + if packs != 1 { + t.Errorf("the build packs files/ into %d archives, not one", packs) + } +} diff --git a/modules/i3/config/config b/modules/i3/files/config similarity index 100% rename from modules/i3/config/config rename to modules/i3/files/config diff --git a/modules/i3/files/env-xdg-current-desktop b/modules/i3/files/env-xdg-current-desktop new file mode 100644 index 00000000..04ad29f0 --- /dev/null +++ b/modules/i3/files/env-xdg-current-desktop @@ -0,0 +1 @@ +i3 diff --git a/modules/i3/files/env-xdg-session-desktop b/modules/i3/files/env-xdg-session-desktop new file mode 100644 index 00000000..04ad29f0 --- /dev/null +++ b/modules/i3/files/env-xdg-session-desktop @@ -0,0 +1 @@ +i3 diff --git a/modules/i3/files/wms/i3 b/modules/i3/files/wms/i3 new file mode 100755 index 00000000..4e269ad2 --- /dev/null +++ b/modules/i3/files/wms/i3 @@ -0,0 +1,9 @@ +#!/bin/sh +# The i3 session (module i3, novox/hq ADR 0208), offered by the login manager. It runs the session's +# start, ~/.xinitrc, which the display server's module writes and which ends by starting i3. +# Replaced at every push. +# +# The login manager hands the session a stdout and stderr whose reading end nobody holds, so a +# program that writes to them dies of EPIPE (an Electron tray app shows it as an error dialog). +# The session's output goes to the journal instead, under `journalctl -t x-session`. +exec systemd-cat -t x-session /bin/sh "$HOME/.xinitrc" diff --git a/modules/i3/files/xinitrc-last b/modules/i3/files/xinitrc-last new file mode 100644 index 00000000..e84ff3d7 --- /dev/null +++ b/modules/i3/files/xinitrc-last @@ -0,0 +1,2 @@ +# The session: i3, with 25 MiB of debug log kept in memory for i3-dump-log. +exec i3 --shmlog-size=26214400 diff --git a/modules/i3/module.json b/modules/i3/module.json index 5dee0726..29fc3e85 100644 --- a/modules/i3/module.json +++ b/modules/i3/module.json @@ -1,6 +1,7 @@ { "module": "i3", "version": "1", + "moved": true, "capabilities": [ "package-manager" ], @@ -30,17 +31,29 @@ "i3_marks", "i3_scratchpad" ], - "environment": { - "variables": { - "XDG_CURRENT_DESKTOP": "i3", - "XDG_SESSION_DESKTOP": "i3" - } - }, - "shell": [ + "items": [ { - "for": "xinitrc", - "slot": "last", - "code": "# The session: i3, with 25 MiB of debug log kept in memory for i3-dump-log.\nexec i3 --shmlog-size=26214400\n" + "collection": "environment", + "name": "xdg-current-desktop", + "attributes": { + "kind": "variable", + "name": "XDG_CURRENT_DESKTOP" + }, + "template": "files/env-xdg-current-desktop" + }, + { + "collection": "environment", + "name": "xdg-session-desktop", + "attributes": { + "kind": "variable", + "name": "XDG_SESSION_DESKTOP" + }, + "template": "files/env-xdg-session-desktop" + }, + { + "collection": "shell:xinitrc:last", + "name": "xinitrc-last", + "template": "files/xinitrc-last" } ], "resources": [ @@ -74,14 +87,14 @@ "path": "${machine:account-home}/.config/i3/config", "owner": "${machine:account}", "mode": "0644", - "content": "# i3 config file (v4), written by the mesh (module i3, novox/hq ADR 0208). Replaced at every push;\n# change the module instead. i3's user guide is the reference.\n#\n# Other modules add to this configuration as contributions to node-display-session (novox/hq ADR\n# 0212): the mesh places their lines near the end, each module's under a line naming it, where every\n# variable set here ($mod, $ws1 … $ws10) is in scope. A file of yours in ~/.config/i3/config.d/ is read\n# after them, by the include at the very end, and is yours.\n#\n# The reload watcher of this module reloads i3 when this file or a drop-in changes, after checking the\n# result with i3 -C; it never reloads into a configuration with errors.\n\n# Font for window titles, and the bars below: the mesh's monospace face (research 026/04).\nfont pango:JetBrainsMono Nerd Font 11\n\n# XDG autostart entries (~/.config/autostart, /etc/xdg/autostart), started once at login.\nexec --no-startup-id dex --autostart --environment i3\n\n#########################################\n###### Keys ####\n#########################################\n# To find key symbols: xmodmap -pke / xmodmap -pm\nset $mod Mod4\nset $alt Mod1\nset $shift Shift\nset $ctrl Control\n\n# use these keys for focus, movement, and resize directions when reaching for\n# the arrows is not convenient\nset $left h\nset $down j\nset $up k\nset $right l\n\n# use Mouse+$mod to drag floating windows to their wanted position\nfloating_modifier $mod\n\n# move tiling windows via drag & drop by left-clicking into the title bar,\n# or left-clicking anywhere into the window while holding the floating modifier.\ntiling_drag modifier titlebar\n\n# start a terminal: whichever the terminal module names in $TERMINAL\nbindsym $mod+Return exec i3-sensible-terminal\n\n# kill focused window\nbindsym $mod+$shift+q kill\n\n# change focus\nbindsym $mod+$left focus left\nbindsym $mod+$down focus down\nbindsym $mod+$up focus up\nbindsym $mod+$right focus right\n\nbindsym $mod+Left focus left\nbindsym $mod+Down focus down\nbindsym $mod+Up focus up\nbindsym $mod+Right focus right\n\n# move focused window\nbindsym $mod+$shift+$left move left\nbindsym $mod+$shift+$down move down\nbindsym $mod+$shift+$up move up\nbindsym $mod+$shift+$right move right\n\nbindsym $mod+$shift+Left move left\nbindsym $mod+$shift+Down move down\nbindsym $mod+$shift+Up move up\nbindsym $mod+$shift+Right move right\n\n# split in horizontal orientation\nbindsym $mod+c split h\n# split in vertical orientation\nbindsym $mod+v split v\n\n# enter fullscreen mode for the focused container\nbindsym $mod+f fullscreen toggle\n\n# change container layout (stacked, tabbed, toggle split)\nbindsym $mod+s layout stacking\nbindsym $mod+w layout tabbed\nbindsym $mod+e layout toggle split\n\n# toggle tiling / floating\nbindsym $mod+$shift+space floating toggle\n\n# change focus between tiling / floating windows\nbindsym $mod+space focus mode_toggle\n\n# focus the parent container\nbindsym $mod+a focus parent\n\n# alt-tab functionality\nbindsym $mod+Tab workspace back_and_forth\n\n#########################################\n###### Workspace mgmt ####\n#########################################\nset $ws1 \"1\"\nset $ws2 \"2\"\nset $ws3 \"3\"\nset $ws4 \"4\"\nset $ws5 \"5\"\nset $ws6 \"6\"\nset $ws7 \"7\"\nset $ws8 \"8\"\nset $ws9 \"9\"\nset $ws10 \"10\"\n\nbindsym $mod+1 workspace number $ws1\nbindsym $mod+2 workspace number $ws2\nbindsym $mod+3 workspace number $ws3\nbindsym $mod+4 workspace number $ws4\nbindsym $mod+5 workspace number $ws5\nbindsym $mod+6 workspace number $ws6\nbindsym $mod+7 workspace number $ws7\nbindsym $mod+8 workspace number $ws8\nbindsym $mod+9 workspace number $ws9\nbindsym $mod+0 workspace number $ws10\n\nbindsym $mod+$shift+1 move container to workspace number $ws1\nbindsym $mod+$shift+2 move container to workspace number $ws2\nbindsym $mod+$shift+3 move container to workspace number $ws3\nbindsym $mod+$shift+4 move container to workspace number $ws4\nbindsym $mod+$shift+5 move container to workspace number $ws5\nbindsym $mod+$shift+6 move container to workspace number $ws6\nbindsym $mod+$shift+7 move container to workspace number $ws7\nbindsym $mod+$shift+8 move container to workspace number $ws8\nbindsym $mod+$shift+9 move container to workspace number $ws9\nbindsym $mod+$shift+0 move container to workspace number $ws10\n\n#########################################\n###### Window mgmt ####\n#########################################\nset $resize_px 10 px\nbindsym $mod+$ctrl+$left resize shrink width $resize_px\nbindsym $mod+$ctrl+$down resize grow height $resize_px\nbindsym $mod+$ctrl+$up resize shrink height $resize_px\nbindsym $mod+$ctrl+$right resize grow width $resize_px\n\n#########################################\n###### Session mgmt ####\n#########################################\nbindsym $mod+$shift+e exec \"i3-nagbar -t warning -m 'You pressed the exit shortcut. Do you really want to exit i3? This will end your X session.' -B 'Yes, exit i3' 'i3-msg exit'\"\n\n# Lock the screen through logind, so the one locker the lock screen's module runs handles it (and\n# suspend and lid close too).\nbindsym $mod+Delete exec --no-startup-id loginctl lock-session\n\n# reload the configuration file\nbindsym $mod+$shift+c reload\n\n# restart i3 inplace (preserves your layout/session, can be used to upgrade i3)\nbindsym $mod+$shift+r restart\n\n#########################################\n###### Borders ####\n#########################################\ndefault_border pixel 1\nsmart_borders on\n\n#########################################\n###### Gaps ####\n#########################################\ngaps inner 0\ngaps outer 0\n\n# Only the accent-bearing slots are themed; background and text keep i3's own defaults. Borders are\n# `pixel`, so no title bar shows: the colour says which window has focus.\n# border background text indicator child_border\nclient.focused #de5200 #de5200 #1E2127 #de5200 #de5200\nclient.urgent #900000 #900000 #ffffff #900000 #900000\n\n#########################################\n###### Until their modules carry them ##\n#########################################\n# Each line below belongs to something other than i3, named on its line. When that module is written\n# it contributes the line to node-display-session, and the line goes from here in the same change.\n# The launcher, the clipboard, the wallpaper, the bars, a machine model's keys and the peripherals'\n# tray already contribute theirs (rofi, clipmenu, feh, i3status-rust, asus-zephyrus-g14,\n# polychromatic).\n\n# the operator's scripts: volume, games volume, sessions, screenshot\nbindsym XF86AudioRaiseVolume exec --no-startup-id volume-notify up\nbindsym XF86AudioLowerVolume exec --no-startup-id volume-notify down\nbindsym XF86AudioMute exec --no-startup-id volume-notify mute\nbindsym XF86AudioMicMute exec --no-startup-id mic-notify\nbindsym $ctrl+XF86AudioRaiseVolume exec --no-startup-id set-games-volume 5\nbindsym $ctrl+XF86AudioLowerVolume exec --no-startup-id set-games-volume -5\nbindsym $mod+$shift+Return exec --no-startup-id ~/scripts/i3-sessions/launcher.sh\nbindsym --release $ctrl+$shift+x exec --no-startup-id $XDG_CONFIG_HOME/i3/scripts/screenshot.sh\n\n#########################################\n###### Other modules' lines ####\n#########################################\n# Placed by the mesh from every other module's contribution (novox/hq ADR 0212): the launcher, the\n# clipboard, the wallpaper, the bars, a machine model's keys, the peripherals' tray. Each module's\n# under a line naming it.\n${contribution:node-display-session:config}\n#########################################\n###### Your own files ####\n#########################################\ninclude ~/.config/i3/config.d/*.conf\n" + "template": "files/config" }, { "id": "session", "type": "file", "path": "/etc/lemurs/wms/i3", "mode": "0755", - "content": "#!/bin/sh\n# The i3 session (module i3, novox/hq ADR 0208), offered by the login manager. It runs the session's\n# start, ~/.xinitrc, which the display server's module writes and which ends by starting i3.\n# Replaced at every push.\n#\n# The login manager hands the session a stdout and stderr whose reading end nobody holds, so a\n# program that writes to them dies of EPIPE (an Electron tray app shows it as an error dialog).\n# The session's output goes to the journal instead, under `journalctl -t x-session`.\nexec systemd-cat -t x-session /bin/sh \"$HOME/.xinitrc\"\n" + "template": "files/wms/i3" } ], "build": { @@ -96,6 +109,11 @@ "loads": [ "i3-tools" ] + }, + { + "name": "files", + "kind": "archive", + "from": "files" } ] } diff --git a/modules/i3status-rust/README.md b/modules/i3status-rust/README.md index f8e94ce0..8d3c9dda 100644 --- a/modules/i3status-rust/README.md +++ b/modules/i3status-rust/README.md @@ -3,7 +3,7 @@ The bars as a module (novox/hq ADR 0208, research 026/05). - Installs `i3status-rust`. The update block's `checkupdates` comes from `pacman-contrib`, which the `pacman` module installs on every node; two modules declaring one package is refused at composition. - Claims the mesh's `node-bar` seat (no verbs yet, ADR 0208 §2) and renders the blocks other modules + Claims the mesh's `node-bar` seat (no verbs yet, ADR 0208 §2) and places the blocks other modules contribute to it (ADR 0255, below). Requires `x11-display` on its own machine: its bars are i3bar's. - Owns `~/.config/i3status-rust/`, both bars (`top-bar.toml`, `bottom-bar.toml`) and their icon set @@ -29,18 +29,19 @@ The bars as a module (novox/hq ADR 0208, research 026/05). ## Blocks other modules contribute (novox/hq ADR 0255) A block that depends on the machine's hardware is not written here. The module that knows the hardware -contributes it to `node-bar` as data that says what it shows, and this module renders it: +contributes it as text, in this module's grammar, to the bar's collection of blocks: -- **What a contribution says:** which bar (`bottom`, `top`), which place (`resources`, beside the - processor, memory and disks; `status`, beside the sound), an order from 0 to 99, what it shows - (`battery`, `command`) and that kind's options. Nothing in it is i3status-rust's. -- **How it is rendered:** the claim's `renders.block` is a template (its source is `files/block.tmpl`) - that turns one piece into a `[[block]]`. The controller runs it and places the result where each bar - says `${contribution:node-bar:block:bar=…,place=…}`. Each bar places both of its places once; the - catalogue check refuses a holder that misses one. -- **Where it applies:** a contribution may name a capability the machine must report. The `power` - module's battery block names `battery` and its power-draw block names `power-meter`, so the laptop's - bar gets both and a desktop only the power draw. +- **What a contributor gives:** an item of the collection `contribution:node-bar:block`, the block text in + i3status-rust's grammar (a `[[block]]`), with the attributes `bar` (`bottom`, `top`), `place` (`resources`, + beside the processor, memory and disks; `status`, beside the sound) and `order` (0 to 99), and, where the + block depends on the hardware, the capability the machine must report. +- **How it is placed:** each bar says `${collected:contribution:node-bar:block:bar=…,place=…}`, where the node's + provisioner client fills the items that carry those attributes, in order, each under a line naming its + contributor. Each bar places both of its places once. Where the client does not fill the file, the controller + fills the same text the same way. +- **Where it applies:** an item may name a capability the machine must report. The `power` module's battery item + names `battery` and its power-draw item names `power-meter`, so the laptop's bar gets both and a desktop only + the power draw. - **A value from the bus:** a block that `shows: state` names another module's state, such as `power.draw`. The mesh grants this module the read, because a module on its machine offers it; the manifest names no other module. This module's bundle watches it and leaves the value's text in the diff --git a/modules/i3status-rust/cmd/i3status-rust-tools/blocks_test.go b/modules/i3status-rust/cmd/i3status-rust-tools/blocks_test.go index 8314ce40..05c068e2 100644 --- a/modules/i3status-rust/cmd/i3status-rust-tools/blocks_test.go +++ b/modules/i3status-rust/cmd/i3status-rust-tools/blocks_test.go @@ -1,87 +1,27 @@ package main import ( - "bytes" - "encoding/json" - "fmt" "os" "path/filepath" "strings" "testing" - "text/template" ) -// i3status-rust renders the blocks other modules contribute (novox/hq ADR 0255): a bar-neutral piece -// — what it shows, on which bar, where — in its own grammar, through the template its claim gives. The -// controller runs the template; these tests run it the same way, with the one function it is given. - -func blockTemplate(t *testing.T) *template.Template { - t.Helper() - m := readManifest(t) - src, err := os.ReadFile("../../files/block.tmpl") - if err != nil { - t.Fatal(err) - } - if m.Claims[0].Renders["block"] != string(src) { - t.Fatal("the claim's template is not files/block.tmpl") - } - quote := func(v any) (string, error) { - b, err := json.Marshal(fmt.Sprint(v)) - return string(b), err - } - return template.Must(template.New("block").Funcs(template.FuncMap{"quote": quote}).Parse(string(src))) -} - -func render(t *testing.T, tmpl *template.Template, piece string) string { - t.Helper() - view := map[string]any{"module": "power", "machine": "laptop"} - if err := json.Unmarshal([]byte(piece), &view); err != nil { - t.Fatal(err) - } - var b bytes.Buffer - if err := tmpl.Execute(&b, view); err != nil { - t.Fatal(err) - } - if strings.Contains(b.String(), "") { - t.Fatalf("%s rendered a missing value: %q", piece, b.String()) - } - return b.String() -} - -func TestAContributedBatteryIsI3statusRustsBatteryBlock(t *testing.T) { - tmpl := blockTemplate(t) - got := render(t, tmpl, `{"bar":"bottom","place":"status","shows":"battery"}`) - if !strings.HasPrefix(got, "[[block]]\nblock = \"battery\"\nformat = ") || strings.Contains(got, "device") || - !strings.Contains(got, `missing_format = ""`) || !strings.HasSuffix(got, "\n\n") { - t.Fatalf("%q", got) - } - got = render(t, tmpl, `{"bar":"bottom","place":"status","shows":"battery","options":{"device":"BAT1"}}`) - if !strings.Contains(got, "block = \"battery\"\ndevice = \"BAT1\"\nformat = ") { - t.Fatalf("%q", got) - } -} - -func TestAContributedCommandIsACustomBlockQuoted(t *testing.T) { - got := render(t, blockTemplate(t), - `{"bar":"bottom","place":"status","shows":"command","options":{"command":"say \"hi\"","interval":5}}`) - want := "[[block]]\nblock = \"custom\"\ncommand = \"say \\\"hi\\\"\"\ninterval = 5\n\n" - if got != want { - t.Fatalf("%q, want %q", got, want) - } -} +// The blocks other modules contribute are a collection of text (design 57 L3, novox/hq ADR 0349 clause 13): each +// contributor gives its block in i3status-rust's grammar, and the bars place the collection where they want it. func TestEachBarPlacesBothPlacesOnce(t *testing.T) { m := readManifest(t) for _, bar := range []string{"bottom", "top"} { - content := m.resource(t, bar+"-bar")["content"].(string) + content := m.text(t, bar+"-bar") for _, place := range []string{"resources", "status"} { - ph := "${contribution:node-bar:block:bar=" + bar + ",place=" + place + "}" + ph := "${collected:contribution:node-bar:block:bar=" + bar + ",place=" + place + "}" if strings.Count(content, ph) != 1 { t.Errorf("the %s bar places %s %d times", bar, ph, strings.Count(content, ph)) } } } - bottom := m.resource(t, "bottom-bar")["content"].(string) + bottom := m.text(t, "bottom-bar") resources := strings.Index(bottom, "place=resources") if !(strings.Index(bottom, `block = "memory"`) < resources && resources < strings.Index(bottom, `block = "docker"`)) || !(strings.Index(bottom, `block = "sound"`) < strings.Index(bottom, "place=status") && @@ -91,30 +31,33 @@ func TestEachBarPlacesBothPlacesOnce(t *testing.T) { } func TestTheMemoryIsWhatProgramsHoldNotTheCache(t *testing.T) { - bottom := readManifest(t).resource(t, "bottom-bar")["content"].(string) + bottom := readManifest(t).text(t, "bottom-bar") if !strings.Contains(bottom, `format = "$icon $mem_used.eng(w:2) ($mem_used_percents.eng(w:2))"`) || strings.Contains(bottom, "mem_total_used") { t.Fatal("the memory block counts the cache as used") } } -func TestAContributedStateIsACustomBlockReadingThisMachinesValue(t *testing.T) { - tmpl := blockTemplate(t) - got := render(t, tmpl, `{"bar":"bottom","place":"status","shows":"state","options":{"state":"power.draw"}}`) - want := "[[block]]\nblock = \"custom\"\ncommand = \"~/.local/bin/i3status-value power.draw laptop\"\ninterval = 5\nhide_when_empty = true\n\n" - if got != want { - t.Fatalf("%q, want %q", got, want) +func TestABarNamesNoTemplateOfItsOwnForTheBlocksItPlaces(t *testing.T) { + m := readManifest(t) + for _, id := range []string{"top-bar", "bottom-bar"} { + if strings.Contains(m.text(t, id), "${contribution:") { + t.Errorf("the %s still places a contribution the old way", id) + } } - got = render(t, tmpl, `{"bar":"bottom","place":"status","shows":"state","options":{"state":"power.draw","key":"desk"}}`) - if !strings.Contains(got, "i3status-value power.draw desk") { - t.Fatalf("%q", got) + if _, err := os.Stat("../../files/block.tmpl"); err == nil { + t.Error("files/block.tmpl is still there, and no claim renders it") } - // And the bundle finds what it rendered. +} + +func TestTheBundleFindsTheValueARenderedBlockShows(t *testing.T) { + // What the power module's block renders on a machine called laptop. + got := "[[block]]\nblock = \"custom\"\ncommand = \"~/.local/bin/i3status-value power.draw laptop\"\ninterval = 5\nhide_when_empty = true\n\n" f := filepath.Join(t.TempDir(), "bottom-bar.toml") if err := os.WriteFile(f, []byte(got), 0o644); err != nil { t.Fatal(err) } - if w := wantedIn([]string{f}); len(w) != 1 || w[0] != (Wanted{"power.draw", "desk"}) { + if w := wantedIn([]string{f}); len(w) != 1 || w[0] != (Wanted{"power.draw", "laptop"}) { t.Fatalf("%v", w) } } diff --git a/modules/i3status-rust/cmd/i3status-rust-tools/manifest_helpers_test.go b/modules/i3status-rust/cmd/i3status-rust-tools/manifest_helpers_test.go index 5c76cde9..c33db7ec 100644 --- a/modules/i3status-rust/cmd/i3status-rust-tools/manifest_helpers_test.go +++ b/modules/i3status-rust/cmd/i3status-rust-tools/manifest_helpers_test.go @@ -14,6 +14,8 @@ import ( type manifest struct { Module string `json:"module"` Version string `json:"version"` + Moved bool `json:"moved"` + Items []item `json:"items"` Capabilities []string `json:"capabilities"` Requires []string `json:"requires"` Claims []claim `json:"claims"` @@ -23,7 +25,7 @@ type manifest struct { Shell []shellCode `json:"shell"` Resources []map[string]any `json:"resources"` // Lines for other modules' seats (novox/hq ADR 0212): the window manager's, here. - Contributions []contribution `json:"contributions"` + Contributions []any `json:"contributions"` Build struct { Artifacts []map[string]any `json:"artifacts"` } `json:"build"` @@ -48,6 +50,53 @@ type shellCode struct { Code string `json:"code"` } +// An item of a collection (novox/hq ADR 0349 clause 13): its text is the template file beside the manifest. +type item struct { + Collection string `json:"collection"` + Name string `json:"name"` + Attributes map[string]string `json:"attributes"` + Template string `json:"template"` +} + +// shell is the code the module adds to a shell's or the session's start, read from the templates of its items of +// the collections shell::, in the order they are declared. +func (m manifest) shell(t *testing.T) []shellCode { + t.Helper() + var out []shellCode + for _, it := range m.Items { + parts := strings.Split(it.Collection, ":") + if len(parts) != 3 || parts[0] != "shell" { + continue + } + text, err := os.ReadFile(filepath.Join("..", "..", it.Template)) + if err != nil { + t.Fatal(err) + } + out = append(out, shellCode{For: parts[1], Slot: parts[2], Code: string(text)}) + } + return out +} + +// environmentOf is the variables the module sets, read from the templates of its items of the environment. +func (m manifest) environmentOf(t *testing.T) *environment { + t.Helper() + var env *environment + for _, it := range m.Items { + if it.Collection != "environment" || it.Attributes["kind"] != "variable" { + continue + } + text, err := os.ReadFile(filepath.Join("..", "..", it.Template)) + if err != nil { + t.Fatal(err) + } + if env == nil { + env = &environment{Variables: map[string]string{}} + } + env.Variables[it.Attributes["name"]] = strings.TrimSuffix(string(text), "\n") + } + return env +} + func readManifest(t *testing.T) manifest { t.Helper() raw, err := os.ReadFile(filepath.Join("..", "..", "module.json")) @@ -60,6 +109,7 @@ func readManifest(t *testing.T) manifest { if err := dec.Decode(&m); err != nil { t.Fatalf("module.json: %v", err) } + m.Environment = m.environmentOf(t) return m } @@ -87,26 +137,66 @@ func (m manifest) packages() (present, absent []string) { return present, absent } -// sameAsSource checks that a file resource's content is byte for byte the module's source file, so -// the readable file in the repository is what the machine gets. -func (m manifest) sameAsSource(t *testing.T, id, source string) { +// fromTemplate checks that a file resource takes its text from its template file in the module's +// repository (novox/hq ADR 0349 §2): it names the template, carries no text of its own beside it, and the +// template is there. The readable file in the repository is the one place the machine's text is written. +func (m manifest) fromTemplate(t *testing.T, id, template string) { t.Helper() - want, err := os.ReadFile(filepath.Join("..", "..", source)) - if err != nil { - t.Fatal(err) - } r := m.resource(t, id) if r["type"] != "file" { t.Fatalf("%s is a %v, not a file", id, r["type"]) } - if got, _ := r["content"].(string); got != string(want) { - t.Fatalf("resource %s's content is not %s: edit the source and copy it into module.json", id, source) + if r["template"] != template { + t.Fatalf("resource %s does not take its text from %s: template %v", id, template, r["template"]) + } + if _, inline := r["content"]; inline { + t.Fatalf("resource %s keeps its text inline beside its template: a file's text is in one place", id) + } + if _, err := os.Stat(filepath.Join("..", "..", template)); err != nil { + t.Fatalf("resource %s names %s: %v", id, template, err) } if r["owner"] != "${machine:account}" && !strings.HasPrefix(r["path"].(string), "/etc/") { t.Fatalf("%s under the home is the account's", id) } } +// everyFileIsATemplate checks that no file resource carries its text inline, and that the build packs +// files/ into the one archive the controller reads the templates back from (novox/hq ADR 0349 §2). +func (m manifest) everyFileIsATemplate(t *testing.T) { + t.Helper() + for _, r := range m.Resources { + if _, inline := r["content"]; inline { + t.Errorf("resource %v keeps its text inline: keep it in files/ and name it as the template", r["id"]) + } + } + packs := 0 + for _, a := range m.Build.Artifacts { + if a["kind"] == "archive" && a["from"] == "files" { + packs++ + if a["name"] != "files" { + t.Errorf("the templates archive is named %v, not files", a["name"]) + } + } + } + if packs != 1 { + t.Errorf("the build packs files/ into %d archives, not one", packs) + } +} + +// text is the text a file resource places on the machine: its template file's. +func (m manifest) text(t *testing.T, id string) string { + t.Helper() + name, _ := m.resource(t, id)["template"].(string) + if name == "" { + t.Fatalf("resource %s names no template", id) + } + raw, err := os.ReadFile(filepath.Join("..", "..", name)) + if err != nil { + t.Fatal(err) + } + return string(raw) +} + // checkTheToolsAgree checks that the manifest lists the module's own tools exactly, that the bundle // serves each seat verb the claims promise as ., and that the Go bundle is declared. func checkTheToolsAgree(t *testing.T, m manifest) { @@ -178,31 +268,42 @@ func checkNoSecretsOrInstallationNames(t *testing.T) { } } -type contribution struct { - Seat string `json:"seat"` - Kind string `json:"kind"` - Content string `json:"content"` -} +// displaySession is the collection the module's lines for the window manager are an item of. +const displaySession = "contribution:node-display-session:config" -// i3Lines is what the module contributes to the window manager. +// i3Lines is what the module contributes to the window manager: the text of its item of the collection +// contribution:node-display-session:config (design 57 L3, issue 848). func (m manifest) i3Lines() string { var out string - for _, c := range m.Contributions { - if c.Seat == "node-display-session" && c.Kind == "config" { - out += c.Content + for _, it := range m.Items { + if it.Collection == displaySession { + text, err := os.ReadFile(filepath.Join("..", "..", it.Template)) + if err != nil { + return "" + } + out += string(text) } } return out } -// i3LinesAreSource checks the window-manager contribution is the source file it is written from. +// i3LinesAreSource checks the window-manager lines are one item whose template is the source file, and that no +// old-form contribution to node-display-session is left. func (m manifest) i3LinesAreSource(t *testing.T, source string) { t.Helper() - want, err := os.ReadFile(filepath.Join("..", "..", source)) - if err != nil { - t.Fatal(err) + if len(m.Contributions) != 0 { + t.Fatalf("the module still holds contributions: the lines are an item of %s", displaySession) } - if m.i3Lines() != string(want) { - t.Fatalf("the contribution to node-display-session is not %s: edit the source and copy it into module.json", source) + var templates []string + for _, it := range m.Items { + if it.Collection == displaySession { + templates = append(templates, it.Template) + } + } + if len(templates) != 1 || templates[0] != source { + t.Fatalf("the items of %s are %v, want the one template %s", displaySession, templates, source) + } + if m.i3Lines() == "" { + t.Fatalf("the template %s holds no line", source) } } diff --git a/modules/i3status-rust/cmd/i3status-rust-tools/manifest_test.go b/modules/i3status-rust/cmd/i3status-rust-tools/manifest_test.go index ec41d8db..e4811c48 100644 --- a/modules/i3status-rust/cmd/i3status-rust-tools/manifest_test.go +++ b/modules/i3status-rust/cmd/i3status-rust-tools/manifest_test.go @@ -13,11 +13,14 @@ import ( func TestItClaimsTheBarSeatAndRequiresTheXDisplay(t *testing.T) { m := readManifest(t) + if !m.Moved { + t.Fatal("the module is not marked moved") + } if m.Module != "i3status-rust" || m.Seats != nil { t.Fatalf("module %q declares seats %v", m.Module, m.Seats) } if len(m.Claims) != 1 || m.Claims[0].Name != "node-bar" || m.Claims[0].Scope != "node" || - len(m.Claims[0].Renders) != 1 || m.Claims[0].Renders["block"] == "" { + len(m.Claims[0].Renders) != 0 { t.Fatalf("claims: %+v", m.Claims) } if !reflect.DeepEqual(m.Requires, []string{"x11-display"}) { @@ -35,13 +38,13 @@ func TestItOwnsItsFilesAsWrittenInTheModule(t *testing.T) { "updates": "files/bin/i3status-updates", "watchdog": "files/bin/i3bar-watchdog", "value": "files/bin/i3status-value", } { - m.sameAsSource(t, id, src) + m.fromTemplate(t, id, src) } } func TestNoBlockFollowsAMachinesHardwareOrAPersonsDevicesOrCarriesAKey(t *testing.T) { m := readManifest(t) - bottom := m.resource(t, "bottom-bar")["content"].(string) + bottom := m.text(t, "bottom-bar") for _, never := range []string{`block = "battery"`, `block = "amd_gpu"`, `block = "nvidia_gpu"`, "mac = ", "api_key", "city_id", "openweathermap", "~/scripts/"} { if strings.Contains(bottom, never) { t.Errorf("the bottom bar has %s", never) @@ -57,9 +60,9 @@ func TestTheBarsAreAContributionToTheWindowManagerAndTheWatchdogRunsOncePerSessi !strings.Contains(c, "font pango:JetBrainsMono Nerd Font 11") { t.Fatalf("%s", c) } - if len(m.Shell) != 1 || m.Shell[0].For != "xinitrc" || m.Shell[0].Slot != "normal" || - !strings.Contains(m.Shell[0].Code, `"$HOME/.local/bin/i3bar-watchdog" "$$" &`) { - t.Fatalf("%+v", m.Shell) + if len(m.shell(t)) != 1 || m.shell(t)[0].For != "xinitrc" || m.shell(t)[0].Slot != "normal" || + !strings.Contains(m.shell(t)[0].Code, `"$HOME/.local/bin/i3bar-watchdog" "$$" &`) { + t.Fatalf("%+v", m.shell(t)) } } @@ -68,3 +71,8 @@ func TestTheToolsAgreeWithTheManifest(t *testing.T) { checkTheToolsAgree(t, m) checkNoSecretsOrInstallationNames(t) } + +// No file resource carries its text inline (novox/hq ADR 0349 §2). +func TestEveryFileTakesItsTextFromATemplate(t *testing.T) { + readManifest(t).everyFileIsATemplate(t) +} diff --git a/modules/i3status-rust/files/block.tmpl b/modules/i3status-rust/files/block.tmpl deleted file mode 100644 index 527a6426..00000000 --- a/modules/i3status-rust/files/block.tmpl +++ /dev/null @@ -1,26 +0,0 @@ -{{- /* A block other modules contribute, in i3status-rust's grammar (novox/hq ADR 0255). The controller -renders one piece at a time: its fields (bar, place, order, shows, options), module and machine. */ -}} -{{- if eq .shows "battery" -}} -[[block]] -block = "battery" -{{with .options}}{{with .device}}device = {{quote .}} -{{end}}{{end -}} -format = "$icon $percentage{ $time|}" -charging_format = "$icon $percentage{ $time|}" -not_charging_format = "$icon $percentage" -full_format = "$icon $percentage" -empty_format = "$icon $percentage" -missing_format = "" -{{- else if eq .shows "command" -}} -[[block]] -block = "custom" -command = {{quote .options.command}} -interval = {{.options.interval}} -{{- else if eq .shows "state" -}} -[[block]] -block = "custom" -command = {{quote (printf "~/.local/bin/i3status-value %s %s" .options.state (or .options.key .machine))}} -interval = 5 -hide_when_empty = true -{{- end}} - diff --git a/modules/i3status-rust/files/bottom-bar.toml b/modules/i3status-rust/files/bottom-bar.toml index 58ef2029..12e6805f 100644 --- a/modules/i3status-rust/files/bottom-bar.toml +++ b/modules/i3status-rust/files/bottom-bar.toml @@ -74,7 +74,7 @@ format_alt = "$icon_swap $swap_used.eng(w:2) ($swap_used_percents.eng(w:2))" # Blocks other modules contribute beside the machine's resources (novox/hq ADR 0255), rendered from # what they show by this module's template, and only on a machine that has what they show. -${contribution:node-bar:block:bar=bottom,place=resources} +${collected:contribution:node-bar:block:bar=bottom,place=resources} [[block]] block = "docker" interval = 2 @@ -88,7 +88,7 @@ cmd = "pavucontrol" # Blocks other modules contribute beside the sound (novox/hq ADR 0255): the power module's battery and # power draw, where the machine has them. -${contribution:node-bar:block:bar=bottom,place=status} +${collected:contribution:node-bar:block:bar=bottom,place=status} # The weather from the Norwegian Meteorological Institute, which needs no key, where the machine is. [[block]] block = "weather" diff --git a/modules/i3status-rust/files/top-bar.toml b/modules/i3status-rust/files/top-bar.toml index b767a765..be198e8a 100644 --- a/modules/i3status-rust/files/top-bar.toml +++ b/modules/i3status-rust/files/top-bar.toml @@ -34,4 +34,4 @@ interval = 60 # Blocks other modules contribute to the top bar (novox/hq ADR 0255). None do yet; both places are kept # so that one that does is never dropped. -${contribution:node-bar:block:bar=top,place=resources}${contribution:node-bar:block:bar=top,place=status} +${collected:contribution:node-bar:block:bar=top,place=resources}${collected:contribution:node-bar:block:bar=top,place=status} diff --git a/modules/i3status-rust/files/xinitrc-normal b/modules/i3status-rust/files/xinitrc-normal new file mode 100644 index 00000000..888acdb8 --- /dev/null +++ b/modules/i3status-rust/files/xinitrc-normal @@ -0,0 +1,3 @@ +# The bar watchdog (module i3status-rust, novox/hq ADR 0208): i3 never restarts a bar that dies; this +# brings it back. Once per session, ending with the session's own process ($$). +"$HOME/.local/bin/i3bar-watchdog" "$$" & diff --git a/modules/i3status-rust/module.json b/modules/i3status-rust/module.json index 373338f4..6704d0dc 100644 --- a/modules/i3status-rust/module.json +++ b/modules/i3status-rust/module.json @@ -1,6 +1,7 @@ { "module": "i3status-rust", "version": "1", + "moved": true, "capabilities": [ "package-manager" ], @@ -10,10 +11,7 @@ "claims": [ { "name": "node-bar", - "scope": "node", - "renders": { - "block": "{{- /* A block other modules contribute, in i3status-rust's grammar (novox/hq ADR 0255). The controller\nrenders one piece at a time: its fields (bar, place, order, shows, options), module and machine. */ -}}\n{{- if eq .shows \"battery\" -}}\n[[block]]\nblock = \"battery\"\n{{with .options}}{{with .device}}device = {{quote .}}\n{{end}}{{end -}}\nformat = \"$icon $percentage{ $time|}\"\ncharging_format = \"$icon $percentage{ $time|}\"\nnot_charging_format = \"$icon $percentage\"\nfull_format = \"$icon $percentage\"\nempty_format = \"$icon $percentage\"\nmissing_format = \"\"\n{{- else if eq .shows \"command\" -}}\n[[block]]\nblock = \"custom\"\ncommand = {{quote .options.command}}\ninterval = {{.options.interval}}\n{{- else if eq .shows \"state\" -}}\n[[block]]\nblock = \"custom\"\ncommand = {{quote (printf \"~/.local/bin/i3status-value %s %s\" .options.state (or .options.key .machine))}}\ninterval = 5\nhide_when_empty = true\n{{- end}}\n\n" - } + "scope": "node" } ], "tools": [ @@ -22,11 +20,16 @@ "i3status_rust_block_run", "i3status_rust_themes" ], - "shell": [ + "items": [ { - "for": "xinitrc", - "slot": "normal", - "code": "# The bar watchdog (module i3status-rust, novox/hq ADR 0208): i3 never restarts a bar that dies; this\n# brings it back. Once per session, ending with the session's own process ($$).\n\"$HOME/.local/bin/i3bar-watchdog\" \"$$\" &\n" + "collection": "shell:xinitrc:normal", + "name": "xinitrc-normal", + "template": "files/xinitrc-normal" + }, + { + "collection": "contribution:node-display-session:config", + "name": "display-session", + "template": "files/i3/60-i3status-rust.conf" } ], "resources": [ @@ -55,7 +58,7 @@ "path": "${machine:account-home}/.config/i3status-rust/top-bar.toml", "owner": "${machine:account}", "mode": "0644", - "content": "# The top bar (module i3status-rust, novox/hq ADR 0208): the focused window's title and the uptime.\n# Owned by the mesh: replaced at every push. Adopted unchanged from the workstations of 2026-10-04.\nicons_format = \"{icon}\"\n\n[theme]\ntheme = \"plain\"\n[theme.overrides]\nidle_bg = \"#000000\"\nidle_fg = \"#f1f1f1\"\ninfo_bg = \"#000000\"\ninfo_fg = \"#f1f1f1\"\ngood_bg = \"#000000\"\ngood_fg = \"#859900\"\nwarning_bg = \"#000000\"\nwarning_fg = \"#de5200\"\ncritical_bg = \"#000000\"\ncritical_fg = \"#ff0000\"\nseparator = \" | \"\nseparator_fg = \"#ffffff\"\nseparator_bg = \"#000000\"\n\n[icons]\nicons = \"custom-icons\"\n\n[[block]]\nblock = \"focused_window\"\n[block.format]\nfull = \" $title.str(max_w:50) |\"\nshort = \" $title.str(max_w:100) |\"\n\n[[block]]\nblock = \"uptime\"\ninterval = 60\n\n# Blocks other modules contribute to the top bar (novox/hq ADR 0255). None do yet; both places are kept\n# so that one that does is never dropped.\n${contribution:node-bar:block:bar=top,place=resources}${contribution:node-bar:block:bar=top,place=status}\n" + "template": "files/top-bar.toml" }, { "id": "bottom-bar", @@ -63,7 +66,7 @@ "path": "${machine:account-home}/.config/i3status-rust/bottom-bar.toml", "owner": "${machine:account}", "mode": "0644", - "content": "# The bottom bar (module i3status-rust, novox/hq ADR 0208). Owned by the mesh: replaced at every\n# push. Adopted from the workstations of 2026-10-04 with what follows a machine's hardware or one\n# person's devices left out: a block that depends on the hardware is contributed by the module that\n# knows it (novox/hq ADR 0255), and a headset's address is not the bar's to know. Every block written\n# here works the same on any machine.\nicons_format = \"{icon}\"\n\n[theme]\ntheme = \"plain\"\n[theme.overrides]\nidle_bg = \"#000000\"\nidle_fg = \"#f1f1f1\"\ninfo_bg = \"#000000\"\ninfo_fg = \"#f1f1f1\"\ngood_bg = \"#000000\"\ngood_fg = \"#859900\"\nwarning_bg = \"#000000\"\nwarning_fg = \"#de5200\"\ncritical_bg = \"#000000\"\ncritical_fg = \"#ff0000\"\nseparator = \" | \"\nseparator_fg = \"#ffffff\"\nseparator_bg = \"#000000\"\n\n[icons]\nicons = \"custom-icons\"\n\n[[block]]\nblock = \"tea_timer\"\nformat = \"$icon{ $minutes:$seconds|}\"\ndone_cmd = \"notify-send 'Timer finished'\"\n\n# Pending updates: official and, where an AUR helper is installed, AUR. The script keeps the last good\n# count through a network gap and shows an error only after ten failed checks in a row. Click: the\n# list, in the launcher's menu.\n[[block]]\nblock = \"custom\"\ncommand = \"~/.local/bin/i3status-updates 10\"\njson = true\ninterval = 1800\n[[block.click]]\nbutton = \"left\"\ncmd = \"checkupdates | dmenu -l 20 -p Updates\"\n\n[[block]]\nblock = \"cpu\"\n\n[[block]]\nblock = \"disk_space\"\npath = \"/\"\ninfo_type = \"used\"\nalert_unit = \"GB\"\ninterval = 20\nwarning = 850\nalert = 920\nformat = \"$icon / $used.eng(w:2) ($percentage.eng(w:2))\"\n\n[[block]]\nblock = \"disk_space\"\npath = \"/home\"\ninfo_type = \"used\"\nalert_unit = \"GB\"\ninterval = 20\nwarning = 850\nalert = 920\nformat = \"$icon /home $used.eng(w:2) ($percentage.eng(w:2))\"\n\n[[block]]\nblock = \"memory\"\n# What programs hold, without the cache the kernel gives back when asked: the figure the memory-pressure\n# module judges by. Counting the cache as used read near full on a machine with room to spare.\nformat = \"$icon $mem_used.eng(w:2) ($mem_used_percents.eng(w:2))\"\nformat_alt = \"$icon_swap $swap_used.eng(w:2) ($swap_used_percents.eng(w:2))\"\n\n# Blocks other modules contribute beside the machine's resources (novox/hq ADR 0255), rendered from\n# what they show by this module's template, and only on a machine that has what they show.\n${contribution:node-bar:block:bar=bottom,place=resources}\n[[block]]\nblock = \"docker\"\ninterval = 2\nformat = \"$icon $running/$total\"\n\n[[block]]\nblock = \"sound\"\n[[block.click]]\nbutton = \"left\"\ncmd = \"pavucontrol\"\n\n# Blocks other modules contribute beside the sound (novox/hq ADR 0255): the power module's battery and\n# power draw, where the machine has them.\n${contribution:node-bar:block:bar=bottom,place=status}\n# The weather from the Norwegian Meteorological Institute, which needs no key, where the machine is.\n[[block]]\nblock = \"weather\"\nformat = \"$icon $weather_verbose ($location) $temp\"\nautolocate = true\nautolocate_interval = 600\n[block.service]\nname = \"metno\"\n\n[[block]]\nblock = \"notify\"\ndriver = \"dunst\"\nformat = \" $icon {($notification_count.eng(w:1))|}\"\n\n[[block]]\nblock = \"time\"\ninterval = 1\nformat = \"$timestamp.datetime(f:'%a %d/%m %H:%M:%S')\"\n" + "template": "files/bottom-bar.toml" }, { "id": "icons", @@ -71,7 +74,7 @@ "path": "${machine:account-home}/.config/i3status-rust/icons/custom-icons.toml", "owner": "${machine:account}", "mode": "0644", - "content": "# Icons for the bars (module i3status-rust, novox/hq ADR 0208), from the JetBrains Mono Nerd Font.\n# Owned by the mesh: replaced at every push. Adopted unchanged from the workstations of 2026-10-04.\n# Material from NerdFont\n# Codepoints from the Nerd Fonts cheat sheet\nbacklight = [\n \"\\ue38d\", # nf-weather-moon_new\n \"\\ue3d4\", # nf-weather-moon_alt_waxing_gibbous_6\n \"\\ue3d3\", # nf-weather-moon_alt_waxing_gibbous_5\n \"\\ue3d2\", # nf-weather-moon_alt_waxing_gibbous_4\n \"\\ue3d1\", # nf-weather-moon_alt_waxing_gibbous_3\n \"\\ue3d0\", # nf-weather-moon_alt_waxing_gibbous_2\n \"\\ue3cf\", # nf-weather-moon_alt_waxing_gibbous_1\n \"\\ue3ce\", # nf-weather-moon_alt_first_quarter\n \"\\ue3cd\", # nf-weather-moon_alt_waxing_crescent_6\n \"\\ue3cc\", # nf-weather-moon_alt_waxing_crescent_5\n \"\\ue3cb\", # nf-weather-moon_alt_waxing_crescent_4\n \"\\ue3ca\", # nf-weather-moon_alt_waxing_crescent_3\n \"\\ue3c9\", # nf-weather-moon_alt_waxing_crescent_2\n \"\\ue3c8\", # nf-weather-moon_alt_waxing_crescent_1\n \"\\ue39b\", # nf-weather-moon_full\n]\nbat_charging = \"\\U000f0084\" # nf-md-battery_charging\nbat_not_available = \"\\U000f0091\" # nf-md-battery_unknown\nbat = [\n \"\\U000f007a\", # nf-md-battery_10\n \"\\U000f007b\", # nf-md-battery_20\n \"\\U000f007c\", # nf-md-battery_30\n \"\\U000f007d\", # nf-md-battery_40\n \"\\U000f007e\", # nf-md-battery_50\n \"\\U000f007f\", # nf-md-battery_60\n \"\\U000f0080\", # nf-md-battery_70\n \"\\U000f0081\", # nf-md-battery_80\n \"\\U000f0082\", # nf-md-battery_90\n \"\\U000f0079\", # nf-md-battery\n]\nbell = \"\\U000f009c\" # nf-md-bell_outline\nbell-slash = \"\\U000f009b\" # nf-md-bell_off\nblackberry = \"\\uf307\" # nf-md-blackberry\nbluetooth = \"\\U000f00af\" # nf-md-bluetooth\ncalendar = \"\\U000f00ed\" # nf-md-calendar\ncogs = \"\\U000f0493\" # nf-md-cog\ncpu = [\n\t\"\\U000F0F86\", # nf-md-speedometer_slow\n\t\"\\U000F0F85\", # nf-md-speedometer_medium\n\t\"\\U000F04C5\", # nf-md-speedometer\n]\ncpu_boost_on = \"\\U000f0521\" # nf-md-toggle_switch\ncpu_boost_off = \"\\U000f0a19\" # nf-md-toggle_switch_off_outline\ndisk_drive = \"\\U000f02ca\" # nf-md-harddisk\ndocker = \"\\uf308\" # nf-linux-docker\ngithub = \"\\U000f02a4\" # nf-md-github\ngpu = \"\\U000f0379\" # nf-md-monitor\nheadphones = \"\\U000f02cb\" # nf-md-headphones\njoystick = \"\\U000f0297\" # nf-md-gamepad_variant\nkeyboard = \"\\U000f030c\" # nf-md-keyboard\nmail = \"\\U000f01ee\" # nf-md-email\nmemory_mem = \"\\U000f035b\" # nf-md-memory\nmemory_swap = \"\\U000f02ca\" # nf-md-harddisk\nmouse = \"\\U000f037d\" # nf-md-mouse\nmusic = \"\\U000f075a\" # nf-md-music\nmusic_next = \"\\U000f04ad\" # nf-md-skip_next\nmusic_pause = \"\\U000f03e4\" # nf-md-pause\nmusic_play = \"\\U000f040a\" # nf-md-play\nmusic_prev = \"\\U000f04ae\" # nf-md-skip_previous\nnet_bridge = \"\\U000f04aa\" # nf-md-sitemap\nnet_down = \"\\U000f01da\" # nf-md-download\nnet_loopback = \"\\U000f006f\" # nf-md-backup_restore\nnet_modem = \"\\U000f03f2\" # nf-md-phone\nnet_cellular = [\n \"\\U000F08FD\", # nf-md-network_strength_off_outline\n \"\\U000F08FE\", # nf-md-network_strength_outline\n \"\\U000F08F4\", # nf-md-network_strength_1\n \"\\U000F08F6\", # nf-md-network_strength_2\n \"\\U000F08F8\", # nf-md-network_strength_3\n \"\\U000F08FA\", # nf-md-network_strength_4\n]\nnet_up = \"\\U000f0552\" # nf-md-upload\nnet_vpn = \"\\U000f0582\" # nf-md-vpn\nnet_wired = \"\\U000f0200\" # nf-md-ethernet\nnet_wireless = [\n\t\"\\U000F092F\", # nf-md-wifi_strength_outline\n\t\"\\U000F091F\", # nf-md-wifi_strength_1\n\t\"\\U000F0922\", # nf-md-wifi_strength_2\n\t\"\\U000F0925\", # nf-md-wifi_strength_3\n\t\"\\U000F0928\", # nf-md-wifi_strength_4\n]\nnotification = \"\\U000f009c\" # nf-md-bell_outline\nphone = \"\\U000f03f2\" # nf-md-phone\nphone_disconnected = \"\\U000f0658\" # nf-md-phone_minus\nping = \"\\U000f051f\" # nf-md-timer_sand\npomodoro = \"\\ue001\" # nf-pom-pomodoro_done\npomodoro_break = \"\\U000f0176\" # nf-md-coffee\npomodoro_paused = \"\\U000f03e4\" # nf-md-pause\npomodoro_started = \"\\U000f040a\" # nf-md-play\npomodoro_stopped = \"\\U000f04db\" # nf-md-stop\nresolution = \"\\U000f0293\" # nf-md-fullscreen\ntasks = \"\\U000f05c7\" # nf-md-playlist_check\ntea = \"\\U000f0d9e\" # nf-md-tea\nthermometer = [\n \"\\U000f10c3\", # nf-md-thermometer_low\n \"\\U000f050f\", # nf-md-thermometer\n \"\\U000f10c2\", # nf-md-thermometer_high\n]\ntime = \"\\U000f0150\" # nf-md-clock_outline\ntoggle_off = \"\\U000f0a19\" # nf-md-toggle_switch_off_outline\ntoggle_on = \"\\U000f0521\" # nf-md-toggle_switch\nunknown = \"\\U000f0186\" # nf-md-comment_question_outline | TODO: Make default?\nupdate = \"\\U000f03d5\" # nf-md-package_up\nuptime = \"\\U000f0153\" # nf-md-clock_in\nvolume_muted = \"\\U000f075f\" # nf-md-volume_mute\nvolume = [\n \"\\U000f057f\", # nf-md-volume_low\n \"\\U000f0580\", # nf-md-volume_medium\n \"\\U000f057e\", # nf-md-volume_high\n]\nmicrophone_muted = \"\\U000f036d\" # nf-md-microphone_off\nmicrophone = [\n\t\"\\U000f036e\", # nf-md-microphone_outline\n \"\\U000f036c\", # nf-md-microphone\n \"\\U000f036c\", # nf-md-microphone\n]\nxrandr = \"\\U000f037a\" # nf-md-monitor_multiple\n\n# Weather icons (the names the weather block documents)\nweather_sun = \"\\ue30d\" # nf-weather-day_sunny (when weather is reported as “Clear” during the day)\nweather_moon = \"\\ue32b\" # nf-weather-night_clear (when weather is reported as “Clear” at night)\nweather_clouds = \"\\ue312\" # nf-weather-cloudy (when weather is reported as “Clouds” during the day)\nweather_clouds_night = \"\\ue37e\" # nf-weather-night_alt_cloudy (when weather is reported as “Clouds” at night)\nweather_fog = \"\\ue313\" # nf-weather-fog (when weather is reported as “Fog” or “Mist” during the day)\nweather_fog_night = \"\\ue346\" # nf-weather-night_fog (when weather is reported as “Fog” or “Mist” at night)\nweather_rain = \"\\ue318\" # nf-weather-rain (when weather is reported as “Rain” or “Drizzle” during the day)\nweather_rain_night = \"\\ue325\" # nf-weather-night_alt_rain (when weather is reported as “Rain” or “Drizzle” at night)\nweather_snow = \"\\ue31a\" # nf-weather-snow (when weather is reported as “Snow”)\nweather_thunder = \"\\ue31d\" # nf-weather-thunderstorm (when weather is reported as “Thunderstorm” during the day)\nweather_thunder_night = \"\\ue32a\" # nf-weather-night_alt_thunderstorm (when weather is reported as “Thunderstorm” at night)\n" + "template": "files/icons/custom-icons.toml" }, { "id": "updates", @@ -79,7 +82,7 @@ "path": "${machine:account-home}/.local/bin/i3status-updates", "owner": "${machine:account}", "mode": "0755", - "content": "#!/usr/bin/env bash\n# i3status-updates [threshold] (module i3status-rust, novox/hq ADR 0208): the pending updates, for\n# the bottom bar's custom block (json = true). Emits one JSON line.\n#\n# Adopted from the operator's pkg-updates of 2026-10-04. The native `packages` block showed a raw\n# red error on every transient network failure, which on a roaming laptop is often. This keeps the\n# last good result through a failure and shows an error only after `threshold` failures in a row\n# (default 10). Official packages through checkupdates (pacman-contrib, a temporary database, no\n# root); AUR packages through paru when it is installed, and none counted when it is not.\nset -uo pipefail\n\nthreshold=\"${1:-10}\"\nicon=\"update\"\nstate_dir=\"${XDG_RUNTIME_DIR:-/tmp}/i3status-updates\"\nmkdir -p \"$state_dir\"\nlast=\"$state_dir/last_good.json\"\nfailures=\"$state_dir/failures\"\n\njson() { printf '{\"icon\":\"%s\",\"state\":\"%s\",\"text\":\"%s\",\"short_text\":\"%s\"}' \"$icon\" \"$1\" \"$2\" \"$3\"; }\ncount() { [ -n \"$1\" ] && printf '%s\\n' \"$1\" | grep -c . || echo 0; }\n\nofficial=\"$(checkupdates 2>/dev/null)\"\nofficial_rc=$?\naur=\"\" aur_rc=0\nif command -v paru >/dev/null 2>&1; then\n\taur=\"$(paru -Qua 2>/dev/null)\"\n\taur_rc=$?\nfi\n\n# checkupdates: 0 updates listed, 2 none pending; anything else is a failure.\nif { [ \"$official_rc\" -eq 0 ] || [ \"$official_rc\" -eq 2 ]; } && [ \"$aur_rc\" -eq 0 ]; then\n\to=\"$(count \"$official\")\"\n\ta=\"$(count \"$aur\")\"\n\ttotal=$((o + a))\n\tif [ \"$total\" -eq 0 ]; then\n\t\tpayload=\"$(json Good \"up to date\" \"\")\"\n\telse\n\t\t# A kernel update wants a reboot: the one worth not putting off.\n\t\tstate=Info\n\t\tprintf '%s\\n' \"$official\" | grep -qE '^(linux|linux-lts|linux-zen) ' && state=Warning\n\t\tlabel=\"$o + $a = $total updates\"\n\t\t[ \"$a\" -eq 0 ] && label=\"$total updates\"\n\t\t[ \"$total\" -eq 1 ] && label=\"1 update\"\n\t\tpayload=\"$(json \"$state\" \"$label\" \"$total\")\"\n\tfi\n\tprintf '%s' \"$payload\" >\"$last\"\n\tprintf '0' >\"$failures\"\n\tprintf '%s\\n' \"$payload\"\n\texit 0\nfi\n\nn=$(($(cat \"$failures\" 2>/dev/null || echo 0) + 1))\nprintf '%s' \"$n\" >\"$failures\"\nif [ \"$n\" -ge \"$threshold\" ]; then\n\tjson Critical \"update check failing (${n}x)\" \"!\"\n\techo\nelif [ -s \"$last\" ]; then\n\tcat \"$last\"\n\techo\nelse\n\tjson Idle \"\" \"\"\n\techo\nfi\n" + "template": "files/bin/i3status-updates" }, { "id": "watchdog", @@ -87,7 +90,7 @@ "path": "${machine:account-home}/.local/bin/i3bar-watchdog", "owner": "${machine:account}", "mode": "0755", - "content": "#!/usr/bin/env bash\n# i3bar-watchdog [session-pid] (module i3status-rust, novox/hq ADR 0208): respawns a bar that died.\n#\n# i3 starts one i3bar per `bar { }` block and never restarts one that exits, and a reload does not\n# either. Changing the monitor setup reliably kills the bar that owns the tray. This brings back the\n# one missing i3bar, without restarting i3, and logs the outputs at that moment: the evidence for the\n# cause, which is i3bar's.\n#\n# Started once per session from the session's start, with the session's own pid; it ends when that\n# process does. Adopted from the predecessor's i3-bar-watchdog user unit of 2026-10-04.\nset -uo pipefail\n\nsession_pid=\"${1:-}\"\ninterval=\"${I3_BAR_WATCHDOG_INTERVAL:-5}\"\n# Two misses in a row before acting: an i3 restart tears every bar down and starts them again.\nconfirm=\"${I3_BAR_WATCHDOG_CONFIRM:-2}\"\n# Never respawn the same bar more often than this, so a bar that dies at once is not a tight loop.\ncooldown=\"${I3_BAR_WATCHDOG_COOLDOWN:-30}\"\n\nlog() { printf 'i3bar-watchdog: %s\\n' \"$*\" >&2; }\n\nfor tool in i3-msg i3bar pgrep; do\n\tcommand -v \"$tool\" >/dev/null 2>&1 || {\n\t\tlog \"$tool is missing; not watching\"\n\t\texit 1\n\t}\ndone\n\ndeclare -A missed=() fixed=()\n\nbar_ids() { i3-msg -t get_bar_config 2>/dev/null | grep -oE '\"[^\"]+\"' | tr -d '\"'; }\noutputs() { xrandr --listmonitors 2>/dev/null | tail -n +2 | awk '{print $2\" \"$3}' | tr '\\n' ' '; }\nsession_alive() { [ -z \"$session_pid\" ] || kill -0 \"$session_pid\" 2>/dev/null; }\n\nwhile session_alive; do\n\tsleep \"$interval\"\n\tsocket=\"$(i3 --get-socketpath 2>/dev/null)\"\n\t[ -n \"$socket\" ] && [ -S \"$socket\" ] || continue\n\tids=\"$(bar_ids)\"\n\t[ -n \"$ids\" ] || continue\n\twhile read -r id; do\n\t\t[ -n \"$id\" ] || continue\n\t\tif pgrep -u \"$EUID\" -f -- \"i3bar --bar_id=$id\" >/dev/null 2>&1; then\n\t\t\tmissed[$id]=0\n\t\t\tcontinue\n\t\tfi\n\t\tmissed[$id]=$((${missed[$id]:-0} + 1))\n\t\t[ \"${missed[$id]}\" -ge \"$confirm\" ] || continue\n\t\tnow=\"$(date +%s)\"\n\t\tif [ $((now - ${fixed[$id]:-0})) -lt \"$cooldown\" ]; then\n\t\t\tcontinue\n\t\tfi\n\t\tlog \"$id is gone; respawning it. Outputs now: $(outputs)\"\n\t\tnohup i3bar --bar_id=\"$id\" --socket=\"$socket\" >/dev/null 2>&1 &\n\t\tdisown 2>/dev/null || true\n\t\tfixed[$id]=\"$now\"\n\t\tsleep 2\n\t\tif pgrep -u \"$EUID\" -f -- \"i3bar --bar_id=$id\" >/dev/null 2>&1; then\n\t\t\tmissed[$id]=0\n\t\telse\n\t\t\tlog \"$id exited within 2s of respawning; check its status_command\"\n\t\tfi\n\tdone <<<\"$ids\"\ndone\n" + "template": "files/bin/i3bar-watchdog" }, { "id": "value", @@ -95,7 +98,7 @@ "path": "${machine:account-home}/.local/bin/i3status-value", "owner": "${machine:account}", "mode": "0755", - "content": "#!/bin/sh\n# i3status-value (module i3status-rust, novox/hq ADR 0260): one value another module keeps\n# on the bus, for a block it contributed.\n#\n# This module's own tools bundle watches the state on the bus (the mesh grants it the read) and leaves\n# the value's text in this module's own directory; this prints it. A value not refreshed for thirty\n# seconds is one its module stopped giving, and prints nothing, so the block hides rather than show an\n# old number as if it were now.\ndir=\"${XDG_RUNTIME_DIR:-/run/user/$(id -u)}/i3status-rust/values\"\nf=\"$dir/$1@$2\"\nif [ -r \"$f\" ] && [ $(( $(date +%s) - $(stat -c %Y \"$f\") )) -le 30 ]; then\n\thead -n 1 \"$f\"\nfi\n" + "template": "files/bin/i3status-value" } ], "build": { @@ -110,14 +113,12 @@ "loads": [ "i3status-rust-tools" ] + }, + { + "name": "files", + "kind": "archive", + "from": "files" } ] - }, - "contributions": [ - { - "seat": "node-display-session", - "kind": "config", - "content": "# The bars (module i3status-rust, novox/hq ADR 0208). Owned by the mesh: replaced at every push.\n# i3 reads this file through its configuration's `include ~/.config/i3/config.d/*.conf`. The bottom\n# bar shows the machine; the top bar the focused window and the tray, on the primary output. The\n# face is the monospace one every desktop module names.\nbar {\n font pango:JetBrainsMono Nerd Font 11\n position bottom\n status_command i3status-rs ~/.config/i3status-rust/bottom-bar.toml\n tray_output none\n colors {\n separator #ffffff\n background #000000\n statusline #ffffff\n # The text on an accent-coloured button is dark: light text on the accent was barely\n # legible.\n focused_workspace #de5200 #de5200 #000000\n active_workspace #de5200 #de5200 #000000\n inactive_workspace #000000 #000000 #ffffff\n urgent_workspace #2f343a #900000 #ffffff\n }\n}\n\nbar {\n font pango:JetBrainsMono Nerd Font 11\n position top\n status_command i3status-rs ~/.config/i3status-rust/top-bar.toml\n workspace_buttons no\n tray_output primary\n colors {\n separator #ffffff\n background #000000\n statusline #ffffff\n }\n}\n" - } - ] + } } diff --git a/modules/icecast/files/config.json b/modules/icecast/files/config.json new file mode 100644 index 00000000..0967ef42 --- /dev/null +++ b/modules/icecast/files/config.json @@ -0,0 +1 @@ +{} diff --git a/modules/icecast/files/icecast.xml b/modules/icecast/files/icecast.xml new file mode 100644 index 00000000..de4dd792 --- /dev/null +++ b/modules/icecast/files/icecast.xml @@ -0,0 +1,56 @@ + + + Earth + icemaster@localhost + + 100 + 2 + 524288 + 30 + 15 + 10 + 1 + 65535 + + + ${secret:source} + ${secret:relay} + admin + ${secret:admin} + + + localhost + + 8000 + + +
+ + 1 + + /usr/share/icecast + /var/log/icecast + /usr/share/icecast/web + /usr/share/icecast/admin + + + + access.log + error.log + 3 + 10000 + + + 0 + + + icecast + icecast + + + diff --git a/modules/icecast/module.json b/modules/icecast/module.json index 4c241ebc..f8288be7 100644 --- a/modules/icecast/module.json +++ b/modules/icecast/module.json @@ -47,6 +47,20 @@ } ] }, + "config": { + "url": { + "type": "string", + "description": "The address the module's tools reach Icecast at, as a URL; unset, the code falls back to its environment, else this machine's Icecast on its port.", + "trusted": true, + "default": "" + }, + "user": { + "type": "string", + "description": "The Icecast admin account the module's tools sign in as; unset, the code falls back to its environment, else admin.", + "trusted": true, + "default": "" + } + }, "resources": [ { "id": "mesh-state", @@ -71,7 +85,7 @@ "type": "file", "path": "${dir:state}/icecast.xml", "mode": "0600", - "content": "\n \n Earth\n icemaster@localhost\n \n 100\n 2\n 524288\n 30\n 15\n 10\n 1\n 65535\n \n \n ${secret:source}\n ${secret:relay}\n admin\n ${secret:admin}\n \n \n localhost\n \n 8000\n \n \n
\n \n 1\n \n /usr/share/icecast\n /var/log/icecast\n /usr/share/icecast/web\n /usr/share/icecast/admin\n \n \n \n access.log\n error.log\n 3\n 10000\n \n \n 0\n \n \n icecast\n icecast\n \n \n\n" + "template": "files/icecast.xml" }, { "id": "net", @@ -100,8 +114,9 @@ "type": "file", "path": "${dir:mesh-state}/config.json", "mode": "0600", - "content": "{}\n", - "merge": "json" + "template": "files/config.json", + "merge": "json", + "takes-config": ["url", "user"] } ], "build": { @@ -123,6 +138,11 @@ "MESH_ICECAST_CONFIG_FILE": "${dir:mesh-state}/config.json", "MESH_ICECAST_ADMIN_PASSWORD_FILE": "${dir:state}/admin.secret" } + }, + { + "name": "files", + "kind": "archive", + "from": "files" } ] } diff --git a/modules/influxdb/files/config.json b/modules/influxdb/files/config.json new file mode 100644 index 00000000..0967ef42 --- /dev/null +++ b/modules/influxdb/files/config.json @@ -0,0 +1 @@ +{} diff --git a/modules/influxdb/files/server.env b/modules/influxdb/files/server.env new file mode 100644 index 00000000..d2552e00 --- /dev/null +++ b/modules/influxdb/files/server.env @@ -0,0 +1,6 @@ +DOCKER_INFLUXDB_INIT_MODE=setup +DOCKER_INFLUXDB_INIT_USERNAME=admin +DOCKER_INFLUXDB_INIT_PASSWORD_FILE=/run/secrets/admin +DOCKER_INFLUXDB_INIT_ADMIN_TOKEN_FILE=/run/secrets/admin-token +DOCKER_INFLUXDB_INIT_ORG=mesh +DOCKER_INFLUXDB_INIT_BUCKET=default diff --git a/modules/influxdb/module.json b/modules/influxdb/module.json index bad5aee4..2d73b8d7 100644 --- a/modules/influxdb/module.json +++ b/modules/influxdb/module.json @@ -26,12 +26,32 @@ "why": "queries, writes and the web UI, over http; consumers granted influxdb-api sign in with the mesh's credential, and a name is a route grant" } ], + "config": { + "org": { + "type": "string", + "description": "The organisation influxdb-api names to its consumers and the module's tools write to; mesh is the one influxdb makes at its first start (server.env), and a machine whose server was set up with another sets its own (novox/hq issue 696, ADR 0348 clause 11).", + "trusted": true, + "default": "mesh" + }, + "bucket": { + "type": "string", + "description": "The bucket influxdb-api names to its consumers and the module's tools write to; default is the one influxdb makes at its first start (server.env), and a machine whose server was set up with another sets its own (novox/hq issue 696, ADR 0348 clause 11).", + "trusted": true, + "default": "default" + }, + "url": { + "type": "string", + "description": "The address the module's tools reach influxdb at, as a URL; empty is this machine's influxdb on its port.", + "trusted": true, + "default": "" + } + }, "serves": { "influxdb-api": { "scheme": "http", "port": 8086, - "org": "mesh", - "bucket": "default" + "org": "${config:org}", + "bucket": "${config:bucket}" } }, "receives": { @@ -98,7 +118,7 @@ "type": "file", "path": "${dir:state}/server.env", "mode": "0600", - "content": "DOCKER_INFLUXDB_INIT_MODE=setup\nDOCKER_INFLUXDB_INIT_USERNAME=admin\nDOCKER_INFLUXDB_INIT_PASSWORD_FILE=/run/secrets/admin\nDOCKER_INFLUXDB_INIT_ADMIN_TOKEN_FILE=/run/secrets/admin-token\nDOCKER_INFLUXDB_INIT_ORG=mesh\nDOCKER_INFLUXDB_INIT_BUCKET=default\n" + "template": "files/server.env" }, { "id": "server", @@ -123,8 +143,14 @@ "type": "file", "path": "${dir:mesh-state}/config.json", "mode": "0600", - "content": "{}\n", - "merge": "json" + "template": "files/config.json", + "merge": "json", + "takes-config": [ + "bucket", + "org", + "url" + ], + "trusted": true } ], "requires": [ @@ -156,6 +182,11 @@ "MESH_INFLUXDB_TOKEN_FILE": "${dir:state}/admin-token.secret", "MESH_RECEIVES": "${dir:grants}/mesh.json" } + }, + { + "name": "files", + "kind": "archive", + "from": "files" } ] } diff --git a/modules/invoicing/files/api.env b/modules/invoicing/files/api.env new file mode 100644 index 00000000..8eaee6f0 --- /dev/null +++ b/modules/invoicing/files/api.env @@ -0,0 +1,9 @@ +NODE_ENV=production +PORT=9000 +MONGO_URL=mongodb://${bound:mongodb-database:as}:${secret:mongodb-database}@${bound:mongodb-database:at}:${bound:mongodb-database:port}/${bound:mongodb-database:as}?authSource=${bound:mongodb-database:as} +MONGO_DB=${bound:mongodb-database:as} +MINIO_BUCKET=${bound:s3-bucket:bucket} +MINIO_ENDPOINT=${bound:s3-bucket:at} +MINIO_PORT=${bound:s3-bucket:port} +MINIO_ACCESSKEY=${bound:s3-bucket:as} +MINIO_SECRET=${secret:s3-bucket} diff --git a/modules/invoicing/module.json b/modules/invoicing/module.json index 9f8dfcf5..5b846d1c 100644 --- a/modules/invoicing/module.json +++ b/modules/invoicing/module.json @@ -68,7 +68,7 @@ "type": "file", "path": "${dir:state}/api.env", "mode": "0600", - "content": "NODE_ENV=production\nPORT=9000\nMONGO_URL=mongodb://${bound:mongodb-database:as}:${secret:mongodb-database}@${bound:mongodb-database:at}:${bound:mongodb-database:port}/${bound:mongodb-database:as}?authSource=${bound:mongodb-database:as}\nMONGO_DB=${bound:mongodb-database:as}\nMINIO_BUCKET=${bound:s3-bucket:bucket}\nMINIO_ENDPOINT=${bound:s3-bucket:at}\nMINIO_PORT=${bound:s3-bucket:port}\nMINIO_ACCESSKEY=${bound:s3-bucket:as}\nMINIO_SECRET=${secret:s3-bucket}\n" + "template": "files/api.env" }, { "id": "net", @@ -113,5 +113,14 @@ "registry-api.novox.be": "built outside the mesh, from the application's own repository, and pulled from the registry that built it; moves when that repository is a build source on the git seat (novox/hq ADR 0155, issue 122)" } } - ] + ], + "build": { + "artifacts": [ + { + "name": "files", + "kind": "archive", + "from": "files" + } + ] + } } diff --git a/modules/jira/files/config.json b/modules/jira/files/config.json new file mode 100644 index 00000000..9e26dfee --- /dev/null +++ b/modules/jira/files/config.json @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/modules/jira/module.json b/modules/jira/module.json index 84a00079..eceb279d 100644 --- a/modules/jira/module.json +++ b/modules/jira/module.json @@ -4,6 +4,20 @@ "own-secrets": { "token": "${dir:state}/token" }, + "config": { + "atlassian-url": { + "type": "string", + "description": "The address of the operator's Atlassian site the jira tools call, as a URL such as https://example.atlassian.net, laid into the file as ATLASSIAN_URL; unset, the code falls back to its environment and answers no calls until it is given.", + "trusted": true, + "default": "" + }, + "atlassian-email": { + "type": "string", + "description": "The email address of the Atlassian account the jira tools sign in as with its API token, laid into the file as ATLASSIAN_EMAIL; unset, the code falls back to its environment and answers no calls until it is given.", + "trusted": true, + "default": "" + } + }, "resources": [ { "id": "state", @@ -16,8 +30,18 @@ "type": "file", "path": "${dir:state}/config.json", "merge": "json", - "content": "{}", - "mode": "0600" + "template": "files/config.json", + "mode": "0600", + "takes-config": [ + { + "key": "atlassian-url", + "as": "ATLASSIAN_URL" + }, + { + "key": "atlassian-email", + "as": "ATLASSIAN_EMAIL" + } + ] } ], "capabilities": [ @@ -39,6 +63,11 @@ "MESH_JIRA_TOKEN_FILE": "${dir:state}/token", "MESH_JIRA_CONFIG_FILE": "${dir:state}/config.json" } + }, + { + "name": "files", + "kind": "archive", + "from": "files" } ] } diff --git a/modules/kernel/cmd/kernel-tools/manifest_test.go b/modules/kernel/cmd/kernel-tools/manifest_test.go index 022877fa..b68dacf5 100644 --- a/modules/kernel/cmd/kernel-tools/manifest_test.go +++ b/modules/kernel/cmd/kernel-tools/manifest_test.go @@ -16,8 +16,8 @@ import ( func TestTheDefaultIsMadviseAndTheUnitFollowsTheSetting(t *testing.T) { m := manifest(t) - s := m.Settings["transparent-huge-pages"] - if s["kind"] != "preference" || s["default"] != "madvise" || s["why"] == "" { + s := m.Config["transparent-huge-pages"] + if s["type"] != "string" || s["default"] != "madvise" || s["description"] == "" || s["trusted"] != true { t.Fatalf("the setting: %v", s) } var ids []string @@ -30,10 +30,10 @@ func TestTheDefaultIsMadviseAndTheUnitFollowsTheSetting(t *testing.T) { script := m.resource(t, "huge-pages") // The script runs as root, so the setting it is written from is set at the terminal alone (issue 339). if script["path"] != "${dir:state}/huge-pages" || script["mode"] != "0755" || script["trusted"] != true || - !strings.Contains(script["content"].(string), "want='${setting:transparent-huge-pages}'") { + !strings.Contains(script.text(t), "want='${config:transparent-huge-pages}'") { t.Fatalf("the script: %v", script["path"]) } - unit := m.resource(t, "unit")["content"].(string) + unit := m.resource(t, "unit").text(t) for _, line := range []string{"Type=oneshot", "RemainAfterExit=yes", "ExecStart=${dir:state}/huge-pages apply", "ExecReload=${dir:state}/huge-pages apply", "ExecStop=${dir:state}/huge-pages restore", "WantedBy=sysinit.target", "DefaultDependencies=no", "ProtectSystem=strict", "ReadWritePaths=${dir:state}", "ProtectHome=yes", "PrivateNetwork=yes", @@ -75,8 +75,8 @@ func newBedIn(t *testing.T, dir, want, found string) bed { t.Helper() b := bed{script: filepath.Join(dir, "huge-pages"), sys: filepath.Join(dir, "thp"), state: filepath.Join(dir, "state"), rotated: filepath.Join(dir, "rotated")} - content := manifest(t).resource(t, "huge-pages")["content"].(string) - content = strings.ReplaceAll(content, "${setting:transparent-huge-pages}", want) + content := manifest(t).resource(t, "huge-pages").text(t) + content = strings.ReplaceAll(content, "${config:transparent-huge-pages}", want) content = strings.ReplaceAll(content, "${dir:state}", b.state) if strings.Contains(content, "${") { t.Fatalf("a substitution the test does not make is left in the script") diff --git a/modules/kernel/cmd/kernel-tools/shape_test.go b/modules/kernel/cmd/kernel-tools/shape_test.go index c30224f3..2350b801 100644 --- a/modules/kernel/cmd/kernel-tools/shape_test.go +++ b/modules/kernel/cmd/kernel-tools/shape_test.go @@ -3,6 +3,7 @@ package main import ( "encoding/json" "os" + "path/filepath" "testing" ) @@ -11,7 +12,7 @@ type resource map[string]any type manifestShape struct { Module string `json:"module"` Capabilities []string `json:"capabilities"` - Settings map[string]map[string]any `json:"settings"` + Config map[string]map[string]any `json:"config"` Tools []string `json:"tools"` Resources []resource `json:"resources"` Build struct { @@ -77,3 +78,13 @@ func TestToolsAreTheManifests(t *testing.T) { t.Fatalf("loads: %v", tools["loads"]) } } + +// text is the file text a resource keeps in its template under files/ (novox/hq ADR 0349 §2). +func (r resource) text(t *testing.T) string { + t.Helper() + raw, err := os.ReadFile(filepath.Join("../..", r["template"].(string))) + if err != nil { + t.Fatal(err) + } + return string(raw) +} diff --git a/modules/kernel/files/huge-pages.sh b/modules/kernel/files/huge-pages.sh new file mode 100644 index 00000000..e0a98155 --- /dev/null +++ b/modules/kernel/files/huge-pages.sh @@ -0,0 +1,84 @@ +#!/bin/sh +# Transparent huge pages as the mesh sets them on this machine (module kernel, novox/hq issue 746). +# Written by the mesh from this machine's setting transparent-huge-pages and run by +# mesh-kernel-huge-pages.service: `apply` at boot and whenever the setting changes, `restore` when the +# unit stops (the module unassigned, or the machine shutting down). Replaced on every push; change the +# setting, never this file. A change of the setting reloads the unit, which runs `apply` again: the mode +# moves straight to the new value and never passes back through the one the machine was found with. +# +# huge-pages apply|restore [ []] +# +# The two directories are arguments only so the module's tests can run this against a stand-in tree. +set -eu + +want='${config:transparent-huge-pages}' +sys=/sys/kernel/mm/transparent_hugepage +state='${dir:state}' +[ $# -ge 2 ] && sys=$2 +[ $# -ge 3 ] && state=$3 + +case "$want" in +always | madvise | never) ;; +*) + echo "the setting transparent-huge-pages is '$want'; the kernel takes always, madvise or never" >&2 + exit 2 + ;; +esac +[ -f "$sys/enabled" ] || { + echo "this kernel has no $sys/enabled: it was built without transparent huge pages, so there is nothing to set" >&2 + exit 1 +} + +# The mode in force is the word the kernel brackets: "always [madvise] never". A file holding one bare +# word (the stand-in tree of the tests) is read as that word. +mode() { sed -n -e 's/.*\[\([a-z]*\)\].*/\1/p' -e t -e 's/^ *\([a-z]*\) *$/\1/p' "$sys/enabled"; } + +[ $# -ge 1 ] || set -- usage +case "$1" in +apply) + now=$(mode) + # What the machine had before the mesh first set it, kept once, so `restore` gives it back. + [ -s "$state/found" ] || printf '%s\n' "$now" >"$state/found" + if [ "$now" = "$want" ]; then + echo "transparent huge pages: $want, as set" + exit 0 + fi + printf '%s\n' "$want" >"$sys/enabled" + got=$(mode) + [ "$got" = "$want" ] || { + echo "wrote $want to $sys/enabled and the kernel says $got" >&2 + exit 1 + } + echo "transparent huge pages: $now -> $want" + # A file folio already in the page cache keeps its size: the active journal files, mapped by journald + # and by every reader, stay in 2 MiB folios until they are archived (issue 746). Rotating starts new + # files that are mapped under the new mode. A failed rotation is said and does not fail the setting. + # At boot nothing is in huge folios yet, and the persistent journal opens only after this unit: no rotation + # while the service manager is still bringing the machine up. + booting=$(systemctl is-system-running 2>/dev/null || true) + case "$booting" in + initializing | starting) echo "the machine is still starting: nothing to rotate yet" ;; + *) + if [ "$want" != always ] && command -v journalctl >/dev/null 2>&1; then + journalctl --rotate || echo "the journal could not be rotated: its active files keep their huge pages until they are archived" >&2 + fi + ;; + esac + ;; +restore) + found=$(cat "$state/found" 2>/dev/null || true) + case "$found" in + always | madvise | never) + printf '%s\n' "$found" >"$sys/enabled" + echo "transparent huge pages: back to $found, as the machine had them" + ;; + *) echo "transparent huge pages: nothing recorded as found, left at $(mode)" ;; + esac + # Unassigned or shut down: nothing of the module is kept; the next start records what it finds again. + rm -f "$state/found" + ;; +*) + echo "usage: huge-pages apply|restore [ []]" >&2 + exit 2 + ;; +esac diff --git a/modules/kernel/files/mesh-kernel-huge-pages.service b/modules/kernel/files/mesh-kernel-huge-pages.service new file mode 100644 index 00000000..77a90595 --- /dev/null +++ b/modules/kernel/files/mesh-kernel-huge-pages.service @@ -0,0 +1,25 @@ +[Unit] +Description=Transparent huge pages as the mesh sets them on this machine +# Module kernel, novox/hq issue 746. Starting this unit sets the mode from the setting +# transparent-huge-pages; reloading it (the setting changed) sets the new mode; stopping it gives back the +# mode the machine had before. It runs early in boot, before the services that fill the page cache. +DefaultDependencies=no +After=local-fs.target +Before=sysinit.target shutdown.target +Conflicts=shutdown.target + +[Service] +Type=oneshot +RemainAfterExit=yes +ExecStart=${dir:state}/huge-pages apply +ExecReload=${dir:state}/huge-pages apply +ExecStop=${dir:state}/huge-pages restore +# It writes one file under /sys and its own state, and asks the journal daemon to rotate over its socket. +ProtectSystem=strict +ReadWritePaths=${dir:state} +ProtectHome=yes +PrivateNetwork=yes +NoNewPrivileges=yes + +[Install] +WantedBy=sysinit.target diff --git a/modules/kernel/module.json b/modules/kernel/module.json index 0f2a0f01..7305c945 100644 --- a/modules/kernel/module.json +++ b/modules/kernel/module.json @@ -4,11 +4,12 @@ "capabilities": [ "service-manager" ], - "settings": { + "config": { "transparent-huge-pages": { - "kind": "preference", - "default": "madvise", - "why": "madvise is the value the mesh chose for every node (novox/hq ADR 0381), not the always the distribution's kernel boots with; a machine that needs another value is given it as its own setting; madvise gives huge pages only to programs that ask for them, which stops a small write to a mapped file (the journal) dirtying and writing back a whole 2 MiB page on a machine whose disks are slow (novox/hq issue 746)" + "type": "string", + "description": "madvise is the value the mesh chose for every node (novox/hq ADR 0381), not the always the distribution's kernel boots with; a machine that needs another value is given it as its own setting; madvise gives huge pages only to programs that ask for them, which stops a small write to a mapped file (the journal) dirtying and writing back a whole 2 MiB page on a machine whose disks are slow (novox/hq issue 746)", + "trusted": true, + "default": "madvise" } }, "tools": [ @@ -27,14 +28,14 @@ "path": "${dir:state}/huge-pages", "mode": "0755", "trusted": true, - "content": "#!/bin/sh\n# Transparent huge pages as the mesh sets them on this machine (module kernel, novox/hq issue 746).\n# Written by the mesh from this machine's setting transparent-huge-pages and run by\n# mesh-kernel-huge-pages.service: `apply` at boot and whenever the setting changes, `restore` when the\n# unit stops (the module unassigned, or the machine shutting down). Replaced on every push; change the\n# setting, never this file. A change of the setting reloads the unit, which runs `apply` again: the mode\n# moves straight to the new value and never passes back through the one the machine was found with.\n#\n# huge-pages apply|restore [ []]\n#\n# The two directories are arguments only so the module's tests can run this against a stand-in tree.\nset -eu\n\nwant='${setting:transparent-huge-pages}'\nsys=/sys/kernel/mm/transparent_hugepage\nstate='${dir:state}'\n[ $# -ge 2 ] && sys=$2\n[ $# -ge 3 ] && state=$3\n\ncase \"$want\" in\nalways | madvise | never) ;;\n*)\n\techo \"the setting transparent-huge-pages is '$want'; the kernel takes always, madvise or never\" >&2\n\texit 2\n\t;;\nesac\n[ -f \"$sys/enabled\" ] || {\n\techo \"this kernel has no $sys/enabled: it was built without transparent huge pages, so there is nothing to set\" >&2\n\texit 1\n}\n\n# The mode in force is the word the kernel brackets: \"always [madvise] never\". A file holding one bare\n# word (the stand-in tree of the tests) is read as that word.\nmode() { sed -n -e 's/.*\\[\\([a-z]*\\)\\].*/\\1/p' -e t -e 's/^ *\\([a-z]*\\) *$/\\1/p' \"$sys/enabled\"; }\n\n[ $# -ge 1 ] || set -- usage\ncase \"$1\" in\napply)\n\tnow=$(mode)\n\t# What the machine had before the mesh first set it, kept once, so `restore` gives it back.\n\t[ -s \"$state/found\" ] || printf '%s\\n' \"$now\" >\"$state/found\"\n\tif [ \"$now\" = \"$want\" ]; then\n\t\techo \"transparent huge pages: $want, as set\"\n\t\texit 0\n\tfi\n\tprintf '%s\\n' \"$want\" >\"$sys/enabled\"\n\tgot=$(mode)\n\t[ \"$got\" = \"$want\" ] || {\n\t\techo \"wrote $want to $sys/enabled and the kernel says $got\" >&2\n\t\texit 1\n\t}\n\techo \"transparent huge pages: $now -> $want\"\n\t# A file folio already in the page cache keeps its size: the active journal files, mapped by journald\n\t# and by every reader, stay in 2 MiB folios until they are archived (issue 746). Rotating starts new\n\t# files that are mapped under the new mode. A failed rotation is said and does not fail the setting.\n\t# At boot nothing is in huge folios yet, and the persistent journal opens only after this unit: no rotation\n\t# while the service manager is still bringing the machine up.\n\tbooting=$(systemctl is-system-running 2>/dev/null || true)\n\tcase \"$booting\" in\n\tinitializing | starting) echo \"the machine is still starting: nothing to rotate yet\" ;;\n\t*)\n\t\tif [ \"$want\" != always ] && command -v journalctl >/dev/null 2>&1; then\n\t\t\tjournalctl --rotate || echo \"the journal could not be rotated: its active files keep their huge pages until they are archived\" >&2\n\t\tfi\n\t\t;;\n\tesac\n\t;;\nrestore)\n\tfound=$(cat \"$state/found\" 2>/dev/null || true)\n\tcase \"$found\" in\n\talways | madvise | never)\n\t\tprintf '%s\\n' \"$found\" >\"$sys/enabled\"\n\t\techo \"transparent huge pages: back to $found, as the machine had them\"\n\t\t;;\n\t*) echo \"transparent huge pages: nothing recorded as found, left at $(mode)\" ;;\n\tesac\n\t# Unassigned or shut down: nothing of the module is kept; the next start records what it finds again.\n\trm -f \"$state/found\"\n\t;;\n*)\n\techo \"usage: huge-pages apply|restore [ []]\" >&2\n\texit 2\n\t;;\nesac\n" + "template": "files/huge-pages.sh" }, { "id": "unit", "type": "file", "path": "/etc/systemd/system/mesh-kernel-huge-pages.service", "mode": "0644", - "content": "[Unit]\nDescription=Transparent huge pages as the mesh sets them on this machine\n# Module kernel, novox/hq issue 746. Starting this unit sets the mode from the setting\n# transparent-huge-pages; reloading it (the setting changed) sets the new mode; stopping it gives back the\n# mode the machine had before. It runs early in boot, before the services that fill the page cache.\nDefaultDependencies=no\nAfter=local-fs.target\nBefore=sysinit.target shutdown.target\nConflicts=shutdown.target\n\n[Service]\nType=oneshot\nRemainAfterExit=yes\nExecStart=${dir:state}/huge-pages apply\nExecReload=${dir:state}/huge-pages apply\nExecStop=${dir:state}/huge-pages restore\n# It writes one file under /sys and its own state, and asks the journal daemon to rotate over its socket.\nProtectSystem=strict\nReadWritePaths=${dir:state}\nProtectHome=yes\nPrivateNetwork=yes\nNoNewPrivileges=yes\n\n[Install]\nWantedBy=sysinit.target\n" + "template": "files/mesh-kernel-huge-pages.service" }, { "id": "apply", @@ -65,6 +66,11 @@ "loads": [ "kernel-tools" ] + }, + { + "name": "files", + "kind": "archive", + "from": "files" } ] } diff --git a/modules/keycloak/cmd/keycloak-provider/main.go b/modules/keycloak/cmd/keycloak-provider/main.go index ee6e9d0e..d8f9334f 100644 --- a/modules/keycloak/cmd/keycloak-provider/main.go +++ b/modules/keycloak/cmd/keycloak-provider/main.go @@ -9,6 +9,7 @@ package main import ( "context" "os" + "time" stdio "git.novox.be/novox/mesh-sdk/go" ) @@ -43,12 +44,33 @@ func main() { go guard.Run(context.Background()) var h *Harness - if receives := os.Getenv("MESH_RECEIVES"); receives == "" { - say("MESH_RECEIVES is not set — the provisioner cannot run without it") - } else if issuer, err := Issuer(os.Getenv); err != nil { + if issuer, err := Issuer(os.Getenv); err != nil { say("%v — the provisioner cannot run without it", err) } else if realm, err := RealmOf(issuer); err != nil { say("%v — the provisioner cannot run without it", err) + } else if consumers := handedConsumersFile(os.Getenv, os.Stat); consumers != "" { + // Handed to the provisioner (novox/hq ADR 0373): the provider helper answers the consumers its + // machine's client fills the file with; the grant directory is not read. Retirement is the + // harness's, never run, so its guards and its tools stand (issue 805). + h = &Harness{ + Resource: ProvisionName, + Adapter: provisioner{clients: OidcClients{KC: kc, Realm: realm}, guard: guard, announce: announce, log: logStderr}, + Log: logStderr, + Announce: announce, + Node: os.Getenv("MESH_NODE"), + } + run, p, bus, err := moved(consumers, h, issuer) + if err != nil { + say("%v", err) + } else { + go func() { + if err := run.Run(context.Background(), p, bus); err != nil { + say("the provider helper stopped: %v", err) + } + }() + } + } else if receives := os.Getenv("MESH_RECEIVES"); receives == "" { + say("MESH_RECEIVES is not set — the provisioner cannot run without it") } else { h = &Harness{ Resource: "oidc-client", @@ -62,9 +84,43 @@ func main() { } go h.Run(context.Background()) } + // Started on the grant path while the module is to be handed, it restarts itself when its client first fills + // the consumers' file, so the first fill moves it onto the helper without a hand restart (issue 805): `restart-on` + // names a service or a container, and the only one here is Keycloak's own. + if at := os.Getenv("MESH_CONSUMERS_FILE"); at != "" && handedConsumersFile(os.Getenv, os.Stat) == "" { + go exitWhenFilled(at, os.Stat, 5*time.Second, func() { + say("%s is filled: restarting to answer from it", at) + os.Exit(0) + }) + } // The retirement tools beside keycloak's own (novox/hq ADR 0230). if err := stdio.Serve("", append(Tools(kc, guard), RetirementTools(h)...)); err != nil { say("%v", err) os.Exit(1) } } + +// handedConsumersFile is the consumers' file when keycloak is handed to the provisioner: the file MESH_CONSUMERS_FILE +// names, which the controller places only where it hands the module. Empty, where it is not set or the file is not +// there, and keycloak then reads the grant directory exactly as it always did (issue 805). +func handedConsumersFile(getenv func(string) string, stat func(string) (os.FileInfo, error)) string { + at := getenv("MESH_CONSUMERS_FILE") + if at == "" { + return "" + } + if _, err := stat(at); err != nil { + return "" + } + return at +} + +// exitWhenFilled calls filled once the file is there, asking every so often. +func exitWhenFilled(at string, stat func(string) (os.FileInfo, error), every time.Duration, filled func()) { + for { + if _, err := stat(at); err == nil { + filled() + return + } + time.Sleep(every) + } +} diff --git a/modules/keycloak/cmd/keycloak-provider/moved.go b/modules/keycloak/cmd/keycloak-provider/moved.go new file mode 100644 index 00000000..34ce390f --- /dev/null +++ b/modules/keycloak/cmd/keycloak-provider/moved.go @@ -0,0 +1,274 @@ +package main + +// keycloak handed to the provisioner (novox/hq ADR 0373, ADR 0349 clause 12, issue 805): where its consumers file +// is set, keycloak runs on the SDK's provider helper instead of the harness reading the controller's grant files. +// +// The helper learns who binds keycloak from the plans, composes each consumer's entry and reads its credential +// through ConsumersFile, the file this machine's provisioner client fills. It leaves the rest to the two hooks +// here: Make brings the consumer's OIDC client into being exactly as the harness's adapter does, and Gone leaves +// retirement to the harness's guards (retirement.go): the client is disabled and marked to delete, never removed. + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "sync" + "time" + + "git.novox.be/novox/mesh-sdk/go/provider" +) + +// ProvisionName is what keycloak provides, as its manifest says. +const ProvisionName = "oidc-client" + +// servedValues is what its manifest serves (`serves.oidc-client`): the paths as they are, and the issuer as its own +// config says, which is the one the code reads (Issuer), not a value the helper reads from the manifest. +func servedValues(issuer string) map[string]any { + return map[string]any{ + "issuer": issuer, + "authorization-path": "/protocol/openid-connect/auth", + "token-path": "/protocol/openid-connect/token", + "userinfo-path": "/protocol/openid-connect/userinfo", + } +} + +// valuesOf is what the consumer's grant file gave the harness as its values: what it contributes (label, endpoint, +// callback), with the names the controller composed for it (name, internal-name) over them (issue 792). +func valuesOf(e provider.Entry) map[string]any { + values := map[string]any{} + for k, v := range e.Contributes { + values[k] = v + } + for k, name := range e.Names { + values[k] = name + } + return values +} + +// movedRun is keycloak on the provider helper, with the harness's own memory and retirement behind it: what is held, +// what is counted unasked, what waits for a person (retirement.go). The harness is never Run here; the helper's +// Make and Gone and the loop below drive the same state, so the guards are the harness's, not a copy. +type movedRun struct { + h *Harness + + mu sync.Mutex // guards assignments and loaded + assignments map[string][]byte + loaded bool + verified map[string]time.Time +} + +func newMovedRun(h *Harness) *movedRun { + h.mu.Lock() + h.init() + h.mu.Unlock() + return &movedRun{h: h, assignments: map[string][]byte{}, verified: map[string]time.Time{}} +} + +// Make brings one consumer's client into being under its client id (its plan's `as`) and the vault's credential as +// its secret, as the harness does: at once when it is new or changed, and otherwise only when Keycloak no longer +// holds it, which is asked every VerifyEvery. +func (m *movedRun) Make(ctx context.Context, c provider.Consumer) error { + h, e := m.h, c.Entry + if e.As == "" { + return fmt.Errorf("%s has no client id to make", e.Key()) + } + if len(c.Credential) == 0 { + return fmt.Errorf("%s has no credential to make its client with", e.Key()) + } + h.mu.Lock() + defer h.mu.Unlock() + as, password := e.As, string(c.Credential) + p := Provision{As: as, Password: password, Values: valuesOf(e), Consumer: e.Consumer} + hash := hashOf(as, password, p.Values, nil) + if was, ok := h.applied[as]; ok && was.hash == hash { + if h.Now().Sub(m.verified[as]) < h.VerifyEvery { + return nil + } + m.verified[as] = h.Now() + hctx, cancel := context.WithTimeout(ctx, h.HoldsTimeout) + held, err := h.Adapter.Holds(hctx, p) + cancel() + if err != nil { + text := scrub(err, password) + h.failed(as, e.Consumer, h.classOf(err, text), text) + return errors.New(text) + } + if held { + h.succeeded(as) + return nil + } + h.say("%s: Keycloak no longer holds it; applying again", as) + } + if err := h.Adapter.Create(ctx, p); err != nil { + text := scrub(err, password) + h.failed(as, e.Consumer, h.classOf(err, text), text) + return errors.New(text) + } + h.succeeded(as) + h.applied[as] = appliedEntry{hash: hash, node: e.Consumer} + m.verified[as] = h.Now() + h.reenabled(as, e.Consumer) + return nil +} + +// Gone is the helper asking whether it may forget a consumer that no longer binds. **It never retires**: retiring is +// the harness's rule, the same unasked set over StablePasses passes and StableFor, at most RetireAtOnce at a time +// (retirement.go, ADR 0230), run by retire below. So a consumer missing from one read of the plans loses nothing: +// the helper is told no, and asks again at its next pass, until the client is retired, or asked for again. +func (m *movedRun) Gone(_ context.Context, e provider.Entry) error { + h := m.h + as := e.As + if as == "" { + return nil // refused: nothing was made for it + } + // The plans as they are now, not as the last retirement pass read them: with none read, nothing is let go. + want, read := m.asked(h.Node) + asked := want[as] + h.mu.Lock() + defer h.mu.Unlock() + _, held := h.applied[as] + _, retired := h.r.retired[as] + switch { + case !read: + case asked: + return nil // another holder of the same client id still binds: nothing to retire + case retired, !held && h.r.seeded: + return nil + } + return fmt.Errorf("%s is kept until its retirement is stable (%d passes and %s) and within the bound, or a "+ + "person approves it", as, h.StablePasses, h.StableFor) +} + +// plan is as much of an assignment as is read to learn who binds keycloak. +type plan struct { + Node string `json:"node"` + Module string `json:"module"` + Provisioning *struct { + Provisions []struct { + Provision string `json:"provision"` + Provider string `json:"provider"` + ProviderModule string `json:"provider-module"` + // As is who the consumer is at its provider's end, its slug first (issue 788). + As string `json:"as"` + } `json:"provisions"` + } `json:"provisioning"` +} + +// asked is the client ids the plans bind keycloak for, and whether they could all be read: a plan that is not the +// controller's, or no read yet, is not an answer, so nothing is counted unasked from it (novox/hq issue 241). +func (m *movedRun) asked(node string) (map[string]bool, bool) { + m.mu.Lock() + defer m.mu.Unlock() + if !m.loaded || len(m.assignments) == 0 { + return nil, false + } + want := map[string]bool{} + for key, raw := range m.assignments { + var a plan + if err := json.Unmarshal(raw, &a); err != nil || a.Node+"."+a.Module != key { + return nil, false + } + if a.Provisioning == nil { + continue + } + for _, b := range a.Provisioning.Provisions { + if b.Provider == node && b.ProviderModule == "keycloak" && b.Provision == ProvisionName { + as := b.As + if as == "" { + as = provider.ConsumerIdentity(a.Node, a.Module) // a plan that names none + } + want[as] = true + } + } + } + return want, true +} + +// retire is one pass of the harness's retirement over what the plans bind now. +func (m *movedRun) retire(ctx context.Context) { + h := m.h + want, ok := m.asked(h.Node) + h.mu.Lock() + defer h.mu.Unlock() + if !ok { + // Not a result: the passes that must agree before anything is retired start again. + h.r.key, h.r.count = "", 0 + return + } + h.seed(ctx, want) + h.retireUnasked(ctx, want) + h.r.lastWant = want + for as := range h.failing { + if !want[as] { + delete(h.failing, as) + } + } + for as := range h.trouble { + if _, held := h.applied[as]; !want[as] && !held { + h.recovered(as, "no longer asked for") + } + } +} + +// watch keeps the plans as the bus holds them, from the first read on. +func (m *movedRun) watch(ctx context.Context, bus provider.Bus) error { + err := bus.Watch(ctx, provider.AssignmentsBucket, func(key string, value []byte) { + m.mu.Lock() + defer m.mu.Unlock() + if value == nil { + delete(m.assignments, key) + } else { + m.assignments[key] = append([]byte(nil), value...) + } + }) + if err == nil { + m.mu.Lock() + m.loaded = true + m.mu.Unlock() + } + return err +} + +// Run answers the consumers through the helper and retires through the harness's rules, until ctx ends. +func (m *movedRun) Run(ctx context.Context, p *provider.Provider, bus provider.Bus) error { + if err := m.watch(ctx, bus); err != nil { + return err + } + go func() { + for { + m.retire(ctx) + select { + case <-ctx.Done(): + return + case <-time.After(m.h.Every): + } + } + }() + return p.Run(ctx) +} + +// moved is keycloak handed to the provisioner: the helper answering its consumers, and the harness (never run) whose +// retirement and tools it shares. consumersFile is where its machine's client fills its consumers, issuer the one +// keycloak's own config gives (Issuer), served to every consumer. +func moved(consumersFile string, h *Harness, issuer string) (*movedRun, *provider.Provider, provider.Bus, error) { + if consumersFile == "" { + return nil, nil, nil, errors.New("no consumers' file") + } + run := newMovedRun(h) + bus := provider.StateBus("keycloak") + return run, provider.New(provider.Options{ + Node: h.Node, + Module: "keycloak", + Offers: map[string]provider.Offer{ProvisionName: { + Credential: provider.PerConsumer, + Serves: servedValues(issuer), + }}, + Bus: bus, + Vault: provider.ConsumersFile{Paths: []string{consumersFile}}, + Make: run.Make, + Gone: run.Gone, + Every: h.Every, + Logf: func(f string, a ...any) { h.say(f, a...) }, + }), bus, nil +} diff --git a/modules/keycloak/cmd/keycloak-provider/moved_test.go b/modules/keycloak/cmd/keycloak-provider/moved_test.go new file mode 100644 index 00000000..cfd33f39 --- /dev/null +++ b/modules/keycloak/cmd/keycloak-provider/moved_test.go @@ -0,0 +1,394 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + "os" + "path/filepath" + "reflect" + "sync" + "testing" + "time" + + "git.novox.be/novox/mesh-sdk/go/provider" +) + +const testIssuer = "https://id.example.org/realms/Novox" + +// bucketBus is the helper's bus over in-memory buckets. +type bucketBus struct { + mu sync.Mutex + buckets map[string]map[string][]byte +} + +func (b *bucketBus) Watch(_ context.Context, bucket string, handle func(string, []byte)) error { + b.mu.Lock() + defer b.mu.Unlock() + for k, v := range b.buckets[bucket] { + handle(k, v) + } + return nil +} + +func (b *bucketBus) Put(_ context.Context, bucket, key string, value []byte) error { + b.mu.Lock() + defer b.mu.Unlock() + if b.buckets[bucket] == nil { + b.buckets[bucket] = map[string][]byte{} + } + b.buckets[bucket][key] = append([]byte(nil), value...) + return nil +} + +func (b *bucketBus) Delete(_ context.Context, bucket, key string) error { + b.mu.Lock() + defer b.mu.Unlock() + delete(b.buckets[bucket], key) + return nil +} + +// bind is one consumer's assignment, whose plan names this machine's keycloak as the provider and carries what the +// consumer contributes (issue 776). +func (b *bucketBus) bind(node, module string, contributes map[string]any) { + b.bindNamed(node, module, contributes, nil) +} + +// bindNamed is bind with the names the controller composed for the consumer (issue 792). +func (b *bucketBus) bindNamed(node, module string, contributes map[string]any, names map[string]string) { + plan, _ := json.Marshal(map[string]any{"node": node, "module": module, "provisioning": map[string]any{ + "provisions": []any{map[string]any{"provision": ProvisionName, "provider": "novox", "provider-module": "keycloak", + "contributes": contributes, "names": names}}}}) + if err := b.Put(context.Background(), provider.AssignmentsBucket, node+"."+module, plan); err != nil { + panic(err) + } +} + +// consumersFile writes the file the machine's client fills: one credential per consumer. +func consumersFile(t *testing.T, bound map[[2]string]string) string { + var list []provider.ConsumerCredential + for k, password := range bound { + list = append(list, provider.ConsumerCredential{Consumer: k[0], Module: k[1], + Credential: provider.PairName("novox", ProvisionName, k[0], k[1], ""), Value: password}) + } + raw, _ := json.Marshal(list) + at := filepath.Join(t.TempDir(), "consumers.json") + if err := os.WriteFile(at, raw, 0o600); err != nil { + t.Fatal(err) + } + return at +} + +// movedWorld is keycloak on the helper over an in-memory bus, with a recorder for a server and a clock to move. +type movedWorld struct { + t *testing.T + a *recorder + bus *bucketBus + h *Harness + run *movedRun + p *provider.Provider + now time.Time + file string +} + +func newMovedWorld(t *testing.T, bound map[[2]string]string) *movedWorld { + w := &movedWorld{t: t, a: &recorder{}, bus: &bucketBus{buckets: map[string]map[string][]byte{}}, + now: time.Date(2026, 10, 15, 12, 0, 0, 0, time.UTC)} + w.file = consumersFile(t, bound) + w.h = &Harness{Resource: ProvisionName, Adapter: w.a, Node: "novox", Now: func() time.Time { return w.now }, + Log: func(string, ...any) {}} + var bus provider.Bus + w.run, w.p, bus, _ = moved(w.file, w.h, testIssuer) + _ = bus + if err := w.run.watch(ctx, w.bus); err != nil { + t.Fatal(err) + } + return w +} + +// pass is one helper pass and one retirement pass, five seconds after the last. +func (w *movedWorld) pass() []provider.Outcome { + w.now = w.now.Add(5 * time.Second) + // the plans as the bus holds them now, as the watch would hand them + w.run.assignments = map[string][]byte{} + for k, v := range w.bus.buckets[provider.AssignmentsBucket] { + w.run.assignments[k] = v + } + // the helper reads its buckets from the in-memory bus + w.p = provider.New(provider.Options{Node: "novox", Module: "keycloak", Bus: w.bus, + Vault: provider.ConsumersFile{Paths: []string{w.file}}, + Offers: map[string]provider.Offer{ProvisionName: {Credential: provider.PerConsumer, + Serves: servedValues(testIssuer)}}, + Make: w.run.Make, Gone: w.run.Gone, Logf: func(string, ...any) {}}) + out, err := w.p.Pass(ctx) + if err != nil { + w.t.Fatal(err) + } + w.run.retire(ctx) + return out +} + +func bindings(n int) (map[[2]string]string, []string) { + bound := map[[2]string]string{} + var logins []string + for i := 1; i <= n; i++ { + module := fmt.Sprintf("app%d", i) + bound[[2]string{"ace", module}] = "pw-" + module + logins = append(logins, provider.ConsumerIdentity("ace", module)) + } + return bound, logins +} + +// grafanaContributes and grafanaNames are what grafana's plan entry carries for keycloak: its contribution, and the +// names the controller composed for it (issues 776, 792). +var grafanaContributes = map[string]any{"label": "grafana", "endpoint": "web", "callback": "/login/generic_oauth"} +var grafanaNames = map[string]string{"name": "grafana.example.org", "internal-name": "grafana.home.internal"} + +// The consumers' file makes the same call on the adapter as the equivalent grant file does (the harness): the same +// client id, secret, and the values the redirect URIs are made from. +func TestTheConsumersFileMakesWhatTheGrantFileMakes(t *testing.T) { + grafana := provider.ConsumerIdentity("home", "grafana") + values := map[string]any{"name": grafanaNames["name"], "internal-name": grafanaNames["internal-name"]} + for k, v := range grafanaContributes { + values[k] = v + } + + // Today: the grant file, through the harness. + g := newWorld(t) + g.give(map[string]any{"as": grafana, "node": "home", "values": values}) + g.h.Reconcile(ctx) + + // The consumers' file, through the provider helper. + w := newMovedWorld(t, map[[2]string]string{{"home", "grafana"}: "pw-" + grafana}) + w.bus.bindNamed("home", "grafana", grafanaContributes, grafanaNames) + for _, o := range w.pass() { + if o.Status != provider.Answered { + t.Fatalf("%+v", o) + } + } + if len(w.a.created) != 1 || len(g.a.created) != 1 { + t.Fatalf("created %d through the helper, %d through the harness, want 1 each", len(w.a.created), len(g.a.created)) + } + c, want := w.a.created[0], g.a.created[0] + if c.As != want.As || c.Password != want.Password || c.Consumer != want.Consumer || + !reflect.DeepEqual(orEmpty(c.Values), orEmpty(want.Values)) { + t.Fatalf("made %+v, the grant file makes %+v", c, want) + } + if _, redirects, err := RedirectsOf(c.Values); err != nil || + !reflect.DeepEqual(redirects, []string{"https://grafana.example.org/login/generic_oauth", "https://grafana.home.internal/login/generic_oauth"}) { + t.Fatalf("redirects %v, %v", redirects, err) + } + // The entry tells grafana what keycloak serves, the issuer from its own config. + var entry provider.Entry + if err := json.Unmarshal(w.bus.buckets[provider.BucketOf("keycloak")]["home.grafana.oidc-client"], &entry); err != nil { + t.Fatal(err) + } + if entry.Values["issuer"] != testIssuer || entry.Values["token-path"] != "/protocol/openid-connect/token" || + entry.Values["authorization-path"] != "/protocol/openid-connect/auth" || entry.Values["userinfo-path"] != "/protocol/openid-connect/userinfo" { + t.Fatalf("served %v", entry.Values) + } + + // Unchanged, and held: nothing is made again, within a minute or after it. No longer held: made again. + w.a.held = true + w.pass() + w.now = w.now.Add(2 * time.Minute) + w.pass() + if len(w.a.created) != 1 { + t.Fatalf("an unchanged pass created again: %d", len(w.a.created)) + } + w.a.held = false + w.now = w.now.Add(2 * time.Minute) + w.pass() + if len(w.a.created) != 2 { + t.Fatalf("not made again: %d", len(w.a.created)) + } +} + +// Against a Keycloak, both paths leave the same client. +func TestBothPathsLeaveTheSameClientInKeycloak(t *testing.T) { + grafana := provider.ConsumerIdentity("home", "grafana") + values := map[string]any{"name": grafanaNames["name"], "internal-name": grafanaNames["internal-name"]} + for k, v := range grafanaContributes { + values[k] = v + } + made := func(through func(p provisioner)) Rep { + f, o := oidcWorld(t) + through(provisioner{clients: o, announce: func(string, map[string]any) {}, log: func(string, ...any) {}}) + return only(t, f, grafana) + } + viaGrant := made(func(p provisioner) { + g := newWorld(t) + g.h.Adapter = p + g.give(map[string]any{"as": grafana, "node": "home", "values": values}) + g.h.Reconcile(ctx) + }) + viaFile := made(func(p provisioner) { + w := newMovedWorld(t, map[[2]string]string{{"home", "grafana"}: "pw-" + grafana}) + w.h.Adapter = p + w.bus.bindNamed("home", "grafana", grafanaContributes, grafanaNames) + w.pass() + }) + // Keycloak's own ids differ between two runs; nothing the mesh sets does. + for _, c := range []Rep{viaGrant, viaFile} { + delete(c, "id") + raw, _ := json.Marshal(c) + var again Rep + _ = json.Unmarshal(raw, &again) + for k := range c { + delete(c, k) + } + for k, v := range again { + c[k] = v + } + if mappers, ok := c["protocolMappers"].([]any); ok { + for _, m := range mappers { + delete(m.(map[string]any), "id") + } + } + } + if !reflect.DeepEqual(viaGrant, viaFile) { + t.Fatalf("through the file %v, through the grant %v", viaFile, viaGrant) + } +} + +// A consumer that is no longer bound is kept until the harness's retirement is stable, then retired, never deleted. +func TestAConsumerMissingFromOneReadRetiresNothing(t *testing.T) { + bound, logins := bindings(2) + w := newMovedWorld(t, bound) + w.bus.bindNamed("ace", "app1", grafanaContributes, grafanaNames) + w.bus.bindNamed("ace", "app2", grafanaContributes, grafanaNames) + w.pass() + + saved := w.bus.buckets[provider.AssignmentsBucket]["ace.app2"] + w.bus.Delete(ctx, provider.AssignmentsBucket, "ace.app2") + w.pass() + if len(w.a.removed) != 0 { + t.Fatalf("retired %v on one read", w.a.removed) + } + w.bus.Put(ctx, provider.AssignmentsBucket, "ace.app2", saved) + for i := 0; i < 8; i++ { + w.pass() + } + w.now = w.now.Add(15 * time.Minute) + w.pass() + if len(w.a.removed) != 0 { + t.Fatalf("retired %v", w.a.removed) + } + + w.bus.Delete(ctx, provider.AssignmentsBucket, "ace.app2") + for i := 0; i < 6; i++ { + w.pass() + } + if len(w.a.removed) != 0 { + t.Fatalf("retired %v before the time was out", w.a.removed) + } + w.now = w.now.Add(11 * time.Minute) + w.pass() + w.pass() + if !reflect.DeepEqual(w.a.removed, []string{logins[1]}) || len(w.a.deleted) != 0 { + t.Fatalf("retired %v, deleted %v", w.a.removed, w.a.deleted) + } + if _, kept := w.bus.buckets[provider.BucketOf("keycloak")]["ace.app2.oidc-client"]; kept { + t.Fatal("the entry stayed after the client was retired") + } +} + +// A plan list that was never read is not nobody asking. +func TestNoPlansReadRetiresNothing(t *testing.T) { + bound, _ := bindings(1) + w := newMovedWorld(t, bound) + w.bus.bindNamed("ace", "app1", grafanaContributes, grafanaNames) + w.pass() + w.bus.Delete(ctx, provider.AssignmentsBucket, "ace.app1") + for i := 0; i < 8; i++ { + w.pass() + } + w.now = w.now.Add(20 * time.Minute) + w.pass() + if len(w.a.removed) != 0 { + t.Fatalf("retired %v with no plans read", w.a.removed) + } +} + +// A consumer whose credential is not in the file yet is waited for: nothing is made for it. +func TestAConsumerWithoutACredentialIsWaitedFor(t *testing.T) { + w := newMovedWorld(t, map[[2]string]string{}) + w.bus.bindNamed("home", "grafana", grafanaContributes, grafanaNames) + outcomes := w.pass() + if len(outcomes) != 1 || outcomes[0].Status != provider.Waiting || len(w.a.created) != 0 { + t.Fatalf("%+v, created %v", outcomes, w.a.created) + } +} + +// A consumer whose plan names its client id (a slugged module) keeps it, and the retirement counts it as asked for. +func TestAConsumerNamedByItsPlanKeepsItsClientId(t *testing.T) { + agent := provider.ConsumerIdentity("home", "dashboard") + w := newMovedWorld(t, map[[2]string]string{{"home", "grafana-2"}: "pw"}) + plan, _ := json.Marshal(map[string]any{"node": "home", "module": "grafana-2", "provisioning": map[string]any{ + "provisions": []any{map[string]any{"provision": ProvisionName, "provider": "novox", "provider-module": "keycloak", + "as": agent, "contributes": grafanaContributes, "names": grafanaNames}}}}) + if err := w.bus.Put(ctx, provider.AssignmentsBucket, "home.grafana-2", plan); err != nil { + t.Fatal(err) + } + for _, o := range w.pass() { + if o.Status != provider.Answered { + t.Fatalf("%+v", o) + } + } + if len(w.a.created) != 1 || w.a.created[0].As != agent { + t.Fatalf("made %+v, want one under %s", w.a.created, agent) + } + for i := 0; i < 8; i++ { + w.pass() + } + w.now = w.now.Add(15 * time.Minute) + w.pass() + w.pass() + if len(w.a.removed) != 0 { + t.Fatalf("retired %v: the plan's client id was counted unasked", w.a.removed) + } +} + +// Where the file is not there, keycloak takes the grant directory, as before; where it is, the helper. +func TestTheGrantPathIsTakenWhereTheConsumersFileIsAbsent(t *testing.T) { + dir := t.TempDir() + at := filepath.Join(dir, "consumers.json") + env := func(k string) string { + if k == "MESH_CONSUMERS_FILE" { + return at + } + return "" + } + if got := handedConsumersFile(env, os.Stat); got != "" { + t.Fatalf("took %q for a file that is not there", got) + } + if got := handedConsumersFile(func(string) string { return "" }, os.Stat); got != "" { + t.Fatalf("took %q with no environment", got) + } + if err := os.WriteFile(at, []byte("[]"), 0o600); err != nil { + t.Fatal(err) + } + if got := handedConsumersFile(env, os.Stat); got != at { + t.Fatalf("took %q, want %q", got, at) + } +} + +// The grant path restarts itself when the consumers file is first filled. +func TestTheGrantPathLeavesWhenTheConsumersFileIsFilled(t *testing.T) { + at := filepath.Join(t.TempDir(), "consumers.json") + done := make(chan struct{}) + go exitWhenFilled(at, os.Stat, time.Millisecond, func() { close(done) }) + select { + case <-done: + t.Fatal("left before the file was there") + case <-time.After(20 * time.Millisecond): + } + if err := os.WriteFile(at, []byte("[]"), 0o600); err != nil { + t.Fatal(err) + } + select { + case <-done: + case <-time.After(2 * time.Second): + t.Fatal("did not leave when the file was filled") + } +} diff --git a/modules/keycloak/cmd/keycloak-provider/templates_test.go b/modules/keycloak/cmd/keycloak-provider/templates_test.go new file mode 100644 index 00000000..7d0718b5 --- /dev/null +++ b/modules/keycloak/cmd/keycloak-provider/templates_test.go @@ -0,0 +1,64 @@ +package main + +import ( + "encoding/json" + "os" + "path/filepath" + "testing" +) + +// The module's directory, where its manifest and its files/ are. +const moduleDir = "../.." + +// templateText is the text a file resource places on the machine: its template's, in the module's files/ +// (novox/hq ADR 0349 §2). A resource that names no template, or a template that is not there, fails the test. +func templateText(t *testing.T, r map[string]any) string { + t.Helper() + name, _ := r["template"].(string) + if name == "" { + t.Fatalf("resource %v names no template", r["id"]) + } + raw, err := os.ReadFile(filepath.Join(moduleDir, filepath.FromSlash(name))) + if err != nil { + t.Fatalf("resource %v names %s: %v", r["id"], name, err) + } + return string(raw) +} + +// No file resource carries its text inline: each names its template in files/, and the build packs files/ +// into the one archive the controller reads the templates back from (novox/hq ADR 0349 §2). +func TestEveryFileTakesItsTextFromATemplate(t *testing.T) { + raw, err := os.ReadFile(filepath.Join(moduleDir, "module.json")) + if err != nil { + t.Fatal(err) + } + var m struct { + Resources []map[string]any `json:"resources"` + Build struct { + Artifacts []map[string]any `json:"artifacts"` + } `json:"build"` + } + if err := json.Unmarshal(raw, &m); err != nil { + t.Fatal(err) + } + for _, r := range m.Resources { + if _, inline := r["content"]; inline { + t.Errorf("resource %v keeps its text inline: keep it in files/ and name it as the template", r["id"]) + } + if r["type"] == "file" { + templateText(t, r) + } + } + packs := 0 + for _, a := range m.Build.Artifacts { + if a["kind"] == "archive" && a["from"] == "files" { + packs++ + if a["name"] != "files" { + t.Errorf("the templates archive is named %v, not files", a["name"]) + } + } + } + if packs != 1 { + t.Errorf("the build packs files/ into %d archives, not one", packs) + } +} diff --git a/modules/keycloak/files/admin.env b/modules/keycloak/files/admin.env new file mode 100644 index 00000000..3e52ae5f --- /dev/null +++ b/modules/keycloak/files/admin.env @@ -0,0 +1,2 @@ +KEYCLOAK_ADMIN=admin +KEYCLOAK_ADMIN_PASSWORD=${secret:admin} diff --git a/modules/keycloak/files/config.json b/modules/keycloak/files/config.json new file mode 100644 index 00000000..0967ef42 --- /dev/null +++ b/modules/keycloak/files/config.json @@ -0,0 +1 @@ +{} diff --git a/modules/keycloak/files/consumers.json b/modules/keycloak/files/consumers.json new file mode 100644 index 00000000..b0af957c --- /dev/null +++ b/modules/keycloak/files/consumers.json @@ -0,0 +1 @@ +${consumers:oidc-client} diff --git a/modules/keycloak/files/database.env b/modules/keycloak/files/database.env new file mode 100644 index 00000000..454ee994 --- /dev/null +++ b/modules/keycloak/files/database.env @@ -0,0 +1,3 @@ +KC_DB_URL=jdbc:postgresql://${bound:postgres-database:at}:${bound:postgres-database:port}/${bound:postgres-database:as} +KC_DB_USERNAME=${bound:postgres-database:as} +KC_DB_PASSWORD=${secret:postgres-database} diff --git a/modules/keycloak/files/hostname.env b/modules/keycloak/files/hostname.env new file mode 100644 index 00000000..92f061a1 --- /dev/null +++ b/modules/keycloak/files/hostname.env @@ -0,0 +1 @@ +KC_HOSTNAME=https://${bound:route:name} diff --git a/modules/keycloak/go.mod b/modules/keycloak/go.mod index a7950e5e..f8dcc045 100644 --- a/modules/keycloak/go.mod +++ b/modules/keycloak/go.mod @@ -2,4 +2,4 @@ module keycloak go 1.25.0 -require git.novox.be/novox/mesh-sdk/go v0.1.7 +require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261015210729-d3e182503c19 diff --git a/modules/keycloak/go.sum b/modules/keycloak/go.sum index b474419e..9a531b0e 100644 --- a/modules/keycloak/go.sum +++ b/modules/keycloak/go.sum @@ -1,2 +1,2 @@ -git.novox.be/novox/mesh-sdk/go v0.1.7 h1:C0sTQmtTiyYH7bnqZb7PusXnqA37gKuT7Nqjn9gG47w= -git.novox.be/novox/mesh-sdk/go v0.1.7/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261015210729-d3e182503c19 h1:mAMt3KFmrHzDiBv4MLL94opWcG9y73ISMrBUyisapQE= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261015210729-d3e182503c19/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY= diff --git a/modules/keycloak/module.json b/modules/keycloak/module.json index 8bbf9d82..03749aa6 100644 --- a/modules/keycloak/module.json +++ b/modules/keycloak/module.json @@ -1,6 +1,7 @@ { "module": "keycloak", "version": "1", + "moved": true, "upgrade": { "policy": "record", "why": "every person's sign-in to every site goes through it, and its new version migrates its database on start: a person takes each build, after a backup (hq ADR 0236)" @@ -63,7 +64,7 @@ "authorization-path": "/protocol/openid-connect/auth", "token-path": "/protocol/openid-connect/token", "userinfo-path": "/protocol/openid-connect/userinfo", - "issuer": "${setting:issuer}" + "issuer": "${config:issuer}" } }, "receives": { @@ -84,6 +85,32 @@ } } }, + "config": { + "url": { + "type": "string", + "description": "The address the module's provider reaches Keycloak's admin API at, as a URL; empty is this machine's Keycloak on its port.", + "trusted": true, + "default": "" + }, + "user": { + "type": "string", + "description": "The Keycloak administrator the module's provider signs in as; empty is admin.", + "trusted": true, + "default": "" + }, + "realm": { + "type": "string", + "description": "The realm the module's provider makes its consumers' clients in; empty is master.", + "trusted": true, + "default": "" + }, + "issuer": { + "type": "string", + "description": "The issuer Keycloak's tokens name and oidc consumers are told, as the realm's URL, such as https://keycloak.example.org/realms/Example; the operator's, with no default (novox/hq ADR 0112, ADR 0155).", + "trusted": true, + "required": true + } + }, "resources": [ { "id": "mesh-state", @@ -102,19 +129,26 @@ "type": "directory", "mode": "0700" }, + { + "id": "consumers", + "type": "file", + "path": "${dir:state}/consumers.json", + "mode": "0600", + "template": "files/consumers.json" + }, { "id": "admin-env", "type": "file", "path": "${dir:state}/admin.env", "mode": "0600", - "content": "KEYCLOAK_ADMIN=admin\nKEYCLOAK_ADMIN_PASSWORD=${secret:admin}\n" + "template": "files/admin.env" }, { "id": "database-env", "type": "file", "path": "${dir:state}/database.env", "mode": "0600", - "content": "KC_DB_URL=jdbc:postgresql://${bound:postgres-database:at}:${bound:postgres-database:port}/${bound:postgres-database:as}\nKC_DB_USERNAME=${bound:postgres-database:as}\nKC_DB_PASSWORD=${secret:postgres-database}\n" + "template": "files/database.env" }, { "id": "net", @@ -126,7 +160,7 @@ "type": "file", "path": "${dir:state}/hostname.env", "mode": "0644", - "content": "KC_HOSTNAME=https://${bound:route:name}\n" + "template": "files/hostname.env" }, { "id": "server", @@ -161,9 +195,9 @@ "type": "file", "path": "${dir:mesh-state}/config.json", "mode": "0600", - "content": "{}\n", + "template": "files/config.json", "merge": "json", - "takes-settings": [ + "takes-config": [ "url", "user", "realm", @@ -188,9 +222,21 @@ "MESH_KEYCLOAK_CONFIG_FILE": "${dir:mesh-state}/config.json", "MESH_KEYCLOAK_PASSWORD_FILE": "${dir:state}/admin.secret", "MESH_RECEIVES": "${dir:grants}/mesh.json", - "MESH_KEYCLOAK_CONTAINER": "keycloak" + "MESH_KEYCLOAK_CONTAINER": "keycloak", + "MESH_CONSUMERS_FILE": "${dir:state}/consumers.json" } + }, + { + "name": "files", + "kind": "archive", + "from": "files" } ] - } + }, + "state": [ + "bound" + ], + "reads": [ + "mesh-controller.assignments" + ] } diff --git a/modules/lab/files/lab.env b/modules/lab/files/lab.env new file mode 100644 index 00000000..35aa1a29 --- /dev/null +++ b/modules/lab/files/lab.env @@ -0,0 +1,3 @@ +MESH_LAB_FORGE=${config:forge} +MESH_LAB_FORGE_BINDING=${dir:state}/package-registry.json +MESH_LAB_FORGE_SECRET=${dir:state}/package-registry.secret diff --git a/modules/lab/module.json b/modules/lab/module.json index c9c14770..66bb4b43 100644 --- a/modules/lab/module.json +++ b/modules/lab/module.json @@ -29,6 +29,14 @@ } ] }, + "config": { + "forge": { + "type": "string", + "description": "The forge the lab's machines take their packages from, as its address: the operator's, with no default (novox/hq ADR 0112).", + "trusted": true, + "required": true + } + }, "resources": [ { "id": "state", @@ -47,7 +55,7 @@ "type": "file", "path": "${dir:state}/lab.env", "mode": "0600", - "content": "MESH_LAB_FORGE=${setting:forge}\nMESH_LAB_FORGE_BINDING=${dir:state}/package-registry.json\nMESH_LAB_FORGE_SECRET=${dir:state}/package-registry.secret\n" + "template": "files/lab.env" }, { "id": "git", @@ -108,6 +116,11 @@ "MESH_LAB_FORGE_BINDING": "${dir:state}/package-registry.json", "MESH_LAB_FORGE_SECRET": "${dir:state}/package-registry.secret" } + }, + { + "name": "files", + "kind": "archive", + "from": "files" } ] } diff --git a/modules/lab/test/runs.test.ts b/modules/lab/test/runs.test.ts index 2780394b..b2071208 100644 --- a/modules/lab/test/runs.test.ts +++ b/modules/lab/test/runs.test.ts @@ -382,6 +382,24 @@ test("a forge setting that is not http(s) trusts no origin of its own", async () assert.match(viaRoot.problem ?? "", /neither/); }); +// The runtime's env file keeps its text once, in files/ as its template (novox/hq ADR 0349 §2, issue 656): no +// file resource carries its text inline, and the build packs files/ into the one archive the controller reads +// the templates back from. +test("every file takes its text from a template in files/", () => { + const dir = join(import.meta.dirname, ".."); + const m = JSON.parse(readFileSync(join(dir, "module.json"), "utf8")) as { + resources: Record[]; build: { artifacts: { name: string; kind?: string; from?: string }[] }; + }; + for (const r of m.resources) { + assert.equal("content" in r, false, `${String(r.id)} keeps its text inline`); + if (r.type === "file") assert.ok(existsSync(join(dir, String(r.template))), `${String(r.id)} names ${String(r.template)}`); + } + const packs = m.build.artifacts.filter((a) => a.kind === "archive" && a.from === "files"); + assert.deepEqual(packs.map((a) => a.name), ["files"]); + const env = m.resources.find((r) => r.id === "runtime-env"); + assert.match(readFileSync(join(dir, String(env?.template)), "utf8"), /^MESH_LAB_FORGE=\$\{config:forge\}\n/); +}); + // Defends novox/hq issue 693: a run whose only test skipped as NOT REACHED read "0 passed, 0 failed", exit 0, // and lab_status said passed. The log's tail, as run-2026-10-14T00-21-44-014Z on shanks wrote it. const oldLog = [ diff --git a/modules/lemurs/README.md b/modules/lemurs/README.md index 1df59e03..ebe732b3 100644 --- a/modules/lemurs/README.md +++ b/modules/lemurs/README.md @@ -17,7 +17,7 @@ The login manager as a module (novox/hq ADR 0208, research 026, to-be 42 phase 2 | path | class | what | |---|---|---| -| `/etc/lemurs/config.toml` | owned (the found file is kept once, ADR 0102) | lemurs 0.4's configuration in its current format: the structure of the shipped file, every option present, as lemurs requires. The source is [`config/config.toml`](config/config.toml), carried whole in the manifest | +| `/etc/lemurs/config.toml` | owned (the found file is kept once, ADR 0102) | lemurs 0.4's configuration in its current format: the structure of the shipped file, every option present, as lemurs requires. The source is [`files/config.toml`](files/config.toml), the file's template | | `/etc/lemurs/xsessions/`, `/etc/lemurs/wayland-sessions/` | owned, empty | where the configuration points lemurs for desktop entries, so none is offered | **Sessions are drop-ins.** The sessions offered are the executable files session modules place in diff --git a/modules/lemurs/cmd/lemurs-tools/lemurs_test.go b/modules/lemurs/cmd/lemurs-tools/lemurs_test.go index c9580cc8..fd8d1362 100644 --- a/modules/lemurs/cmd/lemurs-tools/lemurs_test.go +++ b/modules/lemurs/cmd/lemurs-tools/lemurs_test.go @@ -73,7 +73,7 @@ func TestTheSessionsAreTheEntriesLemursOffersInItsOrder(t *testing.T) { } func TestTheModulesConfigurationIsReadAsLemursReadsIt(t *testing.T) { - raw, err := os.ReadFile(filepath.Join("..", "..", "config", "config.toml")) + raw, err := os.ReadFile(filepath.Join("..", "..", "files", "config.toml")) if err != nil { t.Fatal(err) } diff --git a/modules/lemurs/cmd/lemurs-tools/manifest_test.go b/modules/lemurs/cmd/lemurs-tools/manifest_test.go index 38764694..98a24914 100644 --- a/modules/lemurs/cmd/lemurs-tools/manifest_test.go +++ b/modules/lemurs/cmd/lemurs-tools/manifest_test.go @@ -93,11 +93,12 @@ func TestTheOfficialPackageAndItsServiceEnabledButNeverRestarted(t *testing.T) { func TestTheConfigurationIsTheModulesFileAndOffersOnlyTheSessionsModulesPlace(t *testing.T) { m := readManifest(t) c := resource(t, m, "config") - raw, _ := os.ReadFile(filepath.Join("..", "..", "config", "config.toml")) - if c["path"] != "/etc/lemurs/config.toml" || c["content"] != string(raw) || c["into"] != nil { - t.Fatal("the manifest carries config/config.toml whole, as an owned file") + raw, _ := os.ReadFile(filepath.Join("..", "..", "files", "config.toml")) + if _, inline := c["content"]; inline || c["template"] != "files/config.toml" || len(raw) == 0 || + c["path"] != "/etc/lemurs/config.toml" || c["into"] != nil { + t.Fatal("the manifest takes files/config.toml whole as its template, as an owned file") } - text := c["content"].(string) + text := string(raw) for _, want := range []string{"[x11]", "[wayland]", `xsetup_path = "/etc/lemurs/xsetup.sh"`, `scripts_path = "/etc/lemurs/wms"`, `scripts_path = "/etc/lemurs/wayland"`, `xsessions_path = "/etc/lemurs/xsessions"`, `wayland_sessions_path = "/etc/lemurs/wayland-sessions"`, `tty = 2`, `x11_display = ":1"`, `remember = true`} { @@ -116,3 +117,31 @@ func TestTheConfigurationIsTheModulesFileAndOffersOnlyTheSessionsModulesPlace(t } } } + +// No file resource carries its text inline: each names its template in files/, and the build packs files/ +// into the one archive the controller reads the templates back from (novox/hq ADR 0349 §2). +func TestEveryFileTakesItsTextFromATemplate(t *testing.T) { + m := readManifest(t) + for _, r := range m.Resources { + if _, inline := r["content"]; inline { + t.Errorf("resource %v keeps its text inline: keep it in files/ and name it as the template", r["id"]) + } + if name, ok := r["template"].(string); ok { + if _, err := os.Stat(filepath.Join("..", "..", name)); err != nil { + t.Errorf("resource %v names %s: %v", r["id"], name, err) + } + } + } + packs := 0 + for _, a := range m.Build.Artifacts { + if a["kind"] == "archive" && a["from"] == "files" { + packs++ + if a["name"] != "files" { + t.Errorf("the templates archive is named %v, not files", a["name"]) + } + } + } + if packs != 1 { + t.Errorf("the build packs files/ into %d archives, not one", packs) + } +} diff --git a/modules/lemurs/config/config.toml b/modules/lemurs/files/config.toml similarity index 100% rename from modules/lemurs/config/config.toml rename to modules/lemurs/files/config.toml diff --git a/modules/lemurs/files/xserver b/modules/lemurs/files/xserver new file mode 100755 index 00000000..5908f228 --- /dev/null +++ b/modules/lemurs/files/xserver @@ -0,0 +1,8 @@ +#!/bin/sh +# The X server as lemurs starts it, written by the mesh (module lemurs). Replaced at every push. +# +# -noreset: without it, X resets whenever its last client disconnects, and a reset empties the +# resource database. The session's start merges the X resources with xrdb before any long-lived +# client is connected, so the server reset twice in the four seconds after a login and threw the +# resources away: xterm fell back to the bitmap "fixed" font, without the mesh's face and colours. +exec /usr/bin/X "$@" -noreset diff --git a/modules/lemurs/module.json b/modules/lemurs/module.json index e759bcc1..abaf8b11 100644 --- a/modules/lemurs/module.json +++ b/modules/lemurs/module.json @@ -30,14 +30,14 @@ "type": "file", "path": "/etc/lemurs/xserver", "mode": "0755", - "content": "#!/bin/sh\n# The X server as lemurs starts it, written by the mesh (module lemurs). Replaced at every push.\n#\n# -noreset: without it, X resets whenever its last client disconnects, and a reset empties the\n# resource database. The session's start merges the X resources with xrdb before any long-lived\n# client is connected, so the server reset twice in the four seconds after a login and threw the\n# resources away: xterm fell back to the bitmap \"fixed\" font, without the mesh's face and colours.\nexec /usr/bin/X \"$@\" -noreset\n" + "template": "files/xserver" }, { "id": "config", "type": "file", "path": "/etc/lemurs/config.toml", "mode": "0644", - "content": "# The login manager's configuration, written by the mesh (module lemurs, novox/hq ADR 0208). Replaced\n# at every push; change the module instead. The structure is lemurs 0.4's own (the shipped file, with\n# every option, as lemurs requires); what the mesh changes from it is marked \"mesh:\".\n#\n# The sessions offered are the executable files other modules place in the two scripts directories\n# below (/etc/lemurs/wms for X, /etc/lemurs/wayland for Wayland), one per session module. A file there\n# is named as the session is offered. Desktop entries that packages install (/usr/share/xsessions) are\n# not offered: they start the window manager bare, skipping the session's start (~/.xinitrc), which is\n# where the account's environment, the X resources and every module's session lines are.\n#\n# Lemurs configuration file.\n# Contains all the customization options of lemurs.\n#\n# Note: that as of now you need to have all options in the selected\n# configuration file. Otherwise Lemurs will not work.\n#\n# Colors:\n# ---------\n# There is a list of predefined colors. These include:\n# - black\n# - white\n# - (dark) gray\n# - (light) red\n# - (light) blue\n# - (light) green\n# - (light) magenta\n# - (light) cyan\n# - (light) yellow\n# - orange\n#\n# You can also utilize custom colors with hex color codes.\n# \"#87CEEB\" will create a Sky Blue color.\n#\n# Note: If the color wasn't recognized, it will default to white.\n# ---------\n#\n# Modifiers:\n# ---------\n# There is a number of modifiers you can use. These can be combined by\n# delimiting them with a comma (e.g. \"bold,italic\"). The modifiers are:\n# - bold\n# - dim\n# - italic\n# - underlined\n# - reverse\n# - crossed out\n# - hidden\n# ---------\n#\n\n# The tty which contains lemurs. This has to be mirrored in the lemurs.service\ntty = 2\n\n# Where to log the main lemurs control flow.\nmain_log_path = \"/var/log/lemurs.log\"\n\n# Where to log to for the client. The Client is the Desktop Environment or\n# Window Manager for Xorg, the Compositor for Wayland and the Shell for TTY.\nclient_log_path = \"/var/log/lemurs.client.log\"\n\n# At which point to point the cache. If you want to disable the cache globally\n# you can use `/dev/null`.\ncache_path = \"/var/cache/lemurs\"\n\n# Disable all logging. This is overwritten by the `--no-log` flag.\ndo_log = true\n\n# The PAM service that should be used to login\npam_service = \"lemurs\"\n\n# Path to system shell that gets used to execute linux commands. In almost all\n# cases, this should refer to a bash shell.\nsystem_shell = \"/bin/sh\"\n\n# Initial state of the `PATH` environment variable.\ninitial_path = \"/usr/local/sbin:/usr/local/bin:/usr/bin\"\n\n# The type flag that will be appended to the shell that calls the session\n# environment. This may depend on your shell. Options:\n# - 'none'. Disables calling a login shell\n# - 'short'. Produces the `-l` flag. Supported by most shells.\n# - 'long'. This produces the `--login` flag and is suited for bash and zsh.\nshell_login_flag = \"short\"\n\n# Focus behaviour of fields when Lemurs is initially started\n#\n# Possible values:\n# - default: Initially focus on first non-cached value\n# - no-focus: No initial focus\n# - environment: Initially focus on the environment selector\n# - username: Initially focus on the username field\n# - password: Initially focus on the password field\nfocus_behaviour = \"default\"\n\n# General settings for background style\n[background]\n\n# Control whether to render background widget or not\nshow_background = false\n\n[background.style]\n# Allow to set the default background color for the login shell\ncolor = \"black\"\n# Settings for the background block's borders\nshow_border = true\nborder_color = \"white\"\n\n[power_controls]\n# The margin between hints\nhint_margin = 2\n\n# There are no additional entries by default\nentries = []\n\n# Example\n# Reboot to another os option\n#[[power_controls.entries]]\n## The text in the top-left to display how to reboot.\n#hint = \"Reboot to OS\"\n#\n## The color and modifiers of the hint in the top-left corner\n#hint_color = \"dark gray\"\n#hint_modifiers = \"\"\n#\n## The key used to reboot. Possibilities are F1 to F12.\n#key = \"F3\"\n## The command that is executed when the key is pressed\n#cmd = \"efibootmgr -n0 && systemctl reboot -l\"\n\n\n# If you want to remove the base_entries\n# base_entries = []\n\n# Shutdown option\n[[power_controls.base_entries]]\n# The text in the top-left to display how to shutdown.\nhint = \"Shutdown\"\n\n# The color and modifiers of the hint in the top-left corner\nhint_color = \"dark gray\"\nhint_modifiers = \"\"\n\n# The key used to shutdown. Possibilities are F1 to F12.\nkey = \"F1\"\n# The command that is executed when the key is pressed\ncmd = \"systemctl poweroff -l\"\n\n# Reboot option\n[[power_controls.base_entries]]\n# The text in the top-left to display how to reboot.\nhint = \"Reboot\"\n\n# The color and modifiers of the hint in the top-left corner\nhint_color = \"dark gray\"\nhint_modifiers = \"\"\n\n# The key used to reboot. Possibilities are F1 to F12.\nkey = \"F2\"\n# The command that is executed when the key is pressed\ncmd = \"systemctl reboot -l\"\n\n# Setting for the selector of the desktop environment you are using.\n[environment_switcher]\n# Terms:\n# ---------\n# Movers: indicators which show which direction one can move whilst selecting\n# the desktop environment\n# Selected: The currently selected desktop environment.\n# Neighbours: The adjacent desktop environment to the one current selected\n#\n# Visualisation:\n#\n# < i3 bspwm awesome >\n#\n# ^ ^ ^ ^ ^\n# | | | | |\n# mover | selected | mover\n# | |\n# neighbour neighbour\n# ---------\n#\n\n# Control the visibility of the switcher\n# Options:\n# - \"visible\" - Always show the switcher [default]\n# - \"hidden\" - Always hide the switcher\n# - [key] - F1-F12 to be able to toggle the visibility\n# mesh: hidden, as both workstations had it, but F3 shows it, so a second session can be chosen.\nswitcher_visibility = \"F3\"\n\n# The text in the top-left to display how to toggle the switcher. The text\n# '%key%' will be replaced with the switcher_visibility key. This is not shown\n# if switcher_visibility is set to \"visible\" or \"hidden\".\ntoggle_hint = \"Switcher %key%\"\n\n# The color and modifiers of the hint in the top-left corner\ntoggle_hint_color = \"dark gray\"\ntoggle_hint_modifiers = \"\"\n\n\n# Show an option for the TTY shell when logging in as one of the environments.\n# NOTE: it is always shown when no viable options are found.\ninclude_tty_shell = false\n\n# Remember the selected environment after logging in for the next time\nremember = true\n\n# Enables showing the movers\nshow_movers = true\n\n# Mover's color and modifiers whilst the selector is unfocused\nmover_color = \"dark gray\"\nmover_modifiers = \"\"\n\n# Mover's color and modifiers whilst the selector is focused\nmover_color_focused = \"orange\"\nmover_modifiers_focused = \"bold\"\n\n# The characters used to display the movers. Suggestions are:\n# - \"<\" \">\"\n# - \"<-\" \"->\"\n# - \"<<\" \">>\"\n# - \"[\" \"]\"\nleft_mover = \"<\"\nright_mover = \">\"\n\n# The margin between the movers and the neighbours or selected (depending on\n# `show_neighbours`)\nmover_margin = 1\n\n# Enables showing the neighbours\nshow_neighbours = true\n\n# Neighbours' color and modifiers whilst the selector is unfocused\nneighbour_color = \"dark gray\"\nneighbour_modifiers = \"\"\n\n# Neighbours' color and modifiers whilst the selector is focused\nneighbour_color_focused = \"gray\"\nneighbour_modifiers_focused = \"\"\n\n# Margin between neighbours and selected\nneighbour_margin = 1\n\n# Selected's color and modifiers whilst the selector is unfocused\nselected_color = \"gray\"\nselected_modifiers = \"underlined\"\n\n# Selected's color and modifiers whilst the selector is focused\nselected_color_focused = \"white\"\nselected_modifiers_focused = \"bold\"\n\n# The length of the name of the desktop environment which is displayed.\nmax_display_length = 8\n\n# The text used when no desktop environments are available\nno_envs_text = \"No environments...\"\n\n# The color and modifiers of the 'no desktop environments available text'\n# whilst the selector is unfocused\nno_envs_color = \"white\"\nno_envs_modifiers = \"\"\n\n# The color and modifiers of the 'no desktop environments available text'\n# whilst the selector is focused\nno_envs_color_focused = \"red\"\nno_envs_modifiers_focused = \"\"\n\n[username_field]\n\n# Remember the username for the next time after a successful login attempt.\nremember = true\n\n[username_field.style]\n# Enables showing a title\nshow_title = true\n# The text used within the title\ntitle = \"Login\"\n\n# The title's color and modifiers whilst the username field is unfocused\ntitle_color = \"white\"\ncontent_color = \"white\"\n\n# The title's color and modifiers whilst the username field is focused\ntitle_color_focused = \"orange\"\ncontent_color_focused = \"orange\"\n\n# Enables showing the borders\nshow_border = true\n# The borders' color and modifiers whilst the username field is unfocused\nborder_color = \"white\"\n# The borders' color and modifiers whilst the username field is focused\nborder_color_focused = \"orange\"\n\n# Constrain the width of the username field\nuse_max_width = true\n# The constraint of the username field's width\nmax_width = 48\n\n[password_field]\n\n# The character used for replacement when typing a password. Leave empty for no\n# feedback.\n# Note: Only one character is accepted.\ncontent_replacement_character = \"*\"\n\n[password_field.style]\n# Enables showing a title\nshow_title = true\n# The text used within the title\ntitle = \"Password\"\n\n# The title's color and modifiers whilst the password field is unfocused\ntitle_color = \"white\"\ncontent_color = \"white\"\n\n# The title's color and modifiers whilst the password field is focused\ntitle_color_focused = \"orange\"\ncontent_color_focused = \"orange\"\n\n# Enables showing the borders\nshow_border = true\n# The borders' color and modifiers whilst the password field is unfocused\nborder_color = \"white\"\n# The borders' color and modifiers whilst the password field is focused\nborder_color_focused = \"orange\"\n\n# Constrain the width of the password field\nuse_max_width = true\n# The constraint of the password field's width\nmax_width = 48\n\n[x11]\n# Where to log to for the XServer.\nxserver_log_path = \"/var/log/lemurs.xorg.log\"\n\n# The value of the `DISPLAY` environment variable for X11 sessions\nx11_display = \":1\"\n\n# How many seconds to give the X server to start. To make it infinitely, put it\n# to 0.\nxserver_timeout_secs = 60\n\n# Where to find the X11 server binary\n# mesh: the server through the wrapper below, which adds -noreset.\nxserver_path = \"/etc/lemurs/xserver\"\n\n# Where to find the X11 xauth binary\nxauth_path = \"/usr/bin/xauth\"\n\n# Path to the directory where the startup scripts for the X11 sessions are found\nscripts_path = \"/etc/lemurs/wms\"\n\n# Path to the xsetup script that is needed for the environment setup of the\n# window manager.\nxsetup_path = \"/etc/lemurs/xsetup.sh\"\n\n# The directory to use for desktop entries X11 sessions.\n# mesh: an empty directory of the module's own, so no package's desktop entry is offered.\nxsessions_path = \"/etc/lemurs/xsessions\"\n\n[wayland]\n# Path to the directory where the startup scripts for the Wayland sessions are\n# found\nscripts_path = \"/etc/lemurs/wayland\"\n\n# The directory to use for desktop entries wayland sessions.\n# mesh: likewise for Wayland.\nwayland_sessions_path = \"/etc/lemurs/wayland-sessions\"\n" + "template": "files/config.toml" }, { "id": "xsessions", @@ -71,6 +71,11 @@ "loads": [ "lemurs-tools" ] + }, + { + "name": "files", + "kind": "archive", + "from": "files" } ] } diff --git a/modules/letta/files/config.json b/modules/letta/files/config.json new file mode 100644 index 00000000..8ed994da --- /dev/null +++ b/modules/letta/files/config.json @@ -0,0 +1,3 @@ +{ + "password": "${secret:server-password}" +} diff --git a/modules/letta/files/pgpass b/modules/letta/files/pgpass new file mode 100644 index 00000000..cd41b665 --- /dev/null +++ b/modules/letta/files/pgpass @@ -0,0 +1 @@ +*:*:*:${bound:postgres-database:as}:${secret:postgres-database} diff --git a/modules/letta/files/server.env b/modules/letta/files/server.env new file mode 100644 index 00000000..fe2acb9e --- /dev/null +++ b/modules/letta/files/server.env @@ -0,0 +1,6 @@ +LETTA_PG_URI=postgresql://${bound:postgres-database:as}@${bound:postgres-database:at}:${bound:postgres-database:port}/${bound:postgres-database:as} +PGPASSFILE=/run/secrets/pgpass +LETTA_SERVER_PASSWORD=${secret:server-password} +OPENAI_API_KEY=${secret:openai-api-key} +SECURE=true +TZ=Europe/Brussels diff --git a/modules/letta/files/start.sh b/modules/letta/files/start.sh new file mode 100644 index 00000000..99555927 --- /dev/null +++ b/modules/letta/files/start.sh @@ -0,0 +1,23 @@ +#!/bin/sh +# Generated by the mesh. Do not edit: module letta writes this file and replaces it at every push. +# +# letta 0.6.8 prints secrets it is given to its log (novox/hq issue 268): +# letta/server/rest_api/app.py prints its server password when it starts in secure mode; +# startup.sh, alembic/env.py and letta/server/server.py print LETTA_PG_URI whole. +# The URI carries no password (libpq reads it from PGPASSFILE), and the one print of the server +# password is rewritten before the server starts. Either one failing refuses the start: a letta +# that does not start says why here, and one that leaks says nothing. +set -e +app=/app/letta/server/rest_api/app.py +sed -i 's/Using secure mode with password: {random_password}/Using secure mode (the password is not printed)/' "$app" +if grep -q 'print(.*random_password' "$app"; then + echo "letta: $app still prints the server password; not starting (novox/hq issue 268)" >&2 + exit 1 +fi +case "$LETTA_PG_URI" in + *://*:*@*) + echo "letta: LETTA_PG_URI carries a password, and letta prints that URI; not starting (novox/hq issue 268)" >&2 + exit 1 + ;; +esac +exec ./letta/server/startup.sh diff --git a/modules/letta/module.json b/modules/letta/module.json index 4174b99c..dbcc5590 100644 --- a/modules/letta/module.json +++ b/modules/letta/module.json @@ -47,6 +47,14 @@ "why": "the Letta agent server REST API and web UI, password-protected (--secure); a public name is the route's" } ], + "config": { + "url": { + "type": "string", + "description": "The address the module's tools reach Letta at, as a URL; unset, the code falls back to its environment, else this machine's Letta on port 8283.", + "trusted": true, + "default": "" + } + }, "resources": [ { "id": "mesh-state", @@ -65,21 +73,21 @@ "type": "file", "path": "${dir:state}/server.env", "mode": "0600", - "content": "LETTA_PG_URI=postgresql://${bound:postgres-database:as}@${bound:postgres-database:at}:${bound:postgres-database:port}/${bound:postgres-database:as}\nPGPASSFILE=/run/secrets/pgpass\nLETTA_SERVER_PASSWORD=${secret:server-password}\nOPENAI_API_KEY=${secret:openai-api-key}\nSECURE=true\nTZ=Europe/Brussels\n" + "template": "files/server.env" }, { "id": "pgpass", "type": "file", "path": "${dir:state}/pgpass", "mode": "0600", - "content": "*:*:*:${bound:postgres-database:as}:${secret:postgres-database}\n" + "template": "files/pgpass" }, { "id": "start", "type": "file", "path": "${dir:state}/start.sh", "mode": "0644", - "content": "#!/bin/sh\n# Generated by the mesh. Do not edit: module letta writes this file and replaces it at every push.\n#\n# letta 0.6.8 prints secrets it is given to its log (novox/hq issue 268):\n# letta/server/rest_api/app.py prints its server password when it starts in secure mode;\n# startup.sh, alembic/env.py and letta/server/server.py print LETTA_PG_URI whole.\n# The URI carries no password (libpq reads it from PGPASSFILE), and the one print of the server\n# password is rewritten before the server starts. Either one failing refuses the start: a letta\n# that does not start says why here, and one that leaks says nothing.\nset -e\napp=/app/letta/server/rest_api/app.py\nsed -i 's/Using secure mode with password: {random_password}/Using secure mode (the password is not printed)/' \"$app\"\nif grep -q 'print(.*random_password' \"$app\"; then\n echo \"letta: $app still prints the server password; not starting (novox/hq issue 268)\" >&2\n exit 1\nfi\ncase \"$LETTA_PG_URI\" in\n *://*:*@*)\n echo \"letta: LETTA_PG_URI carries a password, and letta prints that URI; not starting (novox/hq issue 268)\" >&2\n exit 1\n ;;\nesac\nexec ./letta/server/startup.sh\n" + "template": "files/start.sh" }, { "id": "net", @@ -113,8 +121,9 @@ "type": "file", "path": "${dir:mesh-state}/config.json", "mode": "0600", - "content": "{\n \"password\": \"${secret:server-password}\"\n}\n", + "template": "files/config.json", "merge": "json", + "takes-config": ["url"], "protected": ["password"] } ], @@ -134,6 +143,11 @@ "MESH_LETTA_URL": "http://127.0.0.1:${port:8283}", "MESH_LETTA_CONFIG_FILE": "${dir:mesh-state}/config.json" } + }, + { + "name": "files", + "kind": "archive", + "from": "files" } ] } diff --git a/modules/local-model-consumer/files/openai.env b/modules/local-model-consumer/files/openai.env new file mode 100644 index 00000000..9aca3962 --- /dev/null +++ b/modules/local-model-consumer/files/openai.env @@ -0,0 +1,3 @@ +OPENAI_BASE_URL=http://${bound:model-access:at}:${bound:model-access:port}/v1 +OPENAI_MODEL=${bound:model-access:model} +OPENAI_API_KEY=local diff --git a/modules/local-model-consumer/module.json b/modules/local-model-consumer/module.json index e90fbf5c..6fdb90b5 100644 --- a/modules/local-model-consumer/module.json +++ b/modules/local-model-consumer/module.json @@ -25,7 +25,16 @@ "type": "file", "path": "${dir:config}/openai.env", "mode": "0600", - "content": "OPENAI_BASE_URL=http://${bound:model-access:at}:${bound:model-access:port}/v1\nOPENAI_MODEL=${bound:model-access:model}\nOPENAI_API_KEY=local\n" + "template": "files/openai.env" } - ] + ], + "build": { + "artifacts": [ + { + "name": "files", + "kind": "archive", + "from": "files" + } + ] + } } diff --git a/modules/localization/cmd/localization-tools/manifest_test.go b/modules/localization/cmd/localization-tools/manifest_test.go index 27edf2f1..2c6ab5b6 100644 --- a/modules/localization/cmd/localization-tools/manifest_test.go +++ b/modules/localization/cmd/localization-tools/manifest_test.go @@ -15,16 +15,16 @@ func TestTheFilesSayWhatTheToolsCompareAgainst(t *testing.T) { t.Fatalf("%s %s", m.Module, m.Version) } locale := m.resource(t, "locale") - if locale["path"] != "/etc/locale.conf" || locale["into"] != nil || !strings.Contains(locale["content"].(string), "\nLANG="+MeshLang+"\n") { + if locale["path"] != "/etc/locale.conf" || locale["into"] != nil || !strings.Contains(templateText(t, locale), "\nLANG="+MeshLang+"\n") { t.Fatalf("locale: %v", locale) } keymap := m.resource(t, "keymap") - if keymap["path"] != "/etc/vconsole.conf" || !strings.Contains(keymap["content"].(string), "\nKEYMAP="+MeshKeymap+"\n") { + if keymap["path"] != "/etc/vconsole.conf" || !strings.Contains(templateText(t, keymap), "\nKEYMAP="+MeshKeymap+"\n") { t.Fatalf("keymap: %v", keymap) } for _, r := range []resource{locale, keymap} { var settings []string - for _, l := range strings.Split(r["content"].(string), "\n") { + for _, l := range strings.Split(templateText(t, r), "\n") { if l != "" && !strings.HasPrefix(l, "#") { settings = append(settings, l) } diff --git a/modules/localization/cmd/localization-tools/templates_test.go b/modules/localization/cmd/localization-tools/templates_test.go new file mode 100644 index 00000000..7d0718b5 --- /dev/null +++ b/modules/localization/cmd/localization-tools/templates_test.go @@ -0,0 +1,64 @@ +package main + +import ( + "encoding/json" + "os" + "path/filepath" + "testing" +) + +// The module's directory, where its manifest and its files/ are. +const moduleDir = "../.." + +// templateText is the text a file resource places on the machine: its template's, in the module's files/ +// (novox/hq ADR 0349 §2). A resource that names no template, or a template that is not there, fails the test. +func templateText(t *testing.T, r map[string]any) string { + t.Helper() + name, _ := r["template"].(string) + if name == "" { + t.Fatalf("resource %v names no template", r["id"]) + } + raw, err := os.ReadFile(filepath.Join(moduleDir, filepath.FromSlash(name))) + if err != nil { + t.Fatalf("resource %v names %s: %v", r["id"], name, err) + } + return string(raw) +} + +// No file resource carries its text inline: each names its template in files/, and the build packs files/ +// into the one archive the controller reads the templates back from (novox/hq ADR 0349 §2). +func TestEveryFileTakesItsTextFromATemplate(t *testing.T) { + raw, err := os.ReadFile(filepath.Join(moduleDir, "module.json")) + if err != nil { + t.Fatal(err) + } + var m struct { + Resources []map[string]any `json:"resources"` + Build struct { + Artifacts []map[string]any `json:"artifacts"` + } `json:"build"` + } + if err := json.Unmarshal(raw, &m); err != nil { + t.Fatal(err) + } + for _, r := range m.Resources { + if _, inline := r["content"]; inline { + t.Errorf("resource %v keeps its text inline: keep it in files/ and name it as the template", r["id"]) + } + if r["type"] == "file" { + templateText(t, r) + } + } + packs := 0 + for _, a := range m.Build.Artifacts { + if a["kind"] == "archive" && a["from"] == "files" { + packs++ + if a["name"] != "files" { + t.Errorf("the templates archive is named %v, not files", a["name"]) + } + } + } + if packs != 1 { + t.Errorf("the build packs files/ into %d archives, not one", packs) + } +} diff --git a/modules/localization/files/locale.conf b/modules/localization/files/locale.conf new file mode 100644 index 00000000..fe405b50 --- /dev/null +++ b/modules/localization/files/locale.conf @@ -0,0 +1,3 @@ +# The mesh's (module localization, novox/hq to-be 42): the system locale. Written whole at every +# push; an edit here is overwritten. Read at the next login. +LANG=en_US.UTF-8 diff --git a/modules/localization/files/vconsole.conf b/modules/localization/files/vconsole.conf new file mode 100644 index 00000000..88d96f68 --- /dev/null +++ b/modules/localization/files/vconsole.conf @@ -0,0 +1,3 @@ +# The mesh's (module localization, novox/hq to-be 42): the console keymap. Written whole at every +# push; an edit here is overwritten. Read at the next boot. +KEYMAP=us diff --git a/modules/localization/module.json b/modules/localization/module.json index 020b9026..0974ed38 100644 --- a/modules/localization/module.json +++ b/modules/localization/module.json @@ -16,14 +16,14 @@ "type": "file", "path": "/etc/locale.conf", "mode": "0644", - "content": "# The mesh's (module localization, novox/hq to-be 42): the system locale. Written whole at every\n# push; an edit here is overwritten. Read at the next login.\nLANG=en_US.UTF-8\n" + "template": "files/locale.conf" }, { "id": "keymap", "type": "file", "path": "/etc/vconsole.conf", "mode": "0644", - "content": "# The mesh's (module localization, novox/hq to-be 42): the console keymap. Written whole at every\n# push; an edit here is overwritten. Read at the next boot.\nKEYMAP=us\n" + "template": "files/vconsole.conf" }, { "id": "time-zone", @@ -50,6 +50,11 @@ "loads": [ "localization-tools" ] + }, + { + "name": "files", + "kind": "archive", + "from": "files" } ] } diff --git a/modules/logrotate/cmd/logrotate-tools/logrotate_test.go b/modules/logrotate/cmd/logrotate-tools/logrotate_test.go index 94b7b76c..6e0c3464 100644 --- a/modules/logrotate/cmd/logrotate-tools/logrotate_test.go +++ b/modules/logrotate/cmd/logrotate-tools/logrotate_test.go @@ -74,7 +74,7 @@ func TestADirectiveIsNotALog(t *testing.T) { } func TestGlobalsAreTheBaseWithoutIncludesOrBlocks(t *testing.T) { - g := manifest(t).resource(t, "config")["content"].(string) + g := templateText(t, manifest(t).resource(t, "config")) got := Globals(g) if strings.Join(got, "|") != "weekly|rotate 4|create|compress|delaycompress|tabooext + .pacorig .pacnew .pacsave" { t.Fatalf("%v", got) diff --git a/modules/logrotate/cmd/logrotate-tools/manifest_test.go b/modules/logrotate/cmd/logrotate-tools/manifest_test.go index ce887074..b6ca26cd 100644 --- a/modules/logrotate/cmd/logrotate-tools/manifest_test.go +++ b/modules/logrotate/cmd/logrotate-tools/manifest_test.go @@ -18,13 +18,13 @@ func TestItDeclaresThePackageTheBaseAndTheTimer(t *testing.T) { t.Fatalf("%v", p) } c := m.resource(t, "config") - if c["path"] != BaseConf || c["into"] != nil || !strings.HasPrefix(c["content"].(string), "# The mesh's (module logrotate") { + if c["path"] != BaseConf || c["into"] != nil || !strings.HasPrefix(templateText(t, c), "# The mesh's (module logrotate") { t.Fatalf("%v", c) } - if !strings.Contains(c["content"].(string), "\ninclude "+RulesDir+"\n") { + if !strings.Contains(templateText(t, c), "\ninclude "+RulesDir+"\n") { t.Fatal("the base must include the packages' rules, or nothing of theirs rotates") } - if strings.Contains(c["content"].(string), "olddir") { + if strings.Contains(templateText(t, c), "olddir") { t.Fatal("olddir flattens logs of different directories into one, where two of one name collide") } timer := m.resource(t, "timer") @@ -39,7 +39,7 @@ func TestTheBaseParses(t *testing.T) { t.Skip("logrotate is not installed here; the base configuration is not dry-run") } dir := t.TempDir() - content := strings.ReplaceAll(manifest(t).resource(t, "config")["content"].(string), "include "+RulesDir, "include "+filepath.Join(dir, "d")) + content := strings.ReplaceAll(templateText(t, manifest(t).resource(t, "config")), "include "+RulesDir, "include "+filepath.Join(dir, "d")) if err := os.Mkdir(filepath.Join(dir, "d"), 0o755); err != nil { t.Fatal(err) } diff --git a/modules/logrotate/cmd/logrotate-tools/templates_test.go b/modules/logrotate/cmd/logrotate-tools/templates_test.go new file mode 100644 index 00000000..7d0718b5 --- /dev/null +++ b/modules/logrotate/cmd/logrotate-tools/templates_test.go @@ -0,0 +1,64 @@ +package main + +import ( + "encoding/json" + "os" + "path/filepath" + "testing" +) + +// The module's directory, where its manifest and its files/ are. +const moduleDir = "../.." + +// templateText is the text a file resource places on the machine: its template's, in the module's files/ +// (novox/hq ADR 0349 §2). A resource that names no template, or a template that is not there, fails the test. +func templateText(t *testing.T, r map[string]any) string { + t.Helper() + name, _ := r["template"].(string) + if name == "" { + t.Fatalf("resource %v names no template", r["id"]) + } + raw, err := os.ReadFile(filepath.Join(moduleDir, filepath.FromSlash(name))) + if err != nil { + t.Fatalf("resource %v names %s: %v", r["id"], name, err) + } + return string(raw) +} + +// No file resource carries its text inline: each names its template in files/, and the build packs files/ +// into the one archive the controller reads the templates back from (novox/hq ADR 0349 §2). +func TestEveryFileTakesItsTextFromATemplate(t *testing.T) { + raw, err := os.ReadFile(filepath.Join(moduleDir, "module.json")) + if err != nil { + t.Fatal(err) + } + var m struct { + Resources []map[string]any `json:"resources"` + Build struct { + Artifacts []map[string]any `json:"artifacts"` + } `json:"build"` + } + if err := json.Unmarshal(raw, &m); err != nil { + t.Fatal(err) + } + for _, r := range m.Resources { + if _, inline := r["content"]; inline { + t.Errorf("resource %v keeps its text inline: keep it in files/ and name it as the template", r["id"]) + } + if r["type"] == "file" { + templateText(t, r) + } + } + packs := 0 + for _, a := range m.Build.Artifacts { + if a["kind"] == "archive" && a["from"] == "files" { + packs++ + if a["name"] != "files" { + t.Errorf("the templates archive is named %v, not files", a["name"]) + } + } + } + if packs != 1 { + t.Errorf("the build packs files/ into %d archives, not one", packs) + } +} diff --git a/modules/logrotate/files/logrotate.conf b/modules/logrotate/files/logrotate.conf new file mode 100644 index 00000000..d6461375 --- /dev/null +++ b/modules/logrotate/files/logrotate.conf @@ -0,0 +1,32 @@ +# The mesh's (module logrotate, novox/hq to-be 42): the base configuration every rotation inherits. +# Written whole at every push; an edit here is overwritten. Each package's own rules are in +# /etc/logrotate.d and stay the packages'. + +# Weekly, four weeks kept, a new empty log created after each rotation. +weekly +rotate 4 +create + +# Rotated logs are compressed, one rotation late, so a program still writing to the file it had open +# loses nothing to the compression. +compress +delaycompress + +# A package's replaced configuration is never read as a rule. +tabooext + .pacorig .pacnew .pacsave + +include /etc/logrotate.d + +/var/log/wtmp { + monthly + create 0664 root utmp + minsize 1M + rotate 1 +} + +/var/log/btmp { + missingok + monthly + create 0600 root utmp + rotate 1 +} diff --git a/modules/logrotate/module.json b/modules/logrotate/module.json index 23d37678..8318451b 100644 --- a/modules/logrotate/module.json +++ b/modules/logrotate/module.json @@ -25,7 +25,7 @@ "type": "file", "path": "/etc/logrotate.conf", "mode": "0644", - "content": "# The mesh's (module logrotate, novox/hq to-be 42): the base configuration every rotation inherits.\n# Written whole at every push; an edit here is overwritten. Each package's own rules are in\n# /etc/logrotate.d and stay the packages'.\n\n# Weekly, four weeks kept, a new empty log created after each rotation.\nweekly\nrotate 4\ncreate\n\n# Rotated logs are compressed, one rotation late, so a program still writing to the file it had open\n# loses nothing to the compression.\ncompress\ndelaycompress\n\n# A package's replaced configuration is never read as a rule.\ntabooext + .pacorig .pacnew .pacsave\n\ninclude /etc/logrotate.d\n\n/var/log/wtmp {\n monthly\n create 0664 root utmp\n minsize 1M\n rotate 1\n}\n\n/var/log/btmp {\n missingok\n monthly\n create 0600 root utmp\n rotate 1\n}\n" + "template": "files/logrotate.conf" }, { "id": "timer", @@ -47,6 +47,11 @@ "loads": [ "logrotate-tools" ] + }, + { + "name": "files", + "kind": "archive", + "from": "files" } ] } diff --git a/modules/mailu/files/admin.env b/modules/mailu/files/admin.env new file mode 100644 index 00000000..f066bb56 --- /dev/null +++ b/modules/mailu/files/admin.env @@ -0,0 +1,2 @@ +INITIAL_ADMIN_PW=${secret:admin} +API_TOKEN=${secret:api-token} diff --git a/modules/mailu/files/config.json b/modules/mailu/files/config.json new file mode 100644 index 00000000..0967ef42 --- /dev/null +++ b/modules/mailu/files/config.json @@ -0,0 +1 @@ +{} diff --git a/modules/mailu/files/database.env b/modules/mailu/files/database.env new file mode 100644 index 00000000..ba70d3d6 --- /dev/null +++ b/modules/mailu/files/database.env @@ -0,0 +1,5 @@ +DB_FLAVOR=postgresql +DB_HOST=${bound:postgres-database:at}:${bound:postgres-database:port} +DB_USER=${bound:postgres-database:as} +DB_NAME=${bound:postgres-database:as} +DB_PW=${secret:postgres-database} diff --git a/modules/mailu/files/mailu.env b/modules/mailu/files/mailu.env new file mode 100644 index 00000000..2c977d30 --- /dev/null +++ b/modules/mailu/files/mailu.env @@ -0,0 +1,78 @@ +ADMIN_ADDRESS=mailu-admin +ANTISPAM_ADDRESS=mailu-antispam +ANTIVIRUS_ADDRESS=mailu-antivirus +IMAP_ADDRESS=mailu-imap +SMTP_ADDRESS=mailu-smtp +FRONT_ADDRESS=mailu-front +WEBMAIL_ADDRESS=mailu-webmail +WEBDAV_ADDRESS=mailu-webdav +REDIS_ADDRESS=mailu-redis +PORTS=25,80,443,465,993,995,4190,110,143,587 +DOMAIN=${config:domain} +HOSTNAMES=${bound:route:name-web} +POSTMASTER=admin +SITENAME=${config:sitename} +WEBSITE=${config:website} +TLS_FLAVOR=letsencrypt +SUBNET=192.168.203.0/24 +COMPOSE_PROJECT_NAME=mailu +ANTIVIRUS=clamav +WEBMAIL=roundcube +WEBDAV=radicale +FETCHMAIL_ENABLED=True +FETCHMAIL_DELAY=600 +ADMIN=true +WEB_ADMIN=/admin +WEB_WEBMAIL=/webmail +WEBROOT_REDIRECT=/webmail +API=true +WEB_API=/api +AUTH_RATELIMIT_IP=6000/hour +AUTH_RATELIMIT_USER=1000/day +CREDENTIAL_ROUNDS=12 +PASSWORD_SCHEME=PBKDF2 +DISABLE_STATISTICS=True +MESSAGE_SIZE_LIMIT=50000000 +MESSAGE_RATELIMIT=200/day +RECIPIENT_DELIMITER=+ +POSTFIX_MYNETWORKS=127.0.0.0/8 [::1]/128 +RELAYNETS= +RELAYHOST= +REJECT_UNLISTED_RECIPIENT= +DB_FLAVOR=postgresql +INITIAL_ADMIN_ACCOUNT=admin +INITIAL_ADMIN_DOMAIN=${config:domain} +INITIAL_ADMIN_MODE=ifmissing +SMTP_PORT=25 +SMTPS_PORT=465 +SUBMISSION_PORT=587 +POP3_PORT=110 +POP3S_PORT=995 +IMAP_PORT=143 +IMAPS_PORT=993 +HTTP_PORT=7080 +HTTPS_PORT=7443 +AUTOMX_PORT=4243 +AMX_SMTP_ADDRESS=${bound:route:name-web} +AMX_SMTP_PORT=587 +AMX_IMAP_ADDRESS=${bound:route:name-web} +AMX_IMAP_PORT=143 +AMX_MAIL_DOMAINS=${config:domain} +DMARC_RUA=admin +DMARC_RUF=admin +LETSENCRYPT_SHORTCHAIN=True +TZ=Etc/UTC +LOG_LEVEL=INFO +WELCOME=false +REAL_IP_HEADER=X-Real-IP +REAL_IP_FROM=${config:proxy-address} +COMPRESSION= +COMPRESS_LEVEL= +COMPRESSION_LEVEL= +BIND_ADDRESS4=127.0.0.1 +BIND_ADDRESS6=::1 +MAILU_VERSION=1.9 +DOCKER_ORG=mailu +DOCKER_PREFIX= +WELCOME_SUBJECT=Welcome to your new email account +WELCOME_BODY=Welcome to your new email account, if you can read this, then it is configured properly! diff --git a/modules/mailu/files/secret.env b/modules/mailu/files/secret.env new file mode 100644 index 00000000..c6f6b66b --- /dev/null +++ b/modules/mailu/files/secret.env @@ -0,0 +1 @@ +SECRET_KEY=${secret:secret-key} diff --git a/modules/mailu/module.json b/modules/mailu/module.json index bcdf3d31..52750c98 100644 --- a/modules/mailu/module.json +++ b/modules/mailu/module.json @@ -248,6 +248,44 @@ } } }, + "config": { + "domain": { + "type": "string", + "description": "The mail domain Mailu serves, such as example.org: its first mail domain, the postmaster's and the one consumers send as; the operator's, with no default (novox/hq ADR 0112).", + "trusted": true, + "required": true + }, + "proxy-address": { + "type": "string", + "description": "The address of the reverse proxy in front of Mailu, whose X-Real-IP header Mailu believes; the operator's, with no default.", + "trusted": true, + "required": true + }, + "sitename": { + "type": "string", + "description": "The name Mailu's web pages show for the mail service; the operator's, with no default.", + "trusted": true, + "required": true + }, + "website": { + "type": "string", + "description": "The address of the website Mailu's web pages link to, as a URL; the operator's, with no default.", + "trusted": true, + "required": true + }, + "url": { + "type": "string", + "description": "The address the module's tools reach Mailu's API at, as a URL; empty leaves them to MESH_MAILU_URL.", + "trusted": true, + "default": "" + }, + "container": { + "type": "string", + "description": "The name of the container holding Mailu's mailboxes, which the module's tools read mail in; empty is mailu-imap.", + "trusted": true, + "default": "" + } + }, "resources": [ { "id": "mesh-state", @@ -276,28 +314,28 @@ "type": "file", "path": "${dir:state}/mailu.env", "mode": "0644", - "content": "ADMIN_ADDRESS=mailu-admin\nANTISPAM_ADDRESS=mailu-antispam\nANTIVIRUS_ADDRESS=mailu-antivirus\nIMAP_ADDRESS=mailu-imap\nSMTP_ADDRESS=mailu-smtp\nFRONT_ADDRESS=mailu-front\nWEBMAIL_ADDRESS=mailu-webmail\nWEBDAV_ADDRESS=mailu-webdav\nREDIS_ADDRESS=mailu-redis\nPORTS=25,80,443,465,993,995,4190,110,143,587\nDOMAIN=${setting:domain}\nHOSTNAMES=${bound:route:name-web}\nPOSTMASTER=admin\nSITENAME=${setting:sitename}\nWEBSITE=${setting:website}\nTLS_FLAVOR=letsencrypt\nSUBNET=192.168.203.0/24\nCOMPOSE_PROJECT_NAME=mailu\nANTIVIRUS=clamav\nWEBMAIL=roundcube\nWEBDAV=radicale\nFETCHMAIL_ENABLED=True\nFETCHMAIL_DELAY=600\nADMIN=true\nWEB_ADMIN=/admin\nWEB_WEBMAIL=/webmail\nWEBROOT_REDIRECT=/webmail\nAPI=true\nWEB_API=/api\nAUTH_RATELIMIT_IP=6000/hour\nAUTH_RATELIMIT_USER=1000/day\nCREDENTIAL_ROUNDS=12\nPASSWORD_SCHEME=PBKDF2\nDISABLE_STATISTICS=True\nMESSAGE_SIZE_LIMIT=50000000\nMESSAGE_RATELIMIT=200/day\nRECIPIENT_DELIMITER=+\nPOSTFIX_MYNETWORKS=127.0.0.0/8 [::1]/128\nRELAYNETS=\nRELAYHOST=\nREJECT_UNLISTED_RECIPIENT=\nDB_FLAVOR=postgresql\nINITIAL_ADMIN_ACCOUNT=admin\nINITIAL_ADMIN_DOMAIN=${setting:domain}\nINITIAL_ADMIN_MODE=ifmissing\nSMTP_PORT=25\nSMTPS_PORT=465\nSUBMISSION_PORT=587\nPOP3_PORT=110\nPOP3S_PORT=995\nIMAP_PORT=143\nIMAPS_PORT=993\nHTTP_PORT=7080\nHTTPS_PORT=7443\nAUTOMX_PORT=4243\nAMX_SMTP_ADDRESS=${bound:route:name-web}\nAMX_SMTP_PORT=587\nAMX_IMAP_ADDRESS=${bound:route:name-web}\nAMX_IMAP_PORT=143\nAMX_MAIL_DOMAINS=${setting:domain}\nDMARC_RUA=admin\nDMARC_RUF=admin\nLETSENCRYPT_SHORTCHAIN=True\nTZ=Etc/UTC\nLOG_LEVEL=INFO\nWELCOME=false\nREAL_IP_HEADER=X-Real-IP\nREAL_IP_FROM=${setting:proxy-address}\nCOMPRESSION=\nCOMPRESS_LEVEL=\nCOMPRESSION_LEVEL=\nBIND_ADDRESS4=127.0.0.1\nBIND_ADDRESS6=::1\nMAILU_VERSION=1.9\nDOCKER_ORG=mailu\nDOCKER_PREFIX=\nWELCOME_SUBJECT=Welcome to your new email account\nWELCOME_BODY=Welcome to your new email account, if you can read this, then it is configured properly!\n" + "template": "files/mailu.env" }, { "id": "secret-env", "type": "file", "path": "${dir:state}/secret.env", "mode": "0600", - "content": "SECRET_KEY=${secret:secret-key}\n" + "template": "files/secret.env" }, { "id": "database-env", "type": "file", "path": "${dir:state}/database.env", "mode": "0600", - "content": "DB_FLAVOR=postgresql\nDB_HOST=${bound:postgres-database:at}:${bound:postgres-database:port}\nDB_USER=${bound:postgres-database:as}\nDB_NAME=${bound:postgres-database:as}\nDB_PW=${secret:postgres-database}\n" + "template": "files/database.env" }, { "id": "admin-env", "type": "file", "path": "${dir:state}/admin.env", "mode": "0600", - "content": "INITIAL_ADMIN_PW=${secret:admin}\nAPI_TOKEN=${secret:api-token}\n" + "template": "files/admin.env" }, { "id": "data-certs", @@ -618,7 +656,8 @@ "type": "file", "path": "${dir:mesh-state}/config.json", "mode": "0600", - "content": "{}\n", + "template": "files/config.json", + "takes-config": ["domain", "proxy-address", "sitename", "website", "url", "container"], "merge": "json" }, { @@ -673,6 +712,11 @@ "name": "automx", "kind": "image", "from": "automx/Dockerfile" + }, + { + "name": "files", + "kind": "archive", + "from": "files" } ] }, @@ -689,7 +733,7 @@ "serves": { "smtp": { "port": 587, - "domain": "${setting:domain}" + "domain": "${config:domain}" } }, "receives": { diff --git a/modules/matrix/files/conduit.toml b/modules/matrix/files/conduit.toml new file mode 100644 index 00000000..9b724010 --- /dev/null +++ b/modules/matrix/files/conduit.toml @@ -0,0 +1,15 @@ +# Written by the mesh (modules/matrix). Conduit reads this file (CONDUIT_CONFIG); nothing comes +# from the environment. server_name is the homeserver's permanent identity: every user id, room id +# and signature in the database carries it, so it is the name this module is served under +# (${bound:route:name-homeserver}) and never changes once a database exists. +[global] +server_name = "${bound:route:name-homeserver}" +database_backend = "rocksdb" +database_path = "/var/lib/matrix-conduit/" +address = "0.0.0.0" +port = 6167 +max_request_size = 20000000 +allow_registration = false +allow_federation = true +allow_check_for_updates = true +trusted_servers = ["matrix.org"] diff --git a/modules/matrix/files/element.json b/modules/matrix/files/element.json new file mode 100644 index 00000000..305a89f7 --- /dev/null +++ b/modules/matrix/files/element.json @@ -0,0 +1,53 @@ +{ + "default_server_name": "${bound:route:name-homeserver}", + "default_server_config": { + "m.homeserver": { + "base_url": "https://${bound:route:name-homeserver}" + }, + "m.identity_server": { + "base_url": "https://vector.im" + } + }, + "brand": "Element", + "integrations_ui_url": "https://scalar.vector.im/", + "integrations_rest_url": "https://scalar.vector.im/api", + "integrations_widgets_urls": [ + "https://scalar.vector.im/_matrix/integrations/v1", + "https://scalar.vector.im/api", + "https://scalar-staging.vector.im/_matrix/integrations/v1", + "https://scalar-staging.vector.im/api", + "https://scalar-staging.riot.im/scalar/api" + ], + "bug_report_endpoint_url": "https://element.io/bugreports/submit", + "uisi_autorageshake_app": "element-auto-uisi", + "show_labs_settings": true, + "room_directory": { + "servers": [ + "${bound:route:name-homeserver}", + "matrix.org", + "gitter.im", + "libera.chat" + ] + }, + "enable_presence_by_hs_url": { + "https://matrix.org": false, + "https://matrix-client.matrix.org": false + }, + "terms_and_conditions_links": [ + { + "url": "https://element.io/privacy", + "text": "Privacy Policy" + }, + { + "url": "https://element.io/cookie-policy", + "text": "Cookie Policy" + } + ], + "features": { + "feature_video_rooms": true, + "feature_rust_crypto": true + }, + "element_call": { + "url": "https://call.element.dev" + } +} diff --git a/modules/matrix/module.json b/modules/matrix/module.json index eded5f65..2c044576 100644 --- a/modules/matrix/module.json +++ b/modules/matrix/module.json @@ -48,6 +48,113 @@ } ] }, + "config": { + "default_server_name": { + "type": "string", + "description": "Element's default_server_name setting of config.json: the homeserver Element signs in to by default, by name; the file's default is the module's own route.", + "trusted": true, + "default": "${bound:route:name-homeserver}" + }, + "default_server_config": { + "type": "object", + "description": "Element's default_server_config setting of config.json: the homeserver and identity server Element uses by default, by URL.", + "trusted": true, + "default": { + "m.homeserver": { + "base_url": "https://${bound:route:name-homeserver}" + }, + "m.identity_server": { + "base_url": "https://vector.im" + } + } + }, + "brand": { + "type": "string", + "description": "Element's brand setting of config.json: the name Element shows itself as.", + "trusted": true, + "default": "Element" + }, + "integrations_ui_url": { + "type": "string", + "description": "Element's integrations_ui_url setting of config.json: the integration manager's page, a URL.", + "trusted": true, + "default": "https://scalar.vector.im/" + }, + "integrations_rest_url": { + "type": "string", + "description": "Element's integrations_rest_url setting of config.json: the integration manager's API, a URL.", + "trusted": true, + "default": "https://scalar.vector.im/api" + }, + "integrations_widgets_urls": { + "type": "list", + "description": "Element's integrations_widgets_urls setting of config.json: the URLs whose widgets Element trusts as the integration manager's.", + "trusted": true, + "default": [ + "https://scalar.vector.im/_matrix/integrations/v1", + "https://scalar.vector.im/api", + "https://scalar-staging.vector.im/_matrix/integrations/v1", + "https://scalar-staging.vector.im/api", + "https://scalar-staging.riot.im/scalar/api" + ] + }, + "bug_report_endpoint_url": { + "type": "string", + "description": "Element's bug_report_endpoint_url setting of config.json: where Element sends a bug report, a URL; its default here is empty, and the file's own value (Element's, naming outside servers) stays in the file and is never replaced unless this is set.", + "trusted": true, + "default": "" + }, + "uisi_autorageshake_app": { + "type": "string", + "description": "Element's uisi_autorageshake_app setting of config.json: the app name Element files an automatic report of an undecryptable message under.", + "trusted": true, + "default": "element-auto-uisi" + }, + "show_labs_settings": { + "type": "boolean", + "description": "Element's show_labs_settings setting of config.json: whether Element shows its Labs settings.", + "trusted": true, + "default": true + }, + "room_directory": { + "type": "object", + "description": "Element's room_directory setting of config.json: the servers whose room directories Element offers; its default here is empty, and the file's own value (Element's, naming outside servers) stays in the file and is never replaced unless this is set.", + "trusted": true, + "default": {} + }, + "enable_presence_by_hs_url": { + "type": "object", + "description": "Element's enable_presence_by_hs_url setting of config.json: whether Element shows presence, per homeserver URL.", + "trusted": true, + "default": { + "https://matrix.org": false, + "https://matrix-client.matrix.org": false + } + }, + "terms_and_conditions_links": { + "type": "list", + "description": "Element's terms_and_conditions_links setting of config.json: the links Element shows as its terms, each a URL and its text; its default here is empty, and the file's own value (Element's, naming outside servers) stays in the file and is never replaced unless this is set.", + "trusted": true, + "default": [] + }, + "features": { + "type": "object", + "description": "Element's features setting of config.json: Element's feature switches, each on or off.", + "trusted": true, + "default": { + "feature_video_rooms": true, + "feature_rust_crypto": true + } + }, + "element_call": { + "type": "object", + "description": "Element's element_call setting of config.json: where Element starts a call with Element Call, a URL.", + "trusted": true, + "default": { + "url": "https://call.element.dev" + } + } + }, "resources": [ { "id": "state", @@ -65,7 +172,7 @@ "type": "file", "path": "${dir:state}/conduit.toml", "mode": "0644", - "content": "# Written by the mesh (modules/matrix). Conduit reads this file (CONDUIT_CONFIG); nothing comes\n# from the environment. server_name is the homeserver's permanent identity: every user id, room id\n# and signature in the database carries it, so it is the name this module is served under\n# (${bound:route:name-homeserver}) and never changes once a database exists.\n[global]\nserver_name = \"${bound:route:name-homeserver}\"\ndatabase_backend = \"rocksdb\"\ndatabase_path = \"/var/lib/matrix-conduit/\"\naddress = \"0.0.0.0\"\nport = 6167\nmax_request_size = 20000000\nallow_registration = false\nallow_federation = true\nallow_check_for_updates = true\ntrusted_servers = [\"matrix.org\"]\n", + "template": "files/conduit.toml", "names-on-purpose": { "matrix.org": "the federation's public key server, trusted by default; the world's, not this mesh's" } @@ -76,7 +183,8 @@ "path": "${dir:state}/element.json", "mode": "0644", "merge": "json", - "content": "{\n \"default_server_name\": \"${bound:route:name-homeserver}\",\n \"default_server_config\": {\n \"m.homeserver\": {\n \"base_url\": \"https://${bound:route:name-homeserver}\"\n },\n \"m.identity_server\": {\n \"base_url\": \"https://vector.im\"\n }\n },\n \"brand\": \"Element\",\n \"integrations_ui_url\": \"https://scalar.vector.im/\",\n \"integrations_rest_url\": \"https://scalar.vector.im/api\",\n \"integrations_widgets_urls\": [\n \"https://scalar.vector.im/_matrix/integrations/v1\",\n \"https://scalar.vector.im/api\",\n \"https://scalar-staging.vector.im/_matrix/integrations/v1\",\n \"https://scalar-staging.vector.im/api\",\n \"https://scalar-staging.riot.im/scalar/api\"\n ],\n \"bug_report_endpoint_url\": \"https://element.io/bugreports/submit\",\n \"uisi_autorageshake_app\": \"element-auto-uisi\",\n \"show_labs_settings\": true,\n \"room_directory\": {\n \"servers\": [\n \"${bound:route:name-homeserver}\",\n \"matrix.org\",\n \"gitter.im\",\n \"libera.chat\"\n ]\n },\n \"enable_presence_by_hs_url\": {\n \"https://matrix.org\": false,\n \"https://matrix-client.matrix.org\": false\n },\n \"terms_and_conditions_links\": [\n {\n \"url\": \"https://element.io/privacy\",\n \"text\": \"Privacy Policy\"\n },\n {\n \"url\": \"https://element.io/cookie-policy\",\n \"text\": \"Cookie Policy\"\n }\n ],\n \"features\": {\n \"feature_video_rooms\": true,\n \"feature_rust_crypto\": true\n },\n \"element_call\": {\n \"url\": \"https://call.element.dev\"\n }\n}\n", + "takes-config": ["default_server_name", "default_server_config", "brand", "integrations_ui_url", "integrations_rest_url", "integrations_widgets_urls", "bug_report_endpoint_url", "uisi_autorageshake_app", "show_labs_settings", "room_directory", "enable_presence_by_hs_url", "terms_and_conditions_links", "features", "element_call"], + "template": "files/element.json", "names-on-purpose": { "matrix.org": "the public room directory and the federation's largest homeserver; the world's", "matrix-client.matrix.org": "the same homeserver's client endpoint; the world's", @@ -134,5 +242,14 @@ "element-conf" ] } - ] + ], + "build": { + "artifacts": [ + { + "name": "files", + "kind": "archive", + "from": "files" + } + ] + } } diff --git a/modules/memory-pressure/cmd/memory-pressure/templates_test.go b/modules/memory-pressure/cmd/memory-pressure/templates_test.go new file mode 100644 index 00000000..7d0718b5 --- /dev/null +++ b/modules/memory-pressure/cmd/memory-pressure/templates_test.go @@ -0,0 +1,64 @@ +package main + +import ( + "encoding/json" + "os" + "path/filepath" + "testing" +) + +// The module's directory, where its manifest and its files/ are. +const moduleDir = "../.." + +// templateText is the text a file resource places on the machine: its template's, in the module's files/ +// (novox/hq ADR 0349 §2). A resource that names no template, or a template that is not there, fails the test. +func templateText(t *testing.T, r map[string]any) string { + t.Helper() + name, _ := r["template"].(string) + if name == "" { + t.Fatalf("resource %v names no template", r["id"]) + } + raw, err := os.ReadFile(filepath.Join(moduleDir, filepath.FromSlash(name))) + if err != nil { + t.Fatalf("resource %v names %s: %v", r["id"], name, err) + } + return string(raw) +} + +// No file resource carries its text inline: each names its template in files/, and the build packs files/ +// into the one archive the controller reads the templates back from (novox/hq ADR 0349 §2). +func TestEveryFileTakesItsTextFromATemplate(t *testing.T) { + raw, err := os.ReadFile(filepath.Join(moduleDir, "module.json")) + if err != nil { + t.Fatal(err) + } + var m struct { + Resources []map[string]any `json:"resources"` + Build struct { + Artifacts []map[string]any `json:"artifacts"` + } `json:"build"` + } + if err := json.Unmarshal(raw, &m); err != nil { + t.Fatal(err) + } + for _, r := range m.Resources { + if _, inline := r["content"]; inline { + t.Errorf("resource %v keeps its text inline: keep it in files/ and name it as the template", r["id"]) + } + if r["type"] == "file" { + templateText(t, r) + } + } + packs := 0 + for _, a := range m.Build.Artifacts { + if a["kind"] == "archive" && a["from"] == "files" { + packs++ + if a["name"] != "files" { + t.Errorf("the templates archive is named %v, not files", a["name"]) + } + } + } + if packs != 1 { + t.Errorf("the build packs files/ into %d archives, not one", packs) + } +} diff --git a/modules/memory-pressure/files/oomd.conf b/modules/memory-pressure/files/oomd.conf new file mode 100644 index 00000000..39af6def --- /dev/null +++ b/modules/memory-pressure/files/oomd.conf @@ -0,0 +1,8 @@ +# Managed by the mesh (module memory-pressure). Replaced on every push; edit the catalogue instead. +# +# systemd-oomd kills the worst unit before the kernel's OOM killer freezes the machine: when swap is +# 90 % used, or when a watched unit stalls on memory for 20 s above its limit. +[OOM] +SwapUsedLimit=90% +DefaultMemoryPressureLimit=60% +DefaultMemoryPressureDurationSec=20s diff --git a/modules/memory-pressure/files/root-slice.conf b/modules/memory-pressure/files/root-slice.conf new file mode 100644 index 00000000..edae3de8 --- /dev/null +++ b/modules/memory-pressure/files/root-slice.conf @@ -0,0 +1,5 @@ +# Managed by the mesh (module memory-pressure). Replaced on every push; edit the catalogue instead. +# +# systemd-oomd may act on swap exhaustion across the whole machine. +[Slice] +ManagedOOMSwap=kill diff --git a/modules/memory-pressure/files/sysctl.conf b/modules/memory-pressure/files/sysctl.conf new file mode 100644 index 00000000..678251cb --- /dev/null +++ b/modules/memory-pressure/files/sysctl.conf @@ -0,0 +1,9 @@ +# Managed by the mesh (module memory-pressure). Replaced on every push; edit the catalogue instead. +# +# Swap-in reads one page, not eight: read-ahead exists to amortise a disk's seek, and compressed swap in +# RAM has none — the speculation only costs decompression and memory. +vm.page-cluster = 0 +# +# vm.swappiness is deliberately left alone. The usual zram advice (150-180) holds only while the +# compressed swap has room; once it is full, a higher swappiness moves pages to the disk swap, the thing +# being avoided. Raise it only together with zram-size. diff --git a/modules/memory-pressure/files/user-manager.conf b/modules/memory-pressure/files/user-manager.conf new file mode 100644 index 00000000..8ef300c6 --- /dev/null +++ b/modules/memory-pressure/files/user-manager.conf @@ -0,0 +1,7 @@ +# Managed by the mesh (module memory-pressure). Replaced on every push; edit the catalogue instead. +# +# systemd-oomd may kill the worst unit in a person's service manager when its memory pressure stays +# above 80 % — instead of the kernel freezing the machine. +[Service] +ManagedOOMMemoryPressure=kill +ManagedOOMMemoryPressureLimit=80% diff --git a/modules/memory-pressure/files/zram-generator.conf b/modules/memory-pressure/files/zram-generator.conf new file mode 100644 index 00000000..cccfb05a --- /dev/null +++ b/modules/memory-pressure/files/zram-generator.conf @@ -0,0 +1,11 @@ +# Managed by the mesh (module memory-pressure). Replaced on every push; edit the catalogue instead. +# +# Compressed swap in RAM: fast, at a higher priority than any swap on disk, so it takes the everyday +# pressure before anything spills to a disk. Half the RAM, at most 16 GiB, compressed with zstd. +# The swap on disk, its size and whether one exists at all are the machine's, not this module's. +# +# Read by the generator at boot: a change applies at the next boot. +[zram0] +zram-size = min(ram / 2, 16384) +compression-algorithm = zstd +swap-priority = 100 diff --git a/modules/memory-pressure/module.json b/modules/memory-pressure/module.json index 95eeb71b..76ae53be 100644 --- a/modules/memory-pressure/module.json +++ b/modules/memory-pressure/module.json @@ -28,7 +28,7 @@ "type": "file", "path": "/etc/systemd/zram-generator.conf", "mode": "0644", - "content": "# Managed by the mesh (module memory-pressure). Replaced on every push; edit the catalogue instead.\n#\n# Compressed swap in RAM: fast, at a higher priority than any swap on disk, so it takes the everyday\n# pressure before anything spills to a disk. Half the RAM, at most 16 GiB, compressed with zstd.\n# The swap on disk, its size and whether one exists at all are the machine's, not this module's.\n#\n# Read by the generator at boot: a change applies at the next boot.\n[zram0]\nzram-size = min(ram / 2, 16384)\ncompression-algorithm = zstd\nswap-priority = 100\n" + "template": "files/zram-generator.conf" }, { "id": "sysctl-drop-ins", @@ -41,7 +41,7 @@ "type": "file", "path": "/etc/sysctl.d/90-memory-pressure.conf", "mode": "0644", - "content": "# Managed by the mesh (module memory-pressure). Replaced on every push; edit the catalogue instead.\n#\n# Swap-in reads one page, not eight: read-ahead exists to amortise a disk's seek, and compressed swap in\n# RAM has none — the speculation only costs decompression and memory.\nvm.page-cluster = 0\n#\n# vm.swappiness is deliberately left alone. The usual zram advice (150-180) holds only while the\n# compressed swap has room; once it is full, a higher swappiness moves pages to the disk swap, the thing\n# being avoided. Raise it only together with zram-size.\n" + "template": "files/sysctl.conf" }, { "id": "sysctl", @@ -62,7 +62,7 @@ "type": "file", "path": "/etc/systemd/oomd.conf.d/memory-pressure.conf", "mode": "0644", - "content": "# Managed by the mesh (module memory-pressure). Replaced on every push; edit the catalogue instead.\n#\n# systemd-oomd kills the worst unit before the kernel's OOM killer freezes the machine: when swap is\n# 90 % used, or when a watched unit stalls on memory for 20 s above its limit.\n[OOM]\nSwapUsedLimit=90%\nDefaultMemoryPressureLimit=60%\nDefaultMemoryPressureDurationSec=20s\n" + "template": "files/oomd.conf" }, { "id": "root-slice-drop-ins", @@ -75,7 +75,7 @@ "type": "file", "path": "/etc/systemd/system/-.slice.d/10-oomd.conf", "mode": "0644", - "content": "# Managed by the mesh (module memory-pressure). Replaced on every push; edit the catalogue instead.\n#\n# systemd-oomd may act on swap exhaustion across the whole machine.\n[Slice]\nManagedOOMSwap=kill\n" + "template": "files/root-slice.conf" }, { "id": "user-manager-drop-ins", @@ -88,7 +88,7 @@ "type": "file", "path": "/etc/systemd/system/user@.service.d/10-oomd.conf", "mode": "0644", - "content": "# Managed by the mesh (module memory-pressure). Replaced on every push; edit the catalogue instead.\n#\n# systemd-oomd may kill the worst unit in a person's service manager when its memory pressure stays\n# above 80 % — instead of the kernel freezing the machine.\n[Service]\nManagedOOMMemoryPressure=kill\nManagedOOMMemoryPressureLimit=80%\n" + "template": "files/user-manager.conf" }, { "id": "oomd", @@ -115,6 +115,11 @@ "loads": [ "memory-pressure" ] + }, + { + "name": "files", + "kind": "archive", + "from": "files" } ] } diff --git a/modules/mesh-catalog/files/database.url b/modules/mesh-catalog/files/database.url new file mode 100644 index 00000000..dc59da9a --- /dev/null +++ b/modules/mesh-catalog/files/database.url @@ -0,0 +1 @@ +postgresql://${bound:postgres-database:as}:${secret:postgres-database}@${bound:postgres-database:at}:${bound:postgres-database:port}/${bound:postgres-database:as} diff --git a/modules/mesh-catalog/module.json b/modules/mesh-catalog/module.json index 75f0a6d0..5a263800 100644 --- a/modules/mesh-catalog/module.json +++ b/modules/mesh-catalog/module.json @@ -48,7 +48,7 @@ "type": "file", "path": "${dir:state}/database.url", "mode": "0600", - "content": "postgresql://${bound:postgres-database:as}:${secret:postgres-database}@${bound:postgres-database:at}:${bound:postgres-database:port}/${bound:postgres-database:as}\n" + "template": "files/database.url" }, { "id": "prepare", @@ -86,6 +86,11 @@ "env": { "DATABASE_URL_FILE": "${dir:state}/database.url" } + }, + { + "name": "files", + "kind": "archive", + "from": "files" } ] } diff --git a/modules/mesh-cli/cmd/mesh-cli/manifest_test.go b/modules/mesh-cli/cmd/mesh-cli/manifest_test.go index bf143a9b..dec228c1 100644 --- a/modules/mesh-cli/cmd/mesh-cli/manifest_test.go +++ b/modules/mesh-cli/cmd/mesh-cli/manifest_test.go @@ -31,10 +31,22 @@ func TestTheArchivePlacesExactlyOneFile(t *testing.T) { if r["type"] != "archive" || r["path"] != "/usr/local/bin" { t.Fatalf("its resource is %v", r) } - if len(m.Build.Artifacts) != 1 { - t.Fatalf("it builds %d artifacts", len(m.Build.Artifacts)) + // The build has a second artifact, the files archive the controller reads the templates back from; it + // places no file. The artifact the resource unpacks is the one that does. + var a map[string]any + for _, candidate := range m.Build.Artifacts { + if candidate["name"] == r["artifact"] { + if a != nil { + t.Fatalf("two artifacts are named %v", r["artifact"]) + } + a = candidate + } else if candidate["kind"] != "archive" || candidate["from"] != "files" { + t.Fatalf("it builds %v beside the artifact that places the file", candidate) + } + } + if a == nil { + t.Fatalf("no artifact is named %v", r["artifact"]) } - a := m.Build.Artifacts[0] if a["name"] != r["artifact"] || a["kind"] != "bundle" || a["language"] != "go" || a["binary"] != "mesh-cli" || a["from"] != "cmd/mesh-cli" { t.Fatalf("its artifact is %v, unpacked as %v", a, r["artifact"]) diff --git a/modules/mesh-cli/files/zsh-normal b/modules/mesh-cli/files/zsh-normal new file mode 100644 index 00000000..117f2a5a --- /dev/null +++ b/modules/mesh-cli/files/zsh-normal @@ -0,0 +1,3 @@ +# The operator's command (module mesh-cli, novox/hq ADR 0272): nox is mesh-cli, in interactive zsh only. +# A script, sudo and a document meant to work everywhere say mesh-cli. +alias nox=mesh-cli diff --git a/modules/mesh-cli/module.json b/modules/mesh-cli/module.json index 3939eb35..a72f91ac 100644 --- a/modules/mesh-cli/module.json +++ b/modules/mesh-cli/module.json @@ -1,6 +1,7 @@ { "module": "mesh-cli", "version": "1", + "moved": true, "resources": [ { "id": "program", @@ -9,11 +10,10 @@ "path": "/usr/local/bin" } ], - "shell": [ + "items": [ { - "for": "zsh", - "slot": "normal", - "code": "# The operator's command (module mesh-cli, novox/hq ADR 0272): nox is mesh-cli, in interactive zsh only.\n# A script, sudo and a document meant to work everywhere say mesh-cli.\nalias nox=mesh-cli\n" + "collection": "shell:zsh:normal", + "template": "files/zsh-normal" } ], "build": { @@ -25,6 +25,11 @@ "system": "arch", "from": "cmd/mesh-cli", "binary": "mesh-cli" + }, + { + "name": "files", + "kind": "archive", + "from": "files" } ] } diff --git a/modules/mesh-config/README.md b/modules/mesh-config/README.md new file mode 100644 index 00000000..7a9ab019 --- /dev/null +++ b/modules/mesh-config/README.md @@ -0,0 +1,52 @@ +# mesh-config + +Holds the `mesh-config` seat: the owner of every config value (novox/hq ADR 0348 §9, design 57 K3). Go, one +binary (`cmd/mesh-config`), served by a runtime of its own on its own bus account (`runs-as`), so that its +account alone writes its buckets. + +## What it keeps + +- `mesh-config_values`: each value set, keyed by its level: `mesh.` (mesh config), `node..` + (node config) and `module...` (a module's own value on one node), each with who set it, + through which account, why and when. +- `mesh-config_resolved`: each assigned module's value for each key it declares on its node, keyed + `..`, with its source: `default`, `mesh`, `node`, `module`, or `unset` for a required value + nothing sets. +- `mesh-config_predefined`: the list of predefined keys, published from the list the controller kept (its mirror + `mesh-controller.predefined`), the newest trunk build winning. A newer list dropping a key that still holds a + value at the mesh or a node is withheld, and every listing says why, until that value is cleared. +- `mesh-config_proposals`: each value proposed for the operator's Approve, whole, with the value stored at its level + when it was proposed (what the operator is shown the change against), who proposed it, why, and what came of it. +- the log `history`: every set, clear and move of resolved values, with the proven caller and why. + +## How a value resolves + +From the controller's mirrors of the assignments and of each module's declarations (`mesh-controller.assignments`, +`mesh-controller.declared-config`): the manifest's default; mesh config for the predefined key the value falls back +to; node config for that key on the node; the module's own value on that node, the most specific winning. A value +naming no predefined key resolves from its default and the module's own value only (ADR 0348 §2). It resolves at +start, whenever a mirror moves, and every five minutes, writing only the values that moved. + +## The verb + +`mesh-config.config` with `action` `list` (the default), `set`, `clear` or `history`, narrowed by `module`, `node` +and `key`. A set or a clear names one level (no module and no node for mesh config, a node for node config, both +for a module's own value), needs `why`, refuses a key the module or the predefined list does not declare (naming +the declared keys), validates the value, writes it and the resolved values it moves, keeps it in the history and +the controller's hand-act log, raises `config-changed`, and answers with each module on each node whose value +moved. It opens no walk. A trusted value is taken only from the controller, which carries it from its terminal and +names who gave it in `by`, or on the operator's Approve of a proposal. + +## Proposing a value (ADR 0348 §8, design 57 K4) + +`action` `propose` (with `value`, or `clear`) judges the value as a set would, keeps it in `mesh-config_proposals`, +and asks the operator under mesh-config's own name on the `operator-channel` seat (its own grant, from `uses`, and +its own cap of three open asks; the same change twice is refused, and a newer proposal for the same level and key +replaces the older). The ask says the level, the key and the value, exact where who answers is proven, and that, +approved, it takes effect at once on every node it reaches; Approve and Decline are both at the level approve and +each binds the proposal's level, key, value and the value it was shown against. On the answer mesh-config reads +the router's own record of its ask and holds the warrant to the ask composed again from the proposal; it refuses, +and sets nothing, when the proposal or the value stored at the level changed since. An Approve that holds sets the +value as a set does, keeps who approved it beside the value (`approved`), and asks the controller to record the +warrant in its hand-act log. A Decline, an expiry or the router's refusal sets nothing. `action` `proposals` lists +them, newest first; with `proposal`, one read whole, with whether the router's ask is the one composed from it. diff --git a/modules/mesh-config/cmd/mesh-config/config.go b/modules/mesh-config/cmd/mesh-config/config.go new file mode 100644 index 00000000..cae5d361 --- /dev/null +++ b/modules/mesh-config/cmd/mesh-config/config.go @@ -0,0 +1,438 @@ +package main + +import ( + "encoding/json" + "errors" + "fmt" + "sort" + "strings" + "sync" + "time" +) + +// The verb `config` (novox/hq ADR 0348 §1, §8, §9, §10, design 57 K3): set or clear a value at one level with why, +// list the values, and read their history, narrowed by module and node. A set refuses a key the module or the list +// of predefined keys does not declare, naming the declared keys; validates the value; writes it and the resolved +// values it moves; records the change in the history and the hand-act log; raises `config-changed`; and answers +// with each module on each node whose value moved. A config change opens no walk and waits at no first-node gate. +// +// **Trusted values at the terminal or on the operator's Approve.** A trusted value (ADR 0348 §8) is set only by a +// call the controller carries from its terminal, or on the operator's warrant for a proposal (proposals.go, design +// 57 K4); a set through the verb is refused and names the proposal as the way. + +// ControllerCaller is the controller's bus user: the one caller that carries a value given at its terminal. +const ControllerCaller = "controller" + +// Unproven is the caller of a call the runtime named none for (the SDK's word). +const Unproven = "unproven" + +// Entry is one key's current value in a state. +type Entry struct { + Key string + Value json.RawMessage + Revision uint64 +} + +// Store is the bus as this module reaches it: the state it keeps and reads, its history log, its events, and the +// controller's hand-act log. +type Store interface { + Get(state, key string) (*Entry, error) + Put(state, key string, value any) error + Delete(state, key string) error + Keys(state string) ([]string, error) + Append(log, key string, value any) error + Read(log string) ([]LogEntry, error) + Emit(event string, body any) error + HandAct(what, why, cause string) error +} + +// LogEntry is one entry of a log. +type LogEntry struct { + Key string + Value json.RawMessage + At time.Time +} + +// ConfigChanged is the event every change raises: what was set or cleared at which level, by whom and why, and +// each module on each node whose resolved value moved. Never a secret: config holds none. +const ConfigChanged = "config-changed" + +// Change is a set, a clear, or a move of resolved values with no set behind it (a declaration or an assignment +// changed), as the history and the event say it. +type Change struct { + Action string `json:"action"` + Level *Level `json:"level,omitempty"` + Key string `json:"key,omitempty"` + Value json.RawMessage `json:"value,omitempty"` + Was json.RawMessage `json:"was,omitempty"` + By string `json:"by,omitempty"` + Via string `json:"via,omitempty"` + Why string `json:"why,omitempty"` + // Approved is who approved it on a channel that proves who answered, for a value set on the operator's warrant. + Approved string `json:"approved,omitempty"` + At time.Time `json:"at"` + Moved []Moved `json:"moved"` +} + +// Config is the module: its store, its clock, and one lock, so a set and a resolution never interleave. +type Config struct { + Store Store + Now func() time.Time + Logf func(format string, a ...any) + + mu sync.Mutex + // withheld is why the newest list of predefined keys was not published, empty when it was (publish). + withheld string + + // Asks is the operator channel as this module asks on it (proposals.go); nil, nothing can be proposed. + Asks AskBus + // RouterWithin and RouterEvery bound how long a proposal waits for the router's word on its ask, and how often + // it looks; zero is RouterAnswersWithin and RouterAnswersEvery. + RouterWithin, RouterEvery time.Duration + endedMu sync.Mutex + ended map[string]endedWord +} + +// Call is one call of the verb, its arguments read. +type Call struct { + Action string + Key string + Value any + Module string + Node string + Why string + // Caller is the bus user the runtime proved; By is who gave the value at the controller's terminal, said by the + // controller alone. + Caller string + By string + Limit int + // Clear asks a proposal to clear the value instead of setting it; Proposal names one proposal to read. + Clear bool + Proposal string + + // approval is the operator's warrant a set or a clear is made on: given by a decided proposal alone, never by + // a call's arguments. + approval *approval +} + +// SetAnswer is what a set or a clear answers. +type SetAnswer struct { + Said string `json:"said"` + Level Level `json:"level"` + Key string `json:"key"` + Value json.RawMessage `json:"value,omitempty"` + Was json.RawMessage `json:"was,omitempty"` + Moved []Moved `json:"moved"` + Result []NodeResult `json:"result"` + // Unsaid is what could not be recorded or announced after the value was written: said, never dropped. + Unsaid []string `json:"unsaid,omitempty"` +} + +// NodeResult is one affected node's result. Until the provisioner client fills files from mesh-config_resolved +// (design 57 P5, P6), a module on a node takes its new value at its next composition; that is said, not hidden. +type NodeResult struct { + Node string `json:"node"` + Modules []string `json:"modules"` + Result string `json:"result"` +} + +// Do runs one call of the verb. +func (c *Config) Do(call Call) (any, error) { + switch call.Action { + case "", "list": + return c.List(call) + case "set", "clear": + return c.Change(call) + case "history": + return c.History(call) + case "propose": + return c.Propose(call) + case "proposals": + return c.Proposals(call) + } + return nil, fmt.Errorf("config does set, clear, list, history, propose or proposals; %q is none of them", call.Action) +} + +// level is the level a call names, refused as ADR 0348 §3 says: a module's own value is for one node. +func (call Call) level() (Level, error) { + for _, name := range []string{call.Node, call.Module} { + if name != "" && !nodeOrModule.MatchString(name) { + return Level{}, fmt.Errorf("%q is not a node's or a module's name", name) + } + } + switch { + case call.Module != "" && call.Node != "": + return Level{Kind: LevelModule, Node: call.Node, Module: call.Module}, nil + case call.Module != "": + return Level{}, fmt.Errorf("a module's own value is set for one node, and no node is named: there is no "+ + "module value for the whole mesh (novox/hq ADR 0348 §3); name the node, or set mesh config for the "+ + "predefined key %s's value falls back to, or change its default in the catalogue", call.Module) + case call.Node != "": + return Level{Kind: LevelNode, Node: call.Node}, nil + } + return Level{Kind: LevelMesh}, nil +} + +// Change sets or clears one value. +func (c *Config) Change(call Call) (*SetAnswer, error) { + if call.Clear || call.Proposal != "" { + return nil, fmt.Errorf("clear and proposal are a proposal's: %s takes neither", call.Action) + } + call.approval = nil // only a decided proposal gives one + c.mu.Lock() + defer c.mu.Unlock() + return c.change(call) +} + +// change sets or clears one value; the caller holds the lock. +func (c *Config) change(call Call) (*SetAnswer, error) { + level, err := call.level() + if err != nil { + return nil, err + } + if !keyName.MatchString(call.Key) { + return nil, fmt.Errorf("name the key to %s: lower-case letters, digits, dots, dashes and underscores", call.Action) + } + if strings.TrimSpace(call.Why) == "" { + return nil, fmt.Errorf("say why the value is %s: every change keeps its why (novox/hq ADR 0348 §9)", + map[string]string{"set": "set", "clear": "cleared"}[call.Action]) + } + if call.Caller == "" || call.Caller == Unproven { + return nil, errors.New("the call names no proven caller, and every change keeps who made it " + + "(novox/hq ADR 0348 §9): call config through the mesh, which names you") + } + by := call.Caller + if call.By != "" { + if call.Caller != ControllerCaller { + return nil, fmt.Errorf("by names who gave a value at the controller's terminal, which only the controller "+ + "says; %s is not the controller", call.Caller) + } + by = call.By + } + s, err := c.sources() + if err != nil { + return nil, err + } + r, trusted, err := c.declared(s, level, call.Key) + if err != nil { + return nil, err + } + vk := level.ValueKey(call.Key) + was, had := s.Values[vk] + if call.approval != nil { + // **On Approve, the value stored must still be the one the operator was shown the change against** (ADR + // 0348 §8, ADR 0277 §13): a value set or cleared since, at the terminal or on another warrant, refuses it. + if had != call.approval.HadValue || valueDigest(was.Value, had) != call.approval.WasDigest { + return nil, fmt.Errorf("%s at %s is %s now, and the operator was shown the change against %s: it changed "+ + "since, so nothing is set; propose it again", call.Key, level, shownOrNone(was.Value, had), + shownOrNone(call.approval.Was, call.approval.HadValue)) + } + by = call.approval.By + } + answer := &SetAnswer{Level: level, Key: call.Key} + if had { + answer.Was = compact(was.Value) + } + now := c.now() + switch call.Action { + case "set": + if call.Value == nil { + return nil, fmt.Errorf("give the value to set %s to", call.Key) + } + raw, err := json.Marshal(call.Value) + if err != nil { + return nil, fmt.Errorf("the value cannot be read: %v", err) + } + var value any + _ = json.Unmarshal(raw, &value) + if problems := r.check(value); len(problems) > 0 { + return nil, fmt.Errorf("%s is refused for %s at %s: %s", show(value), call.Key, level, + strings.Join(problems, "; ")) + } + if trusted && call.Caller != ControllerCaller && call.approval == nil { + return nil, fmt.Errorf("%s is trusted (it decides what runs or who can reach what), so it is set only at "+ + "the controller's terminal or on the operator's Approve (novox/hq ADR 0348 §8); %s is not the terminal: "+ + "propose it instead, with config's action propose", call.Key, call.Caller) + } + v := Value{Value: compact(raw), By: by, Via: call.Caller, Why: strings.TrimSpace(call.Why), At: now} + if call.approval != nil { + v.Approved = call.approval.Says + } + if err := c.Store.Put(ValuesState, vk, v); err != nil { + return nil, fmt.Errorf("the value was not written: %v", err) + } + s.Values[vk] = v + answer.Value = v.Value + case "clear": + if !had { + return nil, fmt.Errorf("nothing is set for %s at %s, so there is nothing to clear", call.Key, level) + } + if trusted && call.Caller != ControllerCaller && call.approval == nil { + return nil, fmt.Errorf("%s is trusted, so it is cleared only at the controller's terminal or on the "+ + "operator's Approve (novox/hq ADR 0348 §8); %s is not the terminal: propose the clear instead, with "+ + "config's action propose and clear", call.Key, call.Caller) + } + if err := c.Store.Delete(ValuesState, vk); err != nil { + return nil, fmt.Errorf("the value was not cleared: %v", err) + } + delete(s.Values, vk) + } + moved, err := c.writeResolved(s) + answer.Moved = moved + if err != nil { + // The value is written and the resolved values are not all: said, and put right on the next resolution. + answer.Unsaid = append(answer.Unsaid, "the resolved values are not all written yet ("+err.Error()+"); "+ + "the next resolution writes them") + } + change := Change{Action: call.Action, Level: &level, Key: call.Key, Value: answer.Value, Was: answer.Was, By: by, + Via: call.Caller, Why: strings.TrimSpace(call.Why), At: now, Moved: moved} + cause := "the config verb, called by " + call.Caller + if call.approval != nil { + change.Approved = call.approval.Says + cause = "the operator's Approve of proposal " + call.approval.Proposal + ", proposed by " + call.Caller + } + answer.Unsaid = append(answer.Unsaid, c.record(vk, change)...) + if err := c.Store.HandAct(handActWhat(change), change.Why, cause); err != nil { + answer.Unsaid = append(answer.Unsaid, "the hand-act log does not hold it: "+err.Error()) + } + answer.Result = nodeResults(moved) + answer.Said = said(change) + return answer, nil +} + +// declared is the rule a value at a level is held to, and whether it is trusted; refused, naming the declared +// keys, for a key the module or the list of predefined keys does not declare (ADR 0348 §10). +func (c *Config) declared(s Sources, level Level, key string) (rule, bool, error) { + if level.Kind == LevelModule { + decl, ok := s.Declared[level.Module] + if !ok { + return rule{}, false, fmt.Errorf("%s is not a registered module, so it declares no config", level.Module) + } + d, ok := decl.Config[key] + if !ok { + return rule{}, false, fmt.Errorf("%s declares no value %q; it declares: %s", level.Module, key, + joinKeys(decl.Config)) + } + if !assigned(s, level.Node, level.Module) { + return rule{}, false, fmt.Errorf("%s is not assigned to %s, and a module's own value is set for a node it "+ + "is assigned to (novox/hq ADR 0348 §3); it is assigned to: %s", level.Module, level.Node, + strings.Join(nodesOf(s, level.Module), ", ")) + } + trusted := d.Trusted + if p, ok := s.Predefined[d.Predefined]; ok && p.Trusted { + trusted = true + } + return d.rule(), trusted, nil + } + if len(s.Predefined) == 0 { + return rule{}, false, fmt.Errorf("%s holds values only for the keys of the list of predefined keys, and no "+ + "list is published yet (novox/hq ADR 0348 §7)", level) + } + p, ok := s.Predefined[key] + if !ok { + return rule{}, false, fmt.Errorf("%s holds values only for the predefined keys, and %q is not one; the "+ + "predefined keys are: %s (novox/hq ADR 0348 §7)", level, key, joinKeys(s.Predefined)) + } + return p.rule(), p.Trusted, nil +} + +func assigned(s Sources, node, module string) bool { + for _, a := range s.Assignments { + if a.Node == node && a.Module == module { + return true + } + } + return false +} + +func nodesOf(s Sources, module string) []string { + var out []string + for _, a := range s.Assignments { + if a.Module == module { + out = append(out, a.Node) + } + } + sort.Strings(out) + if len(out) == 0 { + return []string{"no node"} + } + return out +} + +// record keeps a change in the history and raises its event; what could not be done is answered, not dropped. +func (c *Config) record(key string, change Change) []string { + var unsaid []string + if err := c.Store.Append(HistoryLog, key, change); err != nil { + unsaid = append(unsaid, "the history does not hold it: "+err.Error()) + } + if err := c.Store.Emit(ConfigChanged, change); err != nil { + unsaid = append(unsaid, "config-changed was not raised: "+err.Error()) + } + for _, u := range unsaid { + c.logf("[mesh-config] %s %s: %s", change.Action, key, u) + } + return unsaid +} + +func handActWhat(ch Change) string { + what := fmt.Sprintf("config %s %s at %s", ch.Action, ch.Key, ch.Level) + if ch.Action == "set" { + what += " to " + string(ch.Value) + } + return what + ", by " + ch.By +} + +func said(ch Change) string { + var b strings.Builder + if ch.Action == "set" { + fmt.Fprintf(&b, "%s is %s at %s", ch.Key, ch.Value, ch.Level) + } else { + fmt.Fprintf(&b, "%s is cleared at %s", ch.Key, ch.Level) + } + switch len(ch.Moved) { + case 0: + b.WriteString("; no module's value moved") + case 1: + m := ch.Moved[0] + fmt.Fprintf(&b, "; %s on %s takes %s (from %s)", m.Module, m.Node, orUnset(m.To), m.Source) + default: + fmt.Fprintf(&b, "; %d values moved on %d node(s)", len(ch.Moved), len(nodeResults(ch.Moved))) + } + return b.String() +} + +func orUnset(raw json.RawMessage) string { + if len(raw) == 0 { + return "no value (unset)" + } + return string(raw) +} + +func nodeResults(moved []Moved) []NodeResult { + byNode := map[string]map[string]bool{} + for _, m := range moved { + if byNode[m.Node] == nil { + byNode[m.Node] = map[string]bool{} + } + byNode[m.Node][m.Module] = true + } + out := []NodeResult{} + for _, node := range sortedKeys(byNode) { + out = append(out, NodeResult{Node: node, Modules: sortedKeys(byNode[node]), + Result: "resolved and written to mesh-config_resolved; the module takes it when its files are filled again"}) + } + return out +} + +func (c *Config) now() time.Time { + if c.Now == nil { + return time.Now().UTC() + } + return c.Now().UTC() +} + +func (c *Config) logf(format string, a ...any) { + if c.Logf != nil { + c.Logf(format, a...) + } +} diff --git a/modules/mesh-config/cmd/mesh-config/config_test.go b/modules/mesh-config/cmd/mesh-config/config_test.go new file mode 100644 index 00000000..e2418422 --- /dev/null +++ b/modules/mesh-config/cmd/mesh-config/config_test.go @@ -0,0 +1,553 @@ +package main + +import ( + "encoding/json" + "errors" + "fmt" + "sort" + "strings" + "sync" + "testing" + "time" +) + +// memStore is the bus as mesh-config reaches it, in memory: its states and the mirrors it reads, its history log, +// the events it raised and the hand-acts it recorded. +type memStore struct { + mu sync.Mutex + states map[string]map[string][]byte + logs map[string][]LogEntry + events []Change + handActs []string + // failHandAct makes the hand-act log refuse. + failHandAct bool +} + +func newStore() *memStore { + return &memStore{states: map[string]map[string][]byte{}, logs: map[string][]LogEntry{}} +} + +func (m *memStore) Get(state, key string) (*Entry, error) { + m.mu.Lock() + defer m.mu.Unlock() + v, ok := m.states[state][key] + if !ok { + return nil, nil + } + return &Entry{Key: key, Value: append([]byte(nil), v...)}, nil +} + +func (m *memStore) Put(state, key string, value any) error { + raw, err := json.Marshal(value) + if err != nil { + return err + } + m.mu.Lock() + defer m.mu.Unlock() + if m.states[state] == nil { + m.states[state] = map[string][]byte{} + } + m.states[state][key] = raw + return nil +} + +func (m *memStore) Delete(state, key string) error { + m.mu.Lock() + defer m.mu.Unlock() + delete(m.states[state], key) + return nil +} + +func (m *memStore) Keys(state string) ([]string, error) { + m.mu.Lock() + defer m.mu.Unlock() + var keys []string + for k := range m.states[state] { + keys = append(keys, k) + } + sort.Strings(keys) + return keys, nil +} + +func (m *memStore) Append(log, key string, value any) error { + raw, err := json.Marshal(value) + if err != nil { + return err + } + m.mu.Lock() + defer m.mu.Unlock() + m.logs[log] = append(m.logs[log], LogEntry{Key: key, Value: raw, At: time.Now()}) + return nil +} + +func (m *memStore) Read(log string) ([]LogEntry, error) { + m.mu.Lock() + defer m.mu.Unlock() + return append([]LogEntry(nil), m.logs[log]...), nil +} + +func (m *memStore) Emit(event string, body any) error { + if event != ConfigChanged { + return fmt.Errorf("mesh-config emits no %s", event) + } + m.mu.Lock() + defer m.mu.Unlock() + m.events = append(m.events, body.(Change)) + return nil +} + +func (m *memStore) HandAct(what, why, cause string) error { + m.mu.Lock() + defer m.mu.Unlock() + if m.failHandAct { + return errors.New("the controller did not answer") + } + m.handActs = append(m.handActs, what+" | "+why+" | "+cause) + return nil +} + +// mirror writes what the controller mirrors, in the shapes of its mirrors. +func (m *memStore) mirror(t *testing.T, read, key, raw string) { + t.Helper() + if !json.Valid([]byte(raw)) { + t.Fatalf("not JSON: %s", raw) + } + m.mu.Lock() + defer m.mu.Unlock() + if m.states[read] == nil { + m.states[read] = map[string][]byte{} + } + m.states[read][key] = []byte(raw) +} + +func (m *memStore) assign(t *testing.T, node, module string) { + m.mirror(t, AssignmentsRead, node+"."+module, + fmt.Sprintf(`{"node":%q,"module":%q,"assigned":"2026-10-14T00:00:00Z"}`, node, module)) +} + +func (m *memStore) resolved(t *testing.T, node, module, key string) Resolved { + t.Helper() + e, _ := m.Get(ResolvedState, ResolvedKey(node, module, key)) + if e == nil { + t.Fatalf("%s holds no resolved value for %s on %s", key, module, node) + } + var r Resolved + if err := json.Unmarshal(e.Value, &r); err != nil { + t.Fatal(err) + } + return r +} + +// dunstDeclared is dunst's declarations as the declared-config mirror on the feature branch writes them: kind, why +// and default always written, so a config-form key without a default comes as kind "", why "" and default null. +const dunstDeclared = `{"module":"dunst","settings":{ + "font-size":{"kind":"","default":10,"why":"","type":"number","description":"text size, in points","minimum":6,"maximum":72,"predefined":"font-size"}, + "width":{"kind":"","default":300,"why":"","type":"number","description":"the width of a notification, in pixels","minimum":100}, + "position":{"kind":"","default":"top-right","why":"","type":"string","description":"where notifications show","enum":["top-right","top-left"]}, + "origin":{"kind":"","default":null,"why":"","type":"string","description":"the operator's own origin","required":true,"pattern":"[a-z]+"}}}` + +// theList is the controller's predefined mirror, as it writes it. +const theList = `{"keys":{"agent-managed-settings":{"type":"object","description":"managed settings","trusted":true}, + "font-size":{"type":"number","description":"text size, in points","minimum":6,"maximum":72}}, + "repository":"mesh-catalog","commit":"aaaaaaaaaaaa","generation":5,"build":"b5"}` + +func aMesh(t *testing.T) (*Config, *memStore) { + t.Helper() + m := newStore() + m.mirror(t, DeclaredConfigRead, "dunst", dunstDeclared) + m.mirror(t, DeclaredConfigRead, "claude-code", `{"module":"claude-code","settings":{ + "managed":{"kind":"","default":{},"why":"","type":"object","description":"managed settings","trusted":true,"predefined":"agent-managed-settings"}}}`) + m.mirror(t, DeclaredConfigRead, "old", `{"module":"old","settings":{"workers":{"kind":"preference","default":4,"why":"two per core"}}}`) + m.mirror(t, PredefinedRead, PredefinedMirrorKey, theList) + for _, node := range []string{"g14", "novox"} { + m.assign(t, node, "dunst") + } + m.assign(t, "g14", "claude-code") + m.assign(t, "g14", "old") + clock := time.Date(2026, 10, 14, 19, 0, 0, 0, time.UTC) + c := &Config{Store: m, Now: func() time.Time { clock = clock.Add(time.Second); return clock }, Logf: t.Logf} + if _, err := c.Reconcile(); err != nil { + t.Fatal(err) + } + m.events = nil // the first resolution's + return c, m +} + +func set(t *testing.T, c *Config, call Call) *SetAnswer { + t.Helper() + if call.Action == "" { + call.Action = "set" + } + if call.Caller == "" { + call.Caller = "g14.node-tools" + } + if call.Why == "" { + call.Why = "the operator asked for it" + } + a, err := c.Change(call) + if err != nil { + t.Fatalf("%s %s: %v", call.Action, call.Key, err) + } + return a +} + +func expect(t *testing.T, m *memStore, node string, value, source string) { + t.Helper() + r := m.resolved(t, node, "dunst", "font-size") + if string(r.Value) != value || r.Source != source { + t.Errorf("dunst's font-size on %s is %s from %s, want %s from %s", node, r.Value, r.Source, value, source) + } +} + +// **The four sources, each in turn** (ADR 0348 §2, How it is checked): the manifest's default; mesh config for the +// predefined key it falls back to; node config for that key on the node; the module's own value on that node. Each +// cleared falls back to the one below. +func TestEachOfTheFourSourcesResolvesInTurnAndEachClearedFallsBack(t *testing.T) { + c, m := aMesh(t) + expect(t, m, "g14", "10", SourceDefault) + expect(t, m, "novox", "10", SourceDefault) + + a := set(t, c, Call{Key: "font-size", Value: 12.0}) + expect(t, m, "g14", "12", SourceMesh) + expect(t, m, "novox", "12", SourceMesh) + if len(a.Moved) != 2 || a.Level.Kind != LevelMesh { + t.Errorf("a mesh value moved %+v", a.Moved) + } + + a = set(t, c, Call{Key: "font-size", Value: 14.0, Node: "g14"}) + expect(t, m, "g14", "14", SourceNode) + expect(t, m, "novox", "12", SourceMesh) + if len(a.Moved) != 1 || a.Moved[0].Node != "g14" || string(a.Moved[0].From) != "12" || string(a.Moved[0].To) != "14" { + t.Errorf("g14's node value moved %+v", a.Moved) + } + + a = set(t, c, Call{Key: "font-size", Value: 16.0, Node: "g14", Module: "dunst"}) + expect(t, m, "g14", "16", SourceModule) + expect(t, m, "novox", "12", SourceMesh) + if len(a.Moved) != 1 || a.Moved[0].Source != SourceModule { + t.Errorf("dunst's own value moved %+v", a.Moved) + } + + // A value set below the module's own moves nothing it overrides. + if a = set(t, c, Call{Key: "font-size", Value: 18.0, Node: "g14"}); len(a.Moved) != 0 { + t.Errorf("a node value under a module's own moved %+v", a.Moved) + } + + set(t, c, Call{Action: "clear", Key: "font-size", Node: "g14", Module: "dunst"}) + expect(t, m, "g14", "18", SourceNode) + set(t, c, Call{Action: "clear", Key: "font-size", Node: "g14"}) + expect(t, m, "g14", "12", SourceMesh) + set(t, c, Call{Action: "clear", Key: "font-size"}) + expect(t, m, "g14", "10", SourceDefault) + expect(t, m, "novox", "10", SourceDefault) +} + +// **A value naming no predefined key ignores mesh and node config** (ADR 0348 §2): it resolves from its default and +// the module's own value only. +func TestAValueNamingNoPredefinedKeyResolvesFromItsDefaultAndItsOwnValueOnly(t *testing.T) { + c, m := aMesh(t) + set(t, c, Call{Key: "font-size", Value: 12.0}) + set(t, c, Call{Key: "font-size", Value: 14.0, Node: "g14"}) + if r := m.resolved(t, "g14", "dunst", "width"); string(r.Value) != "300" || r.Source != SourceDefault { + t.Errorf("width is %s from %s, want its default", r.Value, r.Source) + } + set(t, c, Call{Key: "width", Value: 400.0, Node: "g14", Module: "dunst"}) + if r := m.resolved(t, "g14", "dunst", "width"); string(r.Value) != "400" || r.Source != SourceModule { + t.Errorf("width is %s from %s, want dunst's own value", r.Value, r.Source) + } + // A module not yet moved, in the settings form: its default and its own value. + if r := m.resolved(t, "g14", "old", "workers"); string(r.Value) != "4" || r.Source != SourceDefault { + t.Errorf("old's workers is %s from %s", r.Value, r.Source) + } + set(t, c, Call{Key: "workers", Value: 8.0, Node: "g14", Module: "old"}) + if r := m.resolved(t, "g14", "old", "workers"); string(r.Value) != "8" || r.Source != SourceModule { + t.Errorf("old's workers is %s from %s", r.Value, r.Source) + } +} + +// **A required value nothing sets is answered unset** (ADR 0348 §4), and once set, it is the value. +func TestARequiredValueNothingSetsIsAnsweredUnset(t *testing.T) { + c, m := aMesh(t) + r := m.resolved(t, "g14", "dunst", "origin") + if r.Source != SourceUnset || !r.Required || len(r.Value) != 0 { + t.Errorf("an unset required value resolves as %+v", r) + } + set(t, c, Call{Key: "origin", Value: "home", Node: "g14", Module: "dunst"}) + if r = m.resolved(t, "g14", "dunst", "origin"); r.Source != SourceModule || string(r.Value) != `"home"` { + t.Errorf("a set required value resolves as %+v", r) + } + if r = m.resolved(t, "novox", "dunst", "origin"); r.Source != SourceUnset { + t.Errorf("another node's required value resolves as %+v", r) + } +} + +// **The verb refuses** (ADR 0348 §3, §8, §10, How it is checked): a key not declared, naming the declared keys; a +// module value without a node; a value failing validation; a trusted value without the terminal; a change without +// why or a proven caller. +func TestTheVerbRefusesWhatADeclarationDoesNotAllow(t *testing.T) { + c, m := aMesh(t) + for _, tc := range []struct { + name string + call Call + want string + }{ + {"a key the module does not declare", Call{Key: "colour", Value: "red", Node: "g14", Module: "dunst"}, + "it declares: font-size, origin, position, width"}, + {"a key not on the predefined list, at the mesh", Call{Key: "width", Value: 400.0}, + "the predefined keys are: agent-managed-settings, font-size"}, + {"a key not on the predefined list, at a node", Call{Key: "width", Value: 400.0, Node: "g14"}, "is not one"}, + {"a module value without a node", Call{Key: "width", Value: 400.0, Module: "dunst"}, + "no module value for the whole mesh"}, + {"a module on a node it is not assigned to", Call{Key: "width", Value: 400.0, Node: "ace", Module: "dunst"}, + "it is assigned to: g14, novox"}, + {"a module not registered", Call{Key: "width", Value: 400.0, Node: "g14", Module: "nobody"}, "not a registered module"}, + {"a value of another type", Call{Key: "font-size", Value: "12"}, `"12" is not a number`}, + {"a value below its minimum", Call{Key: "font-size", Value: 2.0, Node: "g14"}, "below the minimum 6"}, + {"a value above its maximum", Call{Key: "font-size", Value: 99.0, Node: "g14", Module: "dunst"}, "above the maximum 72"}, + {"a value not in its enum", Call{Key: "position", Value: "middle", Node: "g14", Module: "dunst"}, "is not one of"}, + {"a value not matching its pattern", Call{Key: "origin", Value: "Home 1", Node: "g14", Module: "dunst"}, + "does not match the pattern"}, + {"a trusted value at the mesh, not from the terminal", Call{Key: "agent-managed-settings", + Value: map[string]any{"a": 1.0}}, "set only at the controller's terminal"}, + {"a trusted module value, not from the terminal", Call{Key: "managed", Value: map[string]any{}, + Node: "g14", Module: "claude-code"}, "set only at the controller's terminal"}, + {"a change without why", Call{Key: "font-size", Value: 12.0, Why: " "}, "say why"}, + {"an unproven caller", Call{Key: "font-size", Value: 12.0, Caller: Unproven}, "no proven caller"}, + {"by from anyone but the controller", Call{Key: "font-size", Value: 12.0, By: "jochen"}, "only the controller says"}, + {"a clear of nothing set", Call{Action: "clear", Key: "font-size", Node: "g14"}, "nothing is set"}, + } { + t.Run(tc.name, func(t *testing.T) { + call := tc.call + if call.Action == "" { + call.Action = "set" + } + if call.Caller == "" { + call.Caller = "g14.node-tools" + } + if call.Why == "" { + call.Why = "a test" + } + _, err := c.Change(call) + if err == nil || !strings.Contains(err.Error(), tc.want) { + t.Fatalf("refused with %v, want %q", err, tc.want) + } + }) + } + if keys, _ := m.Keys(ValuesState); len(keys) != 0 { + t.Errorf("a refused call wrote %v", keys) + } + if len(m.events) != 0 || len(m.handActs) != 0 { + t.Errorf("a refused call raised %d events and recorded %d hand-acts", len(m.events), len(m.handActs)) + } +} + +// **A trusted value from the controller's terminal is set**, with who gave it there (ADR 0348 §8). +func TestATrustedValueFromTheTerminalIsSetWithWhoGaveIt(t *testing.T) { + c, m := aMesh(t) + a := set(t, c, Call{Key: "agent-managed-settings", Value: map[string]any{"b": 2.0, "a": 1.0}, + Caller: ControllerCaller, By: "jochen"}) + if string(a.Value) != `{"a":1,"b":2}` { + t.Errorf("the value is written as %s", a.Value) + } + r := m.resolved(t, "g14", "claude-code", "managed") + if r.Source != SourceMesh || !r.Trusted || string(r.Value) != `{"a":1,"b":2}` { + t.Errorf("claude-code's managed settings resolve as %+v", r) + } + e, _ := m.Get(ValuesState, "mesh.agent-managed-settings") + var v Value + _ = json.Unmarshal(e.Value, &v) + if v.By != "jochen" || v.Via != ControllerCaller { + t.Errorf("the value says it was set by %s via %s", v.By, v.Via) + } +} + +// **Every set and clear is in the history and the hand-act log with the proven caller and why, and raises +// config-changed naming what moved** (ADR 0348 §9, §10). +func TestEveryChangeIsKeptWithWhoAndWhyAndRaisesConfigChanged(t *testing.T) { + c, m := aMesh(t) + m.events = nil + set(t, c, Call{Key: "font-size", Value: 12.0, Caller: "novox.node-tools", Why: "bigger on every desk"}) + set(t, c, Call{Action: "clear", Key: "font-size", Caller: "g14.node-tools", Why: "back to the default"}) + + history, err := c.History(Call{}) + if err != nil { + t.Fatal(err) + } + var changes []HistoryEntry + for _, h := range history { + if h.Action == "set" || h.Action == "clear" { + changes = append(changes, h) + } + } + if len(changes) != 2 || changes[0].By != "novox.node-tools" || changes[0].Why != "bigger on every desk" || + changes[1].Action != "clear" || changes[1].By != "g14.node-tools" || string(changes[1].Was) != "12" { + t.Fatalf("the history holds %+v", changes) + } + if len(m.handActs) != 2 || !strings.Contains(m.handActs[0], "config set font-size at mesh config to 12, by novox.node-tools | bigger on every desk") { + t.Errorf("the hand-act log holds %q", m.handActs) + } + if len(m.events) != 2 || len(m.events[0].Moved) != 2 || m.events[0].Key != "font-size" || m.events[1].Action != "clear" { + t.Errorf("config-changed was raised as %+v", m.events) + } + // Narrowed: g14's dunst sees both, through what they moved; a node that holds no dunst sees neither. + if h, _ := c.History(Call{Module: "dunst", Node: "g14"}); len(h) < 2 { + t.Errorf("dunst on g14's history holds %d changes", len(h)) + } + if h, _ := c.History(Call{Node: "ace"}); len(h) != 0 { + t.Errorf("ace's history holds %+v", h) + } +} + +// **A hand-act the log refuses is said in the answer, never dropped**, and the value is set. +func TestAChangeTheHandActLogRefusesIsSaid(t *testing.T) { + c, m := aMesh(t) + m.failHandAct = true + a := set(t, c, Call{Key: "font-size", Value: 12.0}) + if len(a.Unsaid) != 1 || !strings.Contains(a.Unsaid[0], "hand-act log does not hold it") { + t.Errorf("the answer says %q", a.Unsaid) + } + expect(t, m, "g14", "12", SourceMesh) +} + +// **The newest list of predefined keys is published, and an older one heard later is not** (ADR 0348 §7). +func TestTheNewestListOfPredefinedKeysIsPublished(t *testing.T) { + c, m := aMesh(t) + published := func() map[string]Published { + out := map[string]Published{} + keys, _ := m.Keys(PredefinedState) + for _, k := range keys { + e, _ := m.Get(PredefinedState, k) + var p Published + _ = json.Unmarshal(e.Value, &p) + out[k] = p + } + return out + } + if p := published(); len(p) != 2 || p["font-size"].Generation != 5 || !p["agent-managed-settings"].Trusted { + t.Fatalf("published %+v", p) + } + m.mirror(t, PredefinedRead, PredefinedMirrorKey, `{"keys":{"font-size":{"type":"number","description":"x"}, + "theme":{"type":"string","description":"the theme"}},"commit":"bbbb","generation":7}`) + if _, err := c.Reconcile(); err != nil { + t.Fatal(err) + } + if p := published(); len(p) != 2 || p["theme"].Generation != 7 || p["agent-managed-settings"].Type != "" { + t.Fatalf("a newer list published %+v", p) + } + m.mirror(t, PredefinedRead, PredefinedMirrorKey, theList) // generation 5, heard later + if _, err := c.Reconcile(); err != nil { + t.Fatal(err) + } + if p := published(); p["theme"].Generation != 7 { + t.Fatalf("an older list replaced the newer one: %+v", p) + } +} + +// **A list dropping a key that still holds a value is withheld** (ADR 0348 §7): the list published stands, and the +// listing says why until the value is cleared. +func TestAListDroppingAKeyThatHoldsAValueIsWithheldUntilItIsCleared(t *testing.T) { + c, m := aMesh(t) + set(t, c, Call{Key: "font-size", Value: 12.0, Node: "novox"}) + m.mirror(t, PredefinedRead, PredefinedMirrorKey, `{"keys":{"agent-managed-settings":{"type":"object", + "description":"managed settings","trusted":true}},"commit":"cccccccccc","generation":9}`) + if _, err := c.Reconcile(); err != nil { + t.Fatal(err) + } + listed, err := c.List(Call{}) + if err != nil { + t.Fatal(err) + } + if _, kept := listed.Predefined["font-size"]; !kept || + !strings.Contains(listed.Withheld, "drops font-size (still set as node config on novox)") { + t.Fatalf("the list was published over a value it drops: withheld %q, published %v", listed.Withheld, + joinKeys(listed.Predefined)) + } + set(t, c, Call{Action: "clear", Key: "font-size", Node: "novox"}) + if _, err := c.Reconcile(); err != nil { + t.Fatal(err) + } + listed, _ = c.List(Call{}) + if _, kept := listed.Predefined["font-size"]; kept || listed.Withheld != "" { + t.Errorf("once cleared, the list is still withheld: %q, %v", listed.Withheld, joinKeys(listed.Predefined)) + } +} + +// **An assignment, a declaration or its removal moves the resolved values**, written only where they moved, and +// said as config-changed; a pass over unchanged sources writes nothing. +func TestAnAssignmentMovesTheResolvedValuesAndAQuietPassWritesNothing(t *testing.T) { + c, m := aMesh(t) + r, err := c.Reconcile() + if err != nil || len(r.Moved) != 0 { + t.Fatalf("a pass over unchanged sources moved %+v (%v)", r.Moved, err) + } + m.events = nil + m.assign(t, "ace", "dunst") + if r, _ = c.Reconcile(); len(r.Moved) != 4 { + t.Errorf("an assignment moved %+v", r.Moved) + } + expect(t, m, "ace", "10", SourceDefault) + if len(m.events) != 1 || m.events[0].Action != "resolve" { + t.Errorf("an assignment raised %+v", m.events) + } + _ = m.Delete(AssignmentsRead, "ace.dunst") + if r, _ = c.Reconcile(); len(r.Moved) != 4 || !r.Moved[0].Gone { + t.Errorf("an unassignment moved %+v", r.Moved) + } + if e, _ := m.Get(ResolvedState, ResolvedKey("ace", "dunst", "font-size")); e != nil { + t.Error("an unassigned module keeps its resolved values") + } +} + +// **The declared-config mirror's two shapes read the same** (the release lead's note on issue 652): kind, why and a +// null default written for a config-form key are not given, and a mirror that leaves them out reads alike. +func TestTheDeclaredConfigMirrorsTwoShapesReadTheSame(t *testing.T) { + with, err := ParseModuleDeclarations([]byte(`{"module":"x","settings":{"k":{"kind":"","default":null,"why":"","type":"string","description":"d","required":true}}}`)) + if err != nil { + t.Fatal(err) + } + without, err := ParseModuleDeclarations([]byte(`{"module":"x","settings":{"k":{"type":"string","description":"d","required":true}}}`)) + if err != nil { + t.Fatal(err) + } + a, _ := json.Marshal(with) + b, _ := json.Marshal(without) + if string(a) != string(b) || with.Config["k"].HasDefault() { + t.Errorf("the two shapes read as %s and %s", a, b) + } + if r := resolveOne(Sources{}, "n", "x", "k", with.Config["k"]); r.Source != SourceUnset { + t.Errorf("a null default resolved as %+v", r) + } +} + +// **A listing narrows by module and node**, and says the mesh and node values its modules fall back to. +func TestAListingNarrowsByModuleAndNode(t *testing.T) { + c, _ := aMesh(t) + set(t, c, Call{Key: "font-size", Value: 12.0}) + set(t, c, Call{Key: "font-size", Value: 14.0, Node: "novox"}) + l, err := c.List(Call{Module: "dunst", Node: "g14"}) + if err != nil { + t.Fatal(err) + } + if len(l.Resolved) != 4 || len(l.Set) != 1 || l.Set[0].Level.Kind != LevelMesh || l.Predefined != nil { + t.Errorf("dunst on g14 lists %d resolved and %+v", len(l.Resolved), l.Set) + } + l, _ = c.List(Call{Node: "novox"}) + if len(l.Resolved) != 4 || len(l.Set) != 2 { + t.Errorf("novox lists %d resolved and %+v", len(l.Resolved), l.Set) + } +} + +// **The verb's arguments are read strictly**: one it does not know is refused, naming what it takes. +func TestTheVerbsArgumentsAreReadStrictly(t *testing.T) { + if _, err := callOf(map[string]any{"action": "set", "level": "mesh"}); err == nil || + !strings.Contains(err.Error(), "config takes no level") { + t.Errorf("an unknown argument: %v", err) + } + if _, err := callOf(map[string]any{"key": 3.0}); err == nil { + t.Error("a key that is not a string was read") + } + call, err := callOf(map[string]any{"action": "set", "key": "font-size", "value": 12.0, "why": " w "}) + if err != nil || call.Value != 12.0 || call.Why != "w" { + t.Errorf("read %+v (%v)", call, err) + } +} diff --git a/modules/mesh-config/cmd/mesh-config/declared.go b/modules/mesh-config/cmd/mesh-config/declared.go new file mode 100644 index 00000000..6cf9f63f --- /dev/null +++ b/modules/mesh-config/cmd/mesh-config/declared.go @@ -0,0 +1,232 @@ +package main + +import ( + "bytes" + "encoding/json" + "fmt" + "math" + "regexp" + "slices" + "sort" + "strings" +) + +// What mesh-config resolves from (novox/hq ADR 0348 §2, §4, §7): each module's declarations, the list of predefined +// keys, and the assignments, as the controller mirrors them on the bus (design 57 F3, K3). + +// The types a config value may have (ADR 0348 §4). +const ( + TypeNumber = "number" + TypeString = "string" + TypeBoolean = "boolean" + TypeObject = "object" + TypeList = "list" +) + +// Declaration is one value a module declares, as the declared-config mirror holds it: the config form's fields, +// or the settings form's kind, default and why for a module not yet moved (ADR 0348 §1). The mirror writes kind, +// default and why for every key, so a config-form key comes as kind "", why "" and default null: an empty kind or +// why and a null default are read as not given, never as an empty value or a null default. +type Declaration struct { + Kind string `json:"kind,omitempty"` + Default json.RawMessage `json:"default,omitempty"` + Why string `json:"why,omitempty"` + Type string `json:"type,omitempty"` + Description string `json:"description,omitempty"` + Enum []any `json:"enum,omitempty"` + Minimum *float64 `json:"minimum,omitempty"` + Maximum *float64 `json:"maximum,omitempty"` + Pattern string `json:"pattern,omitempty"` + Trusted bool `json:"trusted,omitempty"` + Predefined string `json:"predefined,omitempty"` + Required bool `json:"required,omitempty"` + Runtime bool `json:"runtime,omitempty"` +} + +// HasDefault is whether the declaration gives a default: present and not null. +func (d Declaration) HasDefault() bool { + return len(bytes.TrimSpace(d.Default)) > 0 && string(bytes.TrimSpace(d.Default)) != "null" +} + +// TypeOf is the value's type: the config form's, or for the settings form the type of its default. +func (d Declaration) TypeOf() string { + if d.Type != "" { + return d.Type + } + if !d.HasDefault() { + return "" + } + var v any + if json.Unmarshal(d.Default, &v) != nil { + return "" + } + return typeOfValue(v) +} + +// ModuleDeclarations is one module's value of the declared-config mirror. +type ModuleDeclarations struct { + Module string `json:"module"` + Config map[string]Declaration `json:"settings"` +} + +// ParseModuleDeclarations reads one key of the declared-config mirror. +func ParseModuleDeclarations(raw []byte) (ModuleDeclarations, error) { + var m ModuleDeclarations + if err := json.Unmarshal(raw, &m); err != nil { + return m, fmt.Errorf("not a module's declarations: %w", err) + } + if m.Module == "" { + return m, fmt.Errorf("a module's declarations that name no module") + } + if m.Config == nil { + m.Config = map[string]Declaration{} + } + for key, d := range m.Config { + if !d.HasDefault() { + d.Default = nil + } + m.Config[key] = d + } + return m, nil +} + +// PredefinedKey is one key of the list of predefined keys (ADR 0348 §7): its type, description, validation and +// whether it is trusted. It has no default. +type PredefinedKey struct { + Type string `json:"type"` + Description string `json:"description"` + Enum []any `json:"enum,omitempty"` + Minimum *float64 `json:"minimum,omitempty"` + Maximum *float64 `json:"maximum,omitempty"` + Pattern string `json:"pattern,omitempty"` + Trusted bool `json:"trusted,omitempty"` +} + +// PredefinedList is the list the controller kept, as its predefined mirror holds it under PredefinedMirrorKey. +type PredefinedList struct { + Keys map[string]PredefinedKey `json:"keys"` + Repository string `json:"repository"` + Commit string `json:"commit"` + Generation int `json:"generation"` + Build string `json:"build"` +} + +// PredefinedMirrorKey is the controller's predefined mirror's one key. +const PredefinedMirrorKey = "list" + +// Published is one key of mesh-config_predefined: the key's declaration, and the build of the list it came from. +type Published struct { + PredefinedKey + Generation int `json:"generation"` + Commit string `json:"commit"` +} + +// Assignment is one key of the assignments mirror: a module on a node. +type Assignment struct { + Node string `json:"node"` + Module string `json:"module"` +} + +// rule is the validation a value is held to, from a module's declaration or a predefined key. +type rule struct { + Type string + Enum []any + Minimum *float64 + Maximum *float64 + Pattern string +} + +func (d Declaration) rule() rule { + return rule{Type: d.TypeOf(), Enum: d.Enum, Minimum: d.Minimum, Maximum: d.Maximum, Pattern: d.Pattern} +} + +func (k PredefinedKey) rule() rule { + return rule{Type: k.Type, Enum: k.Enum, Minimum: k.Minimum, Maximum: k.Maximum, Pattern: k.Pattern} +} + +// check is every way value breaks the rule, said in a sentence each; none for a value that holds. +func (r rule) check(value any) []string { + var problems []string + if r.Type != "" && typeOfValue(value) != r.Type { + return []string{fmt.Sprintf("%s is not a %s", show(value), r.Type)} + } + if len(r.Enum) > 0 && !slices.ContainsFunc(r.Enum, func(e any) bool { return sameValue(e, value) }) { + problems = append(problems, fmt.Sprintf("%s is not one of %s", show(value), show(r.Enum))) + } + if n, ok := value.(float64); ok { + if r.Minimum != nil && n < *r.Minimum { + problems = append(problems, fmt.Sprintf("%s is below the minimum %s", show(n), show(*r.Minimum))) + } + if r.Maximum != nil && n > *r.Maximum { + problems = append(problems, fmt.Sprintf("%s is above the maximum %s", show(n), show(*r.Maximum))) + } + if math.IsNaN(n) || math.IsInf(n, 0) { + problems = append(problems, "a number is finite") + } + } + if s, ok := value.(string); ok && r.Pattern != "" { + pattern, err := regexp.Compile(`^(?:` + r.Pattern + `)$`) + switch { + case err != nil: + problems = append(problems, fmt.Sprintf("its pattern %q cannot be read: %v", r.Pattern, err)) + case !pattern.MatchString(s): + problems = append(problems, fmt.Sprintf("%q does not match the pattern %q", s, r.Pattern)) + } + } + return problems +} + +// typeOfValue is the config type of a value read from JSON. +func typeOfValue(v any) string { + switch v.(type) { + case float64: + return TypeNumber + case string: + return TypeString + case bool: + return TypeBoolean + case map[string]any: + return TypeObject + case []any: + return TypeList + } + return "" +} + +func sameValue(a, b any) bool { + ra, _ := json.Marshal(a) + rb, _ := json.Marshal(b) + return bytes.Equal(ra, rb) +} + +func show(v any) string { + raw, err := json.Marshal(v) + if err != nil { + return fmt.Sprint(v) + } + return string(raw) +} + +// keyName is what a config key may be (the catalogue's rule): lower-case letters, digits, dots, dashes and +// underscores, starting with a letter or digit. +var keyName = regexp.MustCompile(`^[a-z0-9][a-z0-9_.-]*$`) + +// nodeOrModule is what a node or a module name may be: one token of a bucket's key. +var nodeOrModule = regexp.MustCompile(`^[A-Za-z0-9_-]+$`) + +func sortedKeys[V any](m map[string]V) []string { + out := make([]string, 0, len(m)) + for k := range m { + out = append(out, k) + } + sort.Strings(out) + return out +} + +func joinKeys[V any](m map[string]V) string { + keys := sortedKeys(m) + if len(keys) == 0 { + return "none" + } + return strings.Join(keys, ", ") +} diff --git a/modules/mesh-config/cmd/mesh-config/list.go b/modules/mesh-config/cmd/mesh-config/list.go new file mode 100644 index 00000000..a2169023 --- /dev/null +++ b/modules/mesh-config/cmd/mesh-config/list.go @@ -0,0 +1,144 @@ +package main + +import ( + "encoding/json" + "fmt" +) + +// Listing the values and reading their history, narrowed by module and node (ADR 0348 §1). + +// Listed is what a listing answers. +type Listed struct { + // Resolved is each module's value on each node the call narrows to, with where it came from. + Resolved []Resolved `json:"resolved"` + // Set is every value set at a level the call reaches, with who set it and why. + Set []SetValue `json:"set"` + // Predefined is the published list of predefined keys, for a call that names no module. + Predefined map[string]Published `json:"predefined,omitempty"` + // Withheld is why the newest list of predefined keys is not published, when it is not. + Withheld string `json:"withheld,omitempty"` +} + +// SetValue is one value set, at its level. +type SetValue struct { + Level Level `json:"level"` + Key string `json:"key"` + Value +} + +// List answers the resolved values and the values set, narrowed by module, node and key. +func (c *Config) List(call Call) (*Listed, error) { + for _, name := range []string{call.Node, call.Module} { + if name != "" && !nodeOrModule.MatchString(name) { + return nil, fmt.Errorf("%q is not a node's or a module's name", name) + } + } + c.mu.Lock() + withheld := c.withheld + c.mu.Unlock() + s, err := c.sources() + if err != nil { + return nil, err + } + if call.Module != "" { + if _, ok := s.Declared[call.Module]; !ok { + return nil, fmt.Errorf("%s is not a registered module, so it declares no config", call.Module) + } + } + out := &Listed{Resolved: []Resolved{}, Set: []SetValue{}, Withheld: withheld} + resolved := Resolve(s) + for _, k := range sortedKeys(resolved) { + r := resolved[k] + if (call.Module == "" || r.Module == call.Module) && (call.Node == "" || r.Node == call.Node) && + (call.Key == "" || r.Key == call.Key || r.Predefined == call.Key) { + out.Resolved = append(out.Resolved, r) + } + } + // The predefined keys the narrowed modules fall back to: their mesh and node values are listed too. + fallsBack := map[string]bool{} + for _, r := range out.Resolved { + if r.Predefined != "" { + fallsBack[r.Predefined] = true + } + } + for _, k := range sortedKeys(s.Values) { + level, key, ok := parseValueKey(k) + if !ok { + continue + } + if call.Key != "" && key != call.Key { + continue + } + reached := false + switch level.Kind { + case LevelMesh: + reached = call.Module == "" || fallsBack[key] + case LevelNode: + reached = (call.Node == "" || level.Node == call.Node) && (call.Module == "" || fallsBack[key]) + case LevelModule: + reached = (call.Node == "" || level.Node == call.Node) && (call.Module == "" || level.Module == call.Module) + } + if reached { + out.Set = append(out.Set, SetValue{Level: level, Key: key, Value: s.Values[k]}) + } + } + if call.Module == "" { + out.Predefined = s.Predefined + } + return out, nil +} + +// HistoryEntry is one change as the history keeps it. +type HistoryEntry struct { + Key string `json:"key"` + Change +} + +// History is the changes the call narrows to, newest last, at most its limit (100 when it gives none). +func (c *Config) History(call Call) ([]HistoryEntry, error) { + entries, err := c.Store.Read(HistoryLog) + if err != nil { + return nil, fmt.Errorf("the history cannot be read: %w", err) + } + limit := call.Limit + if limit <= 0 { + limit = 100 + } + out := []HistoryEntry{} + for _, e := range entries { + var ch Change + if err := json.Unmarshal(e.Value, &ch); err != nil { + continue + } + if !reaches(call, e.Key, ch) { + continue + } + out = append(out, HistoryEntry{Key: e.Key, Change: ch}) + } + if len(out) > limit { + out = out[len(out)-limit:] + } + return out, nil +} + +// reaches is whether a change concerns what the call narrows to: its level, or a value it moved. +func reaches(call Call, key string, ch Change) bool { + if call.Module == "" && call.Node == "" && call.Key == "" { + return true + } + if level, k, ok := parseValueKey(key); ok { + if (call.Key == "" || k == call.Key) && (call.Node == "" || level.Node == call.Node) && + (call.Module == "" || level.Module == call.Module) { + return true + } + } + // A change at another level reaches the call through what it moved there: mesh config moving a module's + // value on the node named. + for _, m := range ch.Moved { + if (call.Module == "" || m.Module == call.Module) && (call.Node == "" || m.Node == call.Node) && + (call.Key == "" || m.Key == call.Key || ch.Key == call.Key) { + return true + } + } + return false +} diff --git a/modules/mesh-config/cmd/mesh-config/main.go b/modules/mesh-config/cmd/mesh-config/main.go new file mode 100644 index 00000000..c915d3ef --- /dev/null +++ b/modules/mesh-config/cmd/mesh-config/main.go @@ -0,0 +1,283 @@ +// mesh-config: the owner of every config value, holder of the mesh-config seat (novox/hq ADR 0348 §9, design 57 +// K3). A Go bundle served by a runtime of its own, on its own bus account (`runs-as`, ADR 0259 §8), so that its +// account alone writes its buckets: mesh-config_values (each value set, at the mesh, a node, or a module on a +// node), mesh-config_resolved (each module's value on each node, with its source), mesh-config_predefined (the +// published list of predefined keys) and mesh-config_proposals (each value proposed for the operator's Approve, +// asked under this module's own name, design 57 K4). +// It resolves from the controller's mirrors of the assignments, the modules' declarations and the list it kept +// (design 57 F3, K3). stdout is the MCP channel; what this module says, it says on stderr. +package main + +import ( + "encoding/json" + "errors" + "fmt" + "os" + "strings" + "time" + + stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/asks" +) + +// Seat is the role this module holds, once for the mesh, and Verb its one verb. +const ( + Seat = "mesh-config" + Verb = "config" +) + +func logf(format string, a ...any) { fmt.Fprintf(os.Stderr, format+"\n", a...) } + +// runtimeStore is the bus, through the runtime that launched this bundle. +type runtimeStore struct{} + +func (runtimeStore) Get(state, key string) (*Entry, error) { + e, err := stdio.State(state).Get(key) + if err != nil || e == nil { + return nil, err + } + return &Entry{Key: e.Key, Value: e.Value, Revision: e.Revision}, nil +} + +func (runtimeStore) Put(state, key string, value any) error { + _, err := stdio.State(state).Put(key, value) + return err +} + +func (runtimeStore) Delete(state, key string) error { return stdio.State(state).Delete(key) } + +func (runtimeStore) Keys(state string) ([]string, error) { return stdio.State(state).Keys() } + +func (runtimeStore) Append(log, key string, value any) error { + _, err := stdio.Log(log).Append(key, value) + return err +} + +func (runtimeStore) Read(log string) ([]LogEntry, error) { + var out []LogEntry + for from := uint64(0); ; { + page, err := stdio.Log(log).Read(from, "", 1000) + if err != nil { + return nil, err + } + for _, e := range page.Entries { + out = append(out, LogEntry{Key: e.Key, Value: e.Value, At: e.At}) + } + if page.Next == 0 { + return out, nil + } + from = page.Next + } +} + +func (runtimeStore) Emit(event string, body any) error { return stdio.Emit(event, body) } + +// HandAct records a change in the controller's hand-act log (to-be 45 §7) through its seat's verb. +func (runtimeStore) HandAct(what, why, cause string) error { + _, err := stdio.Ask("seat:mesh-controller.hand-act", map[string]any{"what": what, "why": why, "cause": cause}) + return err +} + +// runtimeAsks is the operator channel through the runtime: asks published under this module's own name, the +// router's records of them read under that name alone (its `uses` of the seat, ADR 0259 §3), and the controller's +// hand-act log for a warrant acted on. +type runtimeAsks struct{} + +func (runtimeAsks) Publish(subject string, body any, id string) error { + return stdio.SeatPublish(subject, body, id) +} +func (runtimeAsks) Record(id string) (*stdio.StateEntry, error) { return stdio.SeatRecord("", id) } +func (runtimeAsks) Kinds() ([]stdio.KindHeld, error) { return stdio.SeatKinds() } +func (runtimeAsks) Warranted(ask string) error { + _, err := stdio.Ask("seat:mesh-controller.warranted", map[string]any{"asker": Module, "ask": ask}) + return err +} + +func main() { + c := &Config{Store: runtimeStore{}, Logf: logf, Asks: runtimeAsks{}} + go resolving(c) + go hearing(c) + if err := stdio.Serve("", Tools(c)); err != nil { + logf("%v", err) + os.Exit(1) + } +} + +// resolving resolves at start and again whenever a source moves, and every five minutes besides, so a change +// heard nowhere is still resolved. The runtime answers only once Serve runs, so the watches are asked again +// until they are taken. +func resolving(c *Config) { + moved := make(chan struct{}, 1) + poke := func(stdio.StateChange) error { + select { + case moved <- struct{}{}: + default: + } + return nil + } + for _, read := range []string{AssignmentsRead, DeclaredConfigRead, PredefinedRead} { + go func(read string) { + for wait := 2 * time.Second; ; wait = min(wait*2, time.Minute) { + time.Sleep(wait) + if err := stdio.State(read).Watch("", poke); err != nil { + logf("[mesh-config] %s is not watched yet (%v); asking again in %s", read, err, wait) + continue + } + return + } + }(read) + } + tick := time.NewTicker(5 * time.Minute) + defer tick.Stop() + time.Sleep(time.Second) + for { + r, err := c.Reconcile() + switch { + case err != nil: + logf("[mesh-config] the values are not all resolved: %v", err) + case len(r.Moved) > 0: + logf("[mesh-config] resolved: %d value(s) moved", len(r.Moved)) + } + if r.Published != "" { + logf("[mesh-config] published %s", r.Published) + } + select { + case <-moved: + time.Sleep(500 * time.Millisecond) // a burst of writes is one resolution + select { + case <-moved: + default: + } + case <-tick.C: + } + } +} + +// hearing takes the router's word on this module's asks, asked again until the runtime takes the subscription, and +// reads the router's record of every waiting ask once a minute besides, so an answer heard nowhere is still taken. +func hearing(c *Config) { + go func() { + for wait := 2 * time.Second; ; wait = min(wait*2, time.Minute) { + time.Sleep(wait) + err := stdio.Subscribe(asks.Seat+".decided."+Module, func(e stdio.Envelope) error { + var w asks.Warrant + if json.Unmarshal(e.Body, &w) == nil { + for _, s := range c.HeardEnd(w) { + logf("[mesh-config] %s", s) + } + } + return nil + }) + if err == nil { + logf("[mesh-config] hears the operator's answers") + return + } + logf("[mesh-config] does not hear the operator's answers yet (%v); asking again in %s", err, wait) + } + }() + tick := time.NewTicker(time.Minute) + defer tick.Stop() + for range tick.C { + said, err := c.Decide() + if err != nil { + logf("[mesh-config] reading the operator's answers: %v", err) + } + for _, s := range said { + logf("[mesh-config] %s", s) + } + } +} + +// Tools is the seat's verb as the runtime serves it. +func Tools(c *Config) []stdio.Tool { + return []stdio.Tool{{ + Name: Seat + "." + Verb, + Description: "Set, clear, list or read the history of config values (novox/hq ADR 0348). A value is set at " + + "one level: mesh config (no module, no node) or node config (a node) for a predefined key, or a " + + "module's own value on one node (module and node). Each set or clear needs why, is validated against " + + "its declaration, names each module on each node whose value moves, and is kept in the history with " + + "who and why. A trusted value is set only at the controller's terminal or on the operator's Approve: " + + "propose asks the operator, under mesh-config's own name on a channel that proves who answered, and sets " + + "the value on Approve if what it was shown against still stands; proposals lists them, or one with " + + "proposal. list and history are narrowed by module, node and key.", + Input: map[string]any{"type": "object", "properties": map[string]any{ + "action": map[string]any{"type": "string", + "enum": []string{"list", "set", "clear", "history", "propose", "proposals"}, + "description": "list when not given"}, + "key": map[string]any{"type": "string", "description": "the config key"}, + "value": map[string]any{"description": "for set: the value, of the key's declared type"}, + "module": map[string]any{"type": "string", "description": "a module: its own value, with node"}, + "node": map[string]any{"type": "string", "description": "a node"}, + "why": map[string]any{"type": "string", "description": "for set and clear: why, in a sentence"}, + "by": map[string]any{"type": "string", + "description": "who gave the value at the controller's terminal: said by the controller alone"}, + "limit": map[string]any{"type": "number", "description": "for history: at most this many, newest"}, + "clear": map[string]any{"type": "boolean", + "description": "for propose: propose clearing the value at its level instead of setting one"}, + "proposal": map[string]any{"type": "string", "description": "for proposals: one proposal's id, read whole"}, + }}, + Run: func(args map[string]any) (any, error) { + call, err := callOf(args) + if err != nil { + return nil, err + } + call.Caller = stdio.CallerOf(args) + return c.Do(call) + }, + }} +} + +// callOf reads a call's arguments strictly: an argument it does not know is refused, not ignored. +func callOf(args map[string]any) (Call, error) { + known := map[string]bool{"action": true, "key": true, "value": true, "module": true, "node": true, "why": true, + "by": true, "limit": true, "clear": true, "proposal": true} + var unknown []string + for k := range args { + if !known[k] { + unknown = append(unknown, k) + } + } + if len(unknown) > 0 { + return Call{}, fmt.Errorf("config takes no %s; it takes action, key, value, module, node, why, by, limit, "+ + "clear and proposal", + strings.Join(unknown, ", ")) + } + str := func(k string) (string, error) { + v, ok := args[k] + if !ok || v == nil { + return "", nil + } + s, ok := v.(string) + if !ok { + return "", fmt.Errorf("%s is a string", k) + } + return strings.TrimSpace(s), nil + } + var call Call + var problems []error + for k, into := range map[string]*string{"action": &call.Action, "key": &call.Key, "module": &call.Module, + "node": &call.Node, "why": &call.Why, "by": &call.By, "proposal": &call.Proposal} { + s, err := str(k) + problems = append(problems, err) + *into = s + } + if v, ok := args["limit"]; ok && v != nil { + n, ok := v.(float64) + if !ok || n < 0 || n != float64(int(n)) { + problems = append(problems, errors.New("limit is a whole number")) + } + call.Limit = int(n) + } + if v, ok := args["clear"]; ok && v != nil { + b, ok := v.(bool) + if !ok { + problems = append(problems, errors.New("clear is true or false")) + } + call.Clear = b + } + if v, ok := args["value"]; ok { + // The value as given, a JSON value: a number for a number, never its text; the validation judges it. + call.Value = v + } + return call, errors.Join(problems...) +} diff --git a/modules/mesh-config/cmd/mesh-config/proposals.go b/modules/mesh-config/cmd/mesh-config/proposals.go new file mode 100644 index 00000000..84d53a09 --- /dev/null +++ b/modules/mesh-config/cmd/mesh-config/proposals.go @@ -0,0 +1,928 @@ +package main + +// A config value proposed for the operator's Approve (novox/hq ADR 0348 §8, ADR 0277's rules with mesh-config as +// the asker; design 57 K4). +// +// Whoever the bus admits may PROPOSE a value at one level — mesh config, node config or a module's own value on a +// node — a trusted one among them, which the verb sets only from the controller's terminal. A proposal changes +// nothing. It is judged as a set judges it, then kept in mesh-config_proposals (which this module's account alone +// writes) with the value whole, the value stored at that level when it was proposed (what the operator is shown +// the change against), who proposed it, why and when. It is asked of the operator **under mesh-config's own name** +// on the operator-channel seat — its own grant, from its manifest's `uses`, and its own cap of open asks — as an +// ask whose two answers, Approve and Decline, are both at the level approve, so only a channel that proves who +// answered carries either. The ask says the level, the key and the value, and that, approved, it takes effect at +// once on every node it reaches; each option binds the proposal's level, key, the digest of the exact value and of +// the value it was shown against. +// +// **On the answer** mesh-config reads the router's own record of its ask (which only the router writes, and this +// module reads under its own name alone), composes the ask again from the proposal and holds the warrant to it: +// the same ask digest, an option it offered, at its level, in time (asks.Warrant.For), and the act the option +// bound is this proposal's (asks.Option.Performs). A record whose value or shown value changed after it was asked +// is refused, and so is an Approve when the value stored at the level is no longer the one the operator was shown +// the change against. Then the value is set exactly as a set at the terminal is, with the same judgement and +// history, and who approved it is kept beside the value; the warrant is recorded in the controller's hand-act log. + +import ( + "crypto/rand" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "sort" + "strconv" + "strings" + "time" + + stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/asks" + "git.novox.be/novox/mesh-sdk/go/outward" +) + +// Module is this module's name: the one the bus grants its asks under. +const Module = "mesh-config" + +// ProposalVerb is the act a proposal's answers bind. +const ProposalVerb = Seat + "." + Verb + +// ProposalLasts is how long the operator has to answer: a day less ten minutes (ADR 0277 §9), under the router's +// bound for an approval. +const ProposalLasts = asks.ApproveLasts - 10*time.Minute + +// How long a proposal waits for the router's word on its new ask, and how often it looks (ADR 0277 §20). +const ( + RouterAnswersWithin = 8 * time.Second + RouterAnswersEvery = 250 * time.Millisecond +) + +// The two answers, both at the level approve. +const ( + OptionApprove = "approve" + OptionDecline = "decline" +) + +// Where a proposal stands. +const ( + ProposalWaiting = "waiting" + ProposalApproved = "approved" + ProposalDeclined = "declined" + ProposalEnded = "ended" + ProposalRefused = "refused" + ProposalUnasked = "not asked" +) + +// AskBus is the operator channel as this module reaches it: publish an ask under its own name, read the router's +// record of one of its asks, the kinds of the channel bench with what each promises, and the controller's +// hand-act log for a warrant acted on. +type AskBus interface { + Publish(subject string, body any, id string) error + Record(id string) (*stdio.StateEntry, error) + Kinds() ([]stdio.KindHeld, error) + Warranted(ask string) error +} + +// Proposal is one value proposed, as mesh-config_proposals keeps it. Every field the ask is composed from is here, +// so the ask is composed again from it alone to hold the warrant to. +type Proposal struct { + ID string `json:"id"` + Action string `json:"action"` + Level Level `json:"level"` + Key string `json:"key"` + // Value is the value proposed, whole, and Digest its digest (valueDigest); a clear carries none. + Value json.RawMessage `json:"value,omitempty"` + Digest string `json:"digest"` + // Was is the value stored at the level when it was proposed, HadValue whether there was one, and WasDigest its + // digest: what the operator is shown the change against, and what must still stand on Approve. + Was json.RawMessage `json:"was,omitempty"` + HadValue bool `json:"had-value"` + WasDigest string `json:"was-digest"` + Trusted bool `json:"trusted,omitempty"` + // Reaches is each module on each node whose value it moves, as it was judged when proposed. + Reaches []string `json:"reaches"` + // From is who proposed it, as the bus proved the caller; Why is theirs; At is when. + From string `json:"from"` + Why string `json:"why"` + At time.Time `json:"at"` + + // State is where it stands, Outcome what came of it in words, Decided when; Answered is the warrant a person's + // choice held on (Approve or Decline), and Approved who approved it. + State string `json:"state"` + Outcome string `json:"outcome,omitempty"` + Decided time.Time `json:"decided,omitempty"` + Answered string `json:"answered,omitempty"` + Approved string `json:"approved,omitempty"` + // Recorded says the controller's hand-act log holds the warrant; Record is why not, while it does not. + Recorded bool `json:"recorded,omitempty"` + Record string `json:"record,omitempty"` +} + +// approval is what a decided proposal gives the set it makes: the warrant in words, and what the operator was +// shown the change against. +type approval struct { + Proposal string + By string + Says string + Was json.RawMessage + HadValue bool + WasDigest string +} + +// valueDigest is the digest a proposal binds of a value: SHA-256 over its one encoding (compact), "none" when +// there is no value. +func valueDigest(raw json.RawMessage, had bool) string { + text := "none" + if had { + text = "value:" + string(compact(raw)) + } + sum := sha256.Sum256([]byte(text)) + return "sha256:" + hex.EncodeToString(sum[:]) +} + +func shownOrNone(raw json.RawMessage, had bool) string { + if !had { + return "no value set" + } + return string(compact(raw)) +} + +// fingerprint is a digest as the operator is shown it: its first 24 hexadecimal digits in groups of four. +func fingerprint(digest string) string { + h := strings.TrimPrefix(digest, "sha256:") + if len(h) < 24 { + return h + } + var g []string + for i := 0; i < 24; i += 4 { + g = append(g, h[i:i+4]) + } + return strings.Join(g, " ") +} + +// Fingerprint is the proposal as the operator is shown it: the value proposed, or for a clear the value it clears. +func (p Proposal) Fingerprint() string { + if p.Action == "clear" { + return fingerprint(p.WasDigest) + } + return fingerprint(p.Digest) +} + +// about is what its ask is about: the value's place, so a newer proposal for it replaces an older one still open +// (ADR 0277 §10, at a level, a key). +func (p Proposal) about() string { return "config." + p.Level.ValueKey(p.Key) } + +// Act is what an answer binds: this proposal's level, key, the digest of its exact value and of the value it was +// shown against, who proposed it and when, and the answer. +func (p Proposal) Act(option string) asks.Act { + return asks.Act{"verb": ProposalVerb, "option": option, "proposal": p.ID, "action": p.Action, + "level": p.Level.Kind, "node": p.Level.Node, "module": p.Level.Module, "key": p.Key, "value": p.Digest, + "was": p.WasDigest, "had-value": strconv.FormatBool(p.HadValue), "from": p.From, + "at": p.At.UTC().Format(time.RFC3339Nano)} +} + +// ---- what the operator reads -------------------------------------------------------------------------- + +// Markers for what a phone is not shown. +const ( + markAddress = "‹address›" + markPath = "‹path›" + markWithheld = "‹withheld›" +) + +// sayable is a text as a channel that proves nothing is shown it, under the content rule (outward.Check): as it +// is, or the marker of what it carries. +func sayable(text string) (string, bool) { + refusal, ok := outward.Check(text) + if ok { + return text, true + } + switch refusal.Class { + case "address": + return markAddress, false + case "path": + return markPath, false + } + return markWithheld, false +} + +// change is the change in one line, "+ key: value", "~ key: value (was: old)" or "- key (was: old)": each value +// exact (for a channel that proves who answers, ADR 0277 §6 and issue 383) or under the content rule. +func (p Proposal) change(exact bool) string { + say := func(raw json.RawMessage) string { + text := string(compact(raw)) + if exact { + return text + } + s, _ := sayable(text) + return s + } + switch { + case p.Action == "clear": + return fmt.Sprintf("- %s (was: %s)", p.Key, say(p.Was)) + case p.HadValue: + return fmt.Sprintf("~ %s: %s (was: %s)", p.Key, say(p.Value), say(p.Was)) + } + return fmt.Sprintf("+ %s: %s", p.Key, say(p.Value)) +} + +// whole is whether every value is shown exactly under the content rule. +func (p Proposal) whole() bool { return p.change(false) == p.change(true) } + +// reaches is every node it reaches, in words. +func (p Proposal) reaches() string { + nodes := map[string]bool{} + for _, r := range p.Reaches { + if i := strings.LastIndex(r, " on "); i > 0 { + nodes[r[i+4:]] = true + } + } + switch len(nodes) { + case 0: + return "none today, as it moves no module's value" + case 1: + return sortedKeys(nodes)[0] + " (" + strings.Join(p.Reaches, ", ") + ")" + } + list := sortedKeys(nodes) + return fmt.Sprintf("%d nodes, %s", len(list), strings.Join(list, ", ")) +} + +// summary is the ask's first line: the level, the key and the value, and that it takes effect at once. +func (p Proposal) summary() string { + verb := "Sets " + p.Key + if p.Action == "clear" { + verb = "Clears " + p.Key + } + trusted := "" + if p.Trusted { + trusted = ", a trusted value," + } + reach := p.reaches() + s := fmt.Sprintf("%s%s at %s; approved, it takes effect at once on every node it reaches: %s.", verb, trusted, + p.Level, reach) + if len([]rune(s)) > asks.SummaryLength { + s = fmt.Sprintf("%s%s at %s; approved, it takes effect at once on every node it reaches (%d modules).", verb, + trusted, p.Level, len(p.Reaches)) + } + return s +} + +// point is a line cut to what a phone shows at a glance, saying so. +func point(line string) string { + const cut = "… (cut: read it in the note)" + if r := []rune(line); len(r) > asks.PointLength { + return string(r[:asks.PointLength-len([]rune(cut))]) + cut + } + return line +} + +// headline is the ask's headline, shortened when the names make it longer than the router allows (ADR 0277 §5). +func (p Proposal) headline() string { + verb := "Set" + if p.Action == "clear" { + verb = "Clear" + } + where := "for the whole mesh" + switch p.Level.Kind { + case LevelNode: + where = "on " + p.Level.Node + case LevelModule: + where = "for " + p.Level.Module + " on " + p.Level.Node + } + for _, h := range []string{fmt.Sprintf("%s %s %s?", verb, p.Key, where), fmt.Sprintf("%s a config value %s?", verb, + where), verb + " a config value?"} { + if len([]rune(h)) <= asks.HeadlineLength { + if s, ok := sayable(h); ok { + return s + } + } + } + return verb + " a config value?" +} + +// Ask is the proposal's ask, composed from it alone: its words, its two options each binding its act, who +// answers, its expiry and what it is about. +func (p Proposal) Ask() asks.Ask { + why, _ := sayable("Why: " + p.Why) + if why != "Why: "+p.Why { + why = "Why: given, and read in full with the proposal" + } + compose := func(exact bool) string { + return strings.Join([]string{p.summary(), point(p.change(exact)), point(why)}, "\n") + } + details := func(exact bool) string { + var d strings.Builder + d.WriteString(p.change(exact) + "\n") + from, _ := sayable(p.From) + fmt.Fprintf(&d, "Proposed by %s at %s UTC. Fingerprint %s.\n", from, p.At.UTC().Format("15:04 on 2 Jan"), + p.Fingerprint()) + if !exact && !p.whole() { + d.WriteString("Where who answers is not proven, its values are shown as ‹address›, ‹path› or ‹withheld›.\n") + } + fmt.Fprintf(&d, "Read it whole, with this fingerprint: mesh-config's config verb with action proposals and "+ + "proposal %s, through the mesh MCP server.", p.ID) + return d.String() + } + a := asks.Ask{ID: p.ID, Headline: p.headline(), Explanation: compose(false), Who: asks.Operator, + Expires: p.At.Add(ProposalLasts), OnExpiry: "the proposal is discarded; nothing changes", About: p.about(), + Details: details(false)} + if w := compose(true); w != a.Explanation { + a.Whole = w + } + if w := details(true); w != a.Details { + a.WholeDetails = w + } + does := "the value is set at once, and every node it reaches takes it" + if p.Action == "clear" { + does = "the value is cleared at once, and every node it reaches takes what is below it" + } + for _, o := range []struct{ id, label, does string }{ + {OptionApprove, "Approve", does}, + {OptionDecline, "Decline", "nothing changes; the proposal is discarded"}, + } { + binds, _ := asks.ActDigest(p.Act(o.id)) + a.Options = append(a.Options, asks.Option{ID: o.id, Label: o.label, Does: o.does, Level: asks.Approve, + Binds: binds}) + } + return a +} + +// problem says why a proposal as kept is not one this module made, or "": judged again before an Approve is +// acted on, so a record whose value or shown value changed after it was asked is refused (ADR 0277 §12). +func (p Proposal) problem() string { + switch { + case !asks.UsableID(p.ID): + return "its id is not one" + case p.Action != "set" && p.Action != "clear": + return "its action is neither set nor clear" + case !keyName.MatchString(p.Key): + return "its key is not one" + case p.At.IsZero(): + return "it says no time" + case p.Action == "set" && valueDigest(p.Value, true) != p.Digest: + return "its value is not the value its digest names: it changed after it was asked" + case p.Action == "clear" && (len(p.Value) != 0 || p.Digest != valueDigest(nil, false)): + return "a clear carries no value" + case valueDigest(p.Was, p.HadValue) != p.WasDigest: + return "the value it was shown against is not the one its digest names: it changed after it was asked" + } + return "" +} + +// ---- proposing ----------------------------------------------------------------------------------------- + +// ProposeAnswer is what a proposal answers. +type ProposeAnswer struct { + Said string `json:"said"` + Proposal string `json:"proposal"` + Asked string `json:"asked"` + Fingerprint string `json:"fingerprint"` + Expires time.Time `json:"expires"` + Router string `json:"router"` + Reaches []string `json:"reaches"` + Replaced []string `json:"replaced,omitempty"` +} + +// AtTheTerminalInstead names the other way, said whenever a proposal cannot be asked (ADR 0277 §19). +const AtTheTerminalInstead = "the operator may set it at the controller's terminal instead, through the controller's " + + "config line, which carries the value to mesh-config" + +// approverReady says why no channel that proves who answered can carry an approval, when this module can tell +// from the channel bench's kinds; an empty list says nothing, and the router is the judge (ADR 0259 §4). +func approverReady(bus AskBus) string { + kinds, err := bus.Kinds() + if err != nil || len(kinds) == 0 { + return "" + } + channels := 0 + for _, k := range kinds { + if k.Seat != asks.ChannelSeat { + continue + } + channels++ + if k.Promises("choice") && k.Promises("exact-render") && k.Promises("verified-sender") { + return "" + } + } + if channels == 0 { + return "" + } + return "no channel that proves who answered (one promising a verified sender, such as Telegram) is on the mesh" +} + +func newProposalID() string { + var b [8]byte + _, _ = rand.Read(b[:]) + return "cfg-" + hex.EncodeToString(b[:]) +} + +// endedWord is the router's word that an ask ended without a choice, heard while its proposer waits. +type endedWord struct { + Outcome asks.Outcome + Words string +} + +func (c *Config) heardEnded(id string) (endedWord, bool) { + c.endedMu.Lock() + defer c.endedMu.Unlock() + w, ok := c.ended[id] + return w, ok +} + +func (c *Config) tellEnded(id string, w endedWord) { + c.endedMu.Lock() + defer c.endedMu.Unlock() + if c.ended == nil { + c.ended = map[string]endedWord{} + } + c.ended[id] = w +} + +// Propose judges a value as a set would, keeps it, and asks the operator under mesh-config's own name. Nothing is +// set here: only the operator's Approve sets it (Decide). +func (c *Config) Propose(call Call) (*ProposeAnswer, error) { + refuse := func(format string, a ...any) (*ProposeAnswer, error) { + return nil, errors.New(fmt.Sprintf(format, a...) + ". Nothing was proposed") + } + if c.Asks == nil { + return refuse("the operator cannot be asked from here; %s", AtTheTerminalInstead) + } + if call.By != "" { + return refuse("by names who gave a value at the controller's terminal; a proposal is asked of the operator") + } + if call.Proposal != "" { + return refuse("proposal names a proposal to read, with action proposals") + } + action := "set" + if call.Clear { + action = "clear" + if call.Value != nil { + return refuse("a clear carries no value") + } + } + c.mu.Lock() + prop, ask, replaced, err := c.propose(call, action) + c.mu.Unlock() + if err != nil { + return nil, err + } + answer := &ProposeAnswer{Proposal: prop.ID, Asked: ask.Headline, Fingerprint: prop.Fingerprint(), + Expires: ask.Expires, Reaches: prop.Reaches, Replaced: replaced, + Router: "the router has not said yet whether it took the ask; config's proposals shows it"} + // The router's word, before answering (ADR 0277 §20): it refuses at once an ask no channel can carry. + within, every := c.RouterWithin, c.RouterEvery + if within <= 0 { + within = RouterAnswersWithin + } + if every <= 0 { + every = RouterAnswersEvery + } + for deadline := time.Now().Add(within); time.Now().Before(deadline); time.Sleep(every) { + if w, ended := c.heardEnded(prop.ID); ended { + return nil, fmt.Errorf("the router did not ask the operator: the ask %s (%s). The proposal is ended and "+ + "nothing is set; %s", w.Outcome, w.Words, AtTheTerminalInstead) + } + if e, err := c.Asks.Record(prop.ID); err == nil && e != nil { + answer.Router = "the router took the ask and shows it on the channels that can carry it" + break + } + } + answer.Said = fmt.Sprintf("proposed %s of %s at %s (fingerprint %s): the operator is asked %q on a channel that "+ + "proves who answered. Nothing changes until they approve; on Approve it takes effect at once, and on Decline "+ + "or at %s UTC it is discarded", action, prop.Key, prop.Level, prop.Fingerprint(), ask.Headline, + ask.Expires.UTC().Format("15:04 on 2 Jan")) + return answer, nil +} + +// propose is Propose's judgement, its record and its ask; the caller holds the lock. +func (c *Config) propose(call Call, action string) (Proposal, asks.Ask, []string, error) { + fail := func(format string, a ...any) (Proposal, asks.Ask, []string, error) { + return Proposal{}, asks.Ask{}, nil, errors.New(fmt.Sprintf(format, a...) + ". Nothing was proposed") + } + level, err := call.level() + if err != nil { + return fail("%v", err) + } + if !keyName.MatchString(call.Key) { + return fail("name the key to propose: lower-case letters, digits, dots, dashes and underscores") + } + why := strings.TrimSpace(call.Why) + if why == "" || strings.Contains(why, "\n") { + return fail("say why, in one line: every change keeps its why (novox/hq ADR 0348 §9)") + } + if call.Caller == "" || call.Caller == Unproven { + return fail("the call names no proven caller, and a proposal keeps who made it: call config through the mesh, " + + "which names you") + } + s, err := c.sources() + if err != nil { + return Proposal{}, asks.Ask{}, nil, err + } + // **Judged as a set judges it, before anyone is asked** (ADR 0277 §4): no ask for a value the verb would refuse. + r, trusted, err := c.declared(s, level, call.Key) + if err != nil { + return fail("%v", err) + } + vk := level.ValueKey(call.Key) + was, had := s.Values[vk] + now := c.now() + prop := Proposal{ID: newProposalID(), Action: action, Level: level, Key: call.Key, HadValue: had, + WasDigest: valueDigest(was.Value, had), Trusted: trusted, From: call.Caller, Why: why, At: now, + State: ProposalWaiting, Digest: valueDigest(nil, false)} + if had { + prop.Was = compact(was.Value) + } + after := Sources{Assignments: s.Assignments, Declared: s.Declared, Predefined: s.Predefined, + Values: map[string]Value{}} + for k, v := range s.Values { + after.Values[k] = v + } + switch action { + case "set": + if call.Value == nil { + return fail("give the value to propose for %s", call.Key) + } + raw, err := json.Marshal(call.Value) + if err != nil { + return fail("the value cannot be read: %v", err) + } + var value any + _ = json.Unmarshal(raw, &value) + if problems := r.check(value); len(problems) > 0 { + return fail("%s is refused for %s at %s: %s", show(value), call.Key, level, strings.Join(problems, "; ")) + } + prop.Value, prop.Digest = compact(raw), valueDigest(raw, true) + if had && string(compact(was.Value)) == string(prop.Value) { + return fail("%s at %s already holds exactly %s", call.Key, level, prop.Value) + } + after.Values[vk] = Value{Value: prop.Value} + case "clear": + if !had { + return fail("nothing is set for %s at %s, so there is nothing to clear", call.Key, level) + } + delete(after.Values, vk) + } + for _, m := range diff(Resolve(s), Resolve(after)) { + prop.Reaches = append(prop.Reaches, m.Module+" on "+m.Node) + } + if prop.Reaches == nil { + prop.Reaches = []string{} + } + // **The operator is shown exactly what they approve** where who answered is proven (ADR 0277 §6, §19): a value + // shaped like a secret could be shown nowhere, so it is not proposed; config holds no secret. + if refusal, ok := outward.CheckSecret(prop.change(true)); !ok { + return fail("the change carries %s, which leaves the mesh nowhere, so the operator could not be shown what "+ + "they approve; config holds no secret, and %s", refusal, AtTheTerminalInstead) + } + // **Its own cap of open asks, and the same change once** (ADR 0277 §21); a newer proposal for the same place + // replaces an older one still open (§10). + all, err := c.proposals() + if err != nil { + return Proposal{}, asks.Ask{}, nil, err + } + var replace []Proposal + open := 0 + for _, o := range all { + if o.State != ProposalWaiting || now.After(o.At.Add(ProposalLasts)) { + continue + } + if o.about() == prop.about() { + if o.Action == prop.Action && o.Digest == prop.Digest && o.WasDigest == prop.WasDigest { + return fail("the same change is proposed already, as %s, and waits for the operator's answer", o.ID) + } + replace = append(replace, o) + continue + } + open++ + } + if open >= asks.MostOpen { + return fail("%d proposals already wait for the operator's answer, and mesh-config asks at most %d at once; "+ + "config's proposals lists them", open, asks.MostOpen) + } + if why := approverReady(c.Asks); why != "" { + return fail("%s; %s", why, AtTheTerminalInstead) + } + ask := prop.Ask() + if err := ask.Check(now); err != nil { + return fail("%v", err) + } + if err := c.Store.Put(ProposalsState, prop.ID, prop); err != nil { + return fail("the proposal could not be kept, so the operator was not asked: %v", err) + } + var replaced []string + for _, o := range replace { + o.State, o.Outcome, o.Decided = ProposalEnded, "replaced by the newer proposal "+prop.ID+"; nothing was set", now + if err := c.Store.Put(ProposalsState, o.ID, o); err == nil { + replaced = append(replaced, o.ID) + } + } + if err := c.Asks.Publish(asks.AskSubject(Module), ask, ask.ID); err != nil { + prop.State, prop.Outcome = ProposalUnasked, "the ask could not be published: "+err.Error() + _ = c.Store.Put(ProposalsState, prop.ID, prop) + c.logf("[mesh-config] proposal %s could not be asked: %v", prop.ID, err) + return Proposal{}, asks.Ask{}, nil, fmt.Errorf("the operator could not be asked (%v); the proposal is kept "+ + "and will never be set; %s", err, AtTheTerminalInstead) + } + c.logf("[mesh-config] proposal %s (%s %s at %s, fingerprint %s) asked of the operator", prop.ID, action, prop.Key, + prop.Level, prop.Fingerprint()) + return prop, ask, replaced, nil +} + +// proposals is every proposal kept, oldest first. +func (c *Config) proposals() ([]Proposal, error) { + var out []Proposal + err := c.each(ProposalsState, func(key string, raw []byte) error { + var p Proposal + if json.Unmarshal(raw, &p) != nil || p.ID != key { + return nil // not one this module wrote: never asked, never acted on + } + out = append(out, p) + return nil + }) + if err != nil { + return nil, fmt.Errorf("the proposals cannot be read: %w", err) + } + sort.SliceStable(out, func(i, j int) bool { return out[i].At.Before(out[j].At) }) + return out, nil +} + +// ---- deciding -------------------------------------------------------------------------------------------- + +// routerRecord is the router's record of one ask, as far as this module reads it. +type routerRecord struct { + State string `json:"state"` + Ask *asks.Ask `json:"ask"` + Warrant *asks.Warrant `json:"warrant"` +} + +// Decide decides every waiting proposal whose ask the router has closed, oldest warrant first, each held to the +// warrant as described at the top of this file; and asks the controller to record every warrant acted on that its +// hand-act log does not hold yet. It answers a line for each decision. +func (c *Config) Decide() ([]string, error) { + if c.Asks == nil { + return nil, nil + } + c.mu.Lock() + defer c.mu.Unlock() + all, err := c.proposals() + if err != nil { + return nil, err + } + now := c.now() + type closed struct { + p Proposal + w asks.Warrant + } + var ready []closed + var said []string + for _, p := range all { + if p.State != ProposalWaiting { + continue + } + e, err := c.Asks.Record(p.ID) + if err == nil && e == nil && now.After(p.At.Add(ProposalLasts+time.Hour)) { + // Ended, and the router keeps no record of it: no answer can be proven, so nothing is set. + said = append(said, c.end(p, "the router keeps no record of its ask (refused before it was taken, or "+ + "forgotten); nothing was set")) + continue + } + if err != nil || e == nil { + continue // open, not yet recorded, or not readable now: read again on the next look + } + var rec routerRecord + if json.Unmarshal(e.Value, &rec) != nil || rec.State == "open" || rec.Warrant == nil { + continue + } + ready = append(ready, closed{p, *rec.Warrant}) + } + sort.SliceStable(ready, func(i, j int) bool { return ready[i].w.At.Before(ready[j].w.At) }) + for _, r := range ready { + said = append(said, c.decide(r.p, r.w)) + } + c.recordWarrants(now) + return said, nil +} + +// end records a proposal ended without the operator's choice. +func (c *Config) end(p Proposal, why string) string { + p.State, p.Outcome, p.Decided = ProposalEnded, why, c.now() + if err := c.Store.Put(ProposalsState, p.ID, p); err != nil { + return fmt.Sprintf("proposal %s: %s, and that could not be kept (%v)", p.ID, why, err) + } + return "proposal " + p.ID + ": " + why +} + +// decide holds one closed ask's warrant to its proposal and, on an Approve that holds, sets the value. The caller +// holds the lock. +func (c *Config) decide(p Proposal, w asks.Warrant) string { + now := c.now() + keep := func(state, outcome string) string { + p.State, p.Outcome, p.Decided = state, outcome, now + if err := c.Store.Put(ProposalsState, p.ID, p); err != nil { + return fmt.Sprintf("proposal %s: %s, and that could not be kept (%v)", p.ID, outcome, err) + } + return "proposal " + p.ID + ": " + outcome + } + refused := func(why string) string { + c.logf("[mesh-config] REFUSED the answer to proposal %s: %s", p.ID, why) + return keep(ProposalRefused, "the answer is refused and nothing is set: "+why) + } + if w.Asker != Module || w.Ask != p.ID { + return refused(fmt.Sprintf("the router's record is for %s's ask %s", w.Asker, w.Ask)) + } + if w.Outcome != asks.OutcomeChosen { + why := "the ask " + string(w.Outcome) + if w.Words != "" { + why += " (" + w.Words + ")" + } + return keep(ProposalEnded, why+"; nothing was set") + } + // **The record is held to what was asked** (ADR 0277 §11, §12): its value and the value it was shown against + // still digest to what the ask bound, the warrant answers the ask composed again from it, and the act about to be + // performed is the one the chosen option bound. + if problem := p.problem(); problem != "" { + return refused("the proposal is not the one asked: " + problem) + } + option, err := w.For(Module, p.Ask()) + if err == nil { + err = option.Performs(p.Act(option.ID)) + } + if err != nil { + return refused(err.Error()) + } + switch option.ID { + case OptionDecline: + p.Answered = w.Says() + return keep(ProposalDeclined, "declined by the operator ("+w.Says()+"); nothing was set") + case OptionApprove: + p.Answered, p.Approved = w.Says(), approvedWords(p, w) + default: + return refused("the option " + option.ID + " is not one a proposal offers") + } + // Claimed before it is acted on: a warrant heard twice, or read again, sets nothing twice. + p.State, p.Outcome, p.Decided = ProposalApproved, "approved by the operator; being set", now + if err := c.Store.Put(ProposalsState, p.ID, p); err != nil { + return fmt.Sprintf("proposal %s: approved by the operator, and not set: the decision could not be kept (%v); "+ + "it is read again on the next look", p.ID, err) + } + call := Call{Action: p.Action, Key: p.Key, Module: p.Level.Module, Node: p.Level.Node, Why: p.Why, Caller: p.From, + approval: &approval{Proposal: p.ID, By: "the operator", Says: p.Approved, Was: p.Was, HadValue: p.HadValue, + WasDigest: p.WasDigest}} + if p.Action == "set" { + var v any + _ = json.Unmarshal(p.Value, &v) + call.Value = v + if v == nil { + call.Value = json.RawMessage("null") + } + } + a, err := c.change(call) + if err != nil { + return keep(ProposalRefused, "approved by the operator, and refused on Approve, so nothing is set: "+err.Error()) + } + return keep(ProposalApproved, "approved by the operator and set: "+a.Said) +} + +// approvedWords is who approved it, as kept beside the value (ADR 0277 §15). +func approvedWords(p Proposal, w asks.Warrant) string { + who := "the operator" + if w.By != nil { + who = "the " + w.By.Who + if w.By.Identity != "" { + who += ", as " + w.By.Kind + " identity " + w.By.Identity + } + via := w.By.Kind + if w.By.Verified != "" { + via += " (" + w.By.Verified + ")" + } + who += ", via " + via + } + return fmt.Sprintf("approved by %s at %s, ask %s, proposed by %s", who, w.At.UTC().Format(time.RFC3339), p.ID, p.From) +} + +// recordWarrants asks the controller to record each warrant acted on (Approve or Decline) that its hand-act log +// does not hold yet (ADR 0277 §17): the controller reads it from the router's own record. The caller holds the lock. +func (c *Config) recordWarrants(now time.Time) { + all, err := c.proposals() + if err != nil { + return + } + for _, p := range all { + if p.Recorded || p.Answered == "" || now.Sub(p.Decided) > 7*24*time.Hour { + continue + } + if err := c.Asks.Warranted(p.ID); err != nil { + p.Record = "not recorded in the hand-act log yet: " + err.Error() + } else { + p.Recorded, p.Record = true, "" + } + _ = c.Store.Put(ProposalsState, p.ID, p) + } +} + +// HeardEnd takes the router's word on one of this module's asks. An ask ended without a choice ends its proposal +// at once, and is told to a proposer waiting on it; nothing is set on such a word, so it needs no proof beyond its +// subject. A choice is only ever taken from the router's record (Decide). +func (c *Config) HeardEnd(w asks.Warrant) []string { + if w.Asker != Module || !asks.UsableID(w.Ask) { + return nil + } + if w.Outcome == asks.OutcomeChosen { + said, err := c.Decide() + if err != nil { + said = append(said, "reading the operator's answers: "+err.Error()) + } + return said + } + c.mu.Lock() + defer c.mu.Unlock() + defer c.tellEnded(w.Ask, endedWord{Outcome: w.Outcome, Words: w.Words}) + e, err := c.Store.Get(ProposalsState, w.Ask) + if err != nil || e == nil { + return nil + } + var p Proposal + if json.Unmarshal(e.Value, &p) != nil || p.ID != w.Ask || p.State != ProposalWaiting { + return nil + } + why := "the ask " + string(w.Outcome) + if w.Words != "" { + why += " (" + w.Words + ")" + } + return []string{c.end(p, why+"; nothing was set")} +} + +// ---- the list ---------------------------------------------------------------------------------------------- + +// ProposalRow is one proposal as a listing says it. +type ProposalRow struct { + ID string `json:"id"` + Action string `json:"action"` + Level Level `json:"level"` + Key string `json:"key"` + State string `json:"state"` + Outcome string `json:"outcome,omitempty"` + From string `json:"from"` + At time.Time `json:"at"` + Expires time.Time `json:"expires"` + Fingerprint string `json:"fingerprint"` + Approved string `json:"approved,omitempty"` + // What one proposal read whole says besides. + Value json.RawMessage `json:"value,omitempty"` + Was json.RawMessage `json:"was,omitempty"` + Why string `json:"why,omitempty"` + Reaches []string `json:"reaches,omitempty"` + Router string `json:"router,omitempty"` + Phone string `json:"the phone,omitempty"` + Warning string `json:"warning,omitempty"` +} + +// Proposals is every proposal, newest first; with a proposal's id, that one whole, with whether the router's ask +// is the one composed from it. +func (c *Config) Proposals(call Call) ([]ProposalRow, error) { + all, err := c.proposals() + if err != nil { + return nil, err + } + out := []ProposalRow{} + for i := len(all) - 1; i >= 0; i-- { + p := all[i] + if call.Proposal != "" && p.ID != call.Proposal { + continue + } + if (call.Key != "" && p.Key != call.Key) || (call.Node != "" && p.Level.Node != call.Node) || + (call.Module != "" && p.Level.Module != call.Module) { + continue + } + row := ProposalRow{ID: p.ID, Action: p.Action, Level: p.Level, Key: p.Key, State: p.State, Outcome: p.Outcome, + From: p.From, At: p.At, Expires: p.At.Add(ProposalLasts), Fingerprint: p.Fingerprint(), Approved: p.Approved} + if p.State == ProposalWaiting && c.now().After(row.Expires) { + row.State = "expired, not yet heard" + } + if call.Proposal != "" { + row.Value, row.Was, row.Why, row.Reaches = p.Value, p.Was, p.Why, p.Reaches + if problem := p.problem(); problem != "" { + row.Warning = "this proposal is never set: " + problem + } + if c.Asks != nil { + if e, err := c.Asks.Record(p.ID); err == nil && e != nil { + var rec routerRecord + if json.Unmarshal(e.Value, &rec) == nil { + row.Router = rec.State + switch { + case rec.Ask == nil: + row.Phone = "cannot be compared: the router's record keeps no ask" + case rec.Ask.Digest() == p.Ask().Digest(): + row.Phone = "shows the ask composed from exactly this proposal (fingerprint " + p.Fingerprint() + ")" + default: + row.Phone = "shows ANOTHER ask than this proposal composes: do not approve it" + } + } + } + } + } + out = append(out, row) + } + if call.Proposal != "" && len(out) == 0 { + return nil, errors.New("no proposal " + strconv.Quote(call.Proposal) + " is kept") + } + return out, nil +} diff --git a/modules/mesh-config/cmd/mesh-config/proposals_test.go b/modules/mesh-config/cmd/mesh-config/proposals_test.go new file mode 100644 index 00000000..7ebe7825 --- /dev/null +++ b/modules/mesh-config/cmd/mesh-config/proposals_test.go @@ -0,0 +1,568 @@ +package main + +import ( + "encoding/json" + "errors" + "strings" + "sync" + "testing" + "time" + + stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/asks" +) + +// standIn is the operator channel's router and a channel that proves who answered, in memory: it takes the asks +// mesh-config publishes, keeps its own record of each under mesh-config's name, and records the operator's choice +// as the router's warrant. Nothing of mesh-config's is trusted by it: it digests the ask it took. +type standIn struct { + mu sync.Mutex + subjects []string + asks map[string]asks.Ask + records map[string][]byte + kinds []stdio.KindHeld + refuse error + warranted []string + // refuseWith makes the router refuse every ask it is published, as the router does for an ask no channel can + // carry: no record, and its word on the decided subject (heard through hear). + refuseWith string + hear func(asks.Warrant) + // noRecord keeps the router from recording an ask it took (an ask it has not taken yet). + noRecord bool +} + +func newStandIn() *standIn { + return &standIn{asks: map[string]asks.Ask{}, records: map[string][]byte{}, kinds: []stdio.KindHeld{ + {Seat: asks.ChannelSeat, Kind: "telegram", Module: "telegram", Node: "novox", + Capabilities: []string{"choice", "exact-render", "verified-sender"}}}} +} + +func (r *standIn) Publish(subject string, body any, id string) error { + r.mu.Lock() + defer r.mu.Unlock() + if r.refuse != nil { + return r.refuse + } + a, ok := body.(asks.Ask) + if !ok || a.ID != id { + return errors.New("not an ask") + } + r.subjects = append(r.subjects, subject) + r.asks[id] = a + if r.refuseWith != "" { + w := asks.Warrant{Ask: id, Asker: Module, About: a.About, Outcome: asks.OutcomeRefused, Words: r.refuseWith, + At: time.Now()} + go r.hear(w) + return nil + } + if !r.noRecord { + raw, _ := json.Marshal(map[string]any{"state": "open", "ask": a}) + r.records[id] = raw + } + return nil +} + +func (r *standIn) Record(id string) (*stdio.StateEntry, error) { + r.mu.Lock() + defer r.mu.Unlock() + raw, ok := r.records[id] + if !ok { + return nil, nil + } + return &stdio.StateEntry{Key: Module + "." + id, Value: raw}, nil +} + +func (r *standIn) Kinds() ([]stdio.KindHeld, error) { return r.kinds, nil } + +func (r *standIn) Warranted(ask string) error { + r.mu.Lock() + defer r.mu.Unlock() + r.warranted = append(r.warranted, ask) + return nil +} + +// choose is the operator choosing an option on a channel that proves who answered: the router's record of the ask +// it took, closed with its warrant. +func (r *standIn) choose(t *testing.T, id, option string, tweak func(*asks.Warrant)) asks.Warrant { + t.Helper() + r.mu.Lock() + defer r.mu.Unlock() + a, ok := r.asks[id] + if !ok { + t.Fatalf("the router took no ask %s", id) + } + o, ok := a.Option(option) + if !ok { + t.Fatalf("the ask %s offers no %s", id, option) + } + w := asks.Warrant{Ask: id, Asker: Module, About: a.About, Outcome: asks.OutcomeChosen, Option: o.ID, Label: o.Label, + Level: o.Level, By: &asks.Person{Who: asks.Operator, Kind: "telegram", Identity: "4242", Verified: "user id verified"}, + Channel: "telegram", Proofs: []string{"P1"}, At: time.Date(2026, 10, 14, 19, 30, 0, 0, time.UTC), + AskDigest: a.Digest()} + if tweak != nil { + tweak(&w) + } + raw, _ := json.Marshal(map[string]any{"state": "closed", "ask": a, "warrant": w}) + r.records[id] = raw + return w +} + +// aProposingMesh is aMesh with the stand-in channel, its waits short. +func aProposingMesh(t *testing.T) (*Config, *memStore, *standIn) { + t.Helper() + c, m := aMesh(t) + r := newStandIn() + r.hear = func(w asks.Warrant) { c.HeardEnd(w) } + c.Asks, c.RouterWithin, c.RouterEvery = r, 300*time.Millisecond, 5*time.Millisecond + return c, m, r +} + +func propose(t *testing.T, c *Config, call Call) *ProposeAnswer { + t.Helper() + call.Action = "propose" + if call.Caller == "" { + call.Caller = "g14.node-tools" + } + if call.Why == "" { + call.Why = "the agents need the new rule" + } + a, err := c.Propose(call) + if err != nil { + t.Fatalf("propose %s: %v", call.Key, err) + } + return a +} + +func storedProposal(t *testing.T, m *memStore, id string) Proposal { + t.Helper() + e, _ := m.Get(ProposalsState, id) + if e == nil { + t.Fatalf("no proposal %s is kept", id) + } + var p Proposal + if err := json.Unmarshal(e.Value, &p); err != nil { + t.Fatal(err) + } + return p +} + +func storedValue(t *testing.T, m *memStore, key string) (Value, bool) { + t.Helper() + e, _ := m.Get(ValuesState, key) + if e == nil { + return Value{}, false + } + var v Value + if err := json.Unmarshal(e.Value, &v); err != nil { + t.Fatal(err) + } + return v, true +} + +// **A trusted value proposed end to end** (ADR 0348 §8, How it is checked: "A proposal's test"): mesh-config asks +// under its own name; its ask names the level, the key and the value and says the change takes effect at once; +// each option binds them; nothing is set until the operator's Approve; on Approve the value is set as a set is, +// who approved it is kept beside it, and the warrant is recorded in the hand-act log. +func TestATrustedValueIsProposedAskedUnderItsOwnNameAndSetOnTheOperatorsApprove(t *testing.T) { + c, m, r := aProposingMesh(t) + m.events, m.handActs = nil, nil + value := map[string]any{"permissions": map[string]any{"deny": []any{"Bash(ssh:*)"}}} + a := propose(t, c, Call{Key: "agent-managed-settings", Value: value}) + + // Nothing is set by a proposal. + if _, set := storedValue(t, m, "mesh.agent-managed-settings"); set || len(m.events) != 0 || len(m.handActs) != 0 { + t.Fatalf("a proposal set something: %d events, %d hand-acts", len(m.events), len(m.handActs)) + } + if !strings.Contains(a.Router, "took the ask") || a.Proposal == "" || len(a.Reaches) != 1 || + a.Reaches[0] != "claude-code on g14" { + t.Errorf("the proposal answered %+v", a) + } + // Asked under mesh-config's own name, on the operator channel. + if len(r.subjects) != 1 || r.subjects[0] != asks.AskSubject("mesh-config") { + t.Fatalf("asked on %v", r.subjects) + } + ask := r.asks[a.Proposal] + if err := ask.Check(c.now()); err != nil { + t.Fatalf("the ask: %v", err) + } + if err := asks.Glance(ask.Explanation); err != nil { + t.Errorf("the ask does not read at a glance: %v\n%s", err, ask.Explanation) + } + for _, want := range []string{"agent-managed-settings", "mesh config", "takes effect at once", "claude-code", + "a trusted value"} { + if !strings.Contains(ask.Explanation, want) { + t.Errorf("the ask's explanation does not say %q:\n%s", want, ask.Explanation) + } + } + if !strings.Contains(ask.Headline, "agent-managed-settings") || !strings.Contains(ask.Headline, "whole mesh") { + t.Errorf("the headline is %q", ask.Headline) + } + // The value exact where who answers is proven; under the content rule elsewhere. + exact := `+ agent-managed-settings: {"permissions":{"deny":["Bash(ssh:*)"]}}` + shown := ask.Explanation + if ask.Whole != "" { + shown = ask.Whole + } + if !strings.Contains(shown, exact) || !strings.Contains(ask.Details+ask.WholeDetails, a.Proposal) { + t.Errorf("the ask does not show the value whole:\n%s\n%s", shown, ask.WholeDetails) + } + // Each option binds the level, the key and the value. + p := storedProposal(t, m, a.Proposal) + for _, o := range ask.Options { + if o.Level != asks.Approve { + t.Errorf("the option %s is at %s", o.ID, o.Level) + } + act := p.Act(o.ID) + want, _ := asks.ActDigest(act) + if o.Binds != want || act["level"] != LevelMesh || act["key"] != "agent-managed-settings" || + act["value"] != valueDigest(json.RawMessage(`{"permissions":{"deny":["Bash(ssh:*)"]}}`), true) || + act["was"] != valueDigest(nil, false) { + t.Errorf("the option %s binds %s for %v", o.ID, o.Binds, act) + } + } + // Still nothing while the ask is open. + if said, _ := c.Decide(); len(said) != 0 { + t.Errorf("an open ask decided %v", said) + } + + r.choose(t, a.Proposal, OptionApprove, nil) + said, err := c.Decide() + if err != nil || len(said) != 1 || !strings.Contains(said[0], "approved by the operator and set") { + t.Fatalf("the Approve decided %v (%v)", said, err) + } + v, set := storedValue(t, m, "mesh.agent-managed-settings") + if !set || string(v.Value) != `{"permissions":{"deny":["Bash(ssh:*)"]}}` || v.By != "the operator" || + v.Via != "g14.node-tools" || !strings.Contains(v.Approved, "approved by the operator, as telegram identity 4242, "+ + "via telegram (user id verified)") || !strings.Contains(v.Approved, "ask "+a.Proposal) || + !strings.Contains(v.Approved, "proposed by g14.node-tools") { + t.Fatalf("the value is %+v", v) + } + if res := m.resolved(t, "g14", "claude-code", "managed"); res.Source != SourceMesh || + string(res.Value) != string(v.Value) { + t.Errorf("claude-code's managed settings resolve as %+v", res) + } + if len(m.events) != 1 || m.events[0].Approved != v.Approved || len(m.handActs) != 1 || + !strings.Contains(m.handActs[0], "the operator's Approve of proposal "+a.Proposal) { + t.Errorf("the set raised %+v and recorded %q", m.events, m.handActs) + } + listed, _ := c.List(Call{Key: "agent-managed-settings"}) + if len(listed.Set) != 1 || listed.Set[0].Approved != v.Approved { + t.Errorf("the listing says %+v", listed.Set) + } + if p = storedProposal(t, m, a.Proposal); p.State != ProposalApproved || !p.Recorded || + len(r.warranted) != 1 || r.warranted[0] != a.Proposal { + t.Errorf("the proposal is %+v; warrants recorded %v", p, r.warranted) + } + // A warrant heard again, or read again, sets nothing twice. + if said, _ = c.Decide(); len(said) != 0 || len(m.events) != 1 { + t.Errorf("the warrant read again decided %v", said) + } +} + +// **An Approve against a changed value is refused** (ADR 0348 §8, ADR 0277 §13; design 57 K4's proof): the value +// stored at the level changed after the operator was shown the change against it, so nothing is set. +func TestAnApproveIsRefusedWhenTheStoredValueChangedSince(t *testing.T) { + c, m, r := aProposingMesh(t) + a := propose(t, c, Call{Key: "agent-managed-settings", Value: map[string]any{"a": 1.0}}) + // Set at the terminal meanwhile. + set(t, c, Call{Key: "agent-managed-settings", Value: map[string]any{"b": 2.0}, Caller: ControllerCaller, By: "jochen"}) + m.events = nil + r.choose(t, a.Proposal, OptionApprove, nil) + said, _ := c.Decide() + if len(said) != 1 || !strings.Contains(said[0], "it changed since, so nothing is set") { + t.Fatalf("the Approve decided %v", said) + } + if v, _ := storedValue(t, m, "mesh.agent-managed-settings"); string(v.Value) != `{"b":2}` || v.Approved != "" { + t.Errorf("the value is %+v", v) + } + if len(m.events) != 0 { + t.Errorf("a refused Approve raised %+v", m.events) + } + if p := storedProposal(t, m, a.Proposal); p.State != ProposalRefused { + t.Errorf("the proposal is %s: %s", p.State, p.Outcome) + } + + // And a clear since: the value it was shown against (a value) is no longer stored. + set(t, c, Call{Key: "font-size", Value: 12.0, Node: "g14"}) + a = propose(t, c, Call{Key: "font-size", Value: 14.0, Node: "g14"}) + set(t, c, Call{Action: "clear", Key: "font-size", Node: "g14"}) + r.choose(t, a.Proposal, OptionApprove, nil) + if said, _ = c.Decide(); len(said) != 1 || !strings.Contains(said[0], "no value set") { + t.Fatalf("the Approve after a clear decided %v", said) + } + if _, isSet := storedValue(t, m, "node.g14.font-size"); isSet { + t.Error("an Approve against a cleared value set it") + } +} + +// **A record that changed after it was asked is refused on Approve** (ADR 0277 §12): its value, or the value it was +// shown against, no longer what the ask bound. +func TestAnApproveIsRefusedWhenTheProposalChangedAfterItWasAsked(t *testing.T) { + for _, tc := range []struct { + name string + tamper func(p *Proposal) + want string + }{ + {"its value", func(p *Proposal) { p.Value = json.RawMessage(`{"evil":true}`) }, "its value is not the value"}, + {"its value and digest", func(p *Proposal) { + p.Value = json.RawMessage(`{"evil":true}`) + p.Digest = valueDigest(p.Value, true) + }, "it was not the ask the person was shown"}, + {"the value it was shown against", func(p *Proposal) { + p.Was, p.HadValue = json.RawMessage(`{"x":1}`), true + }, "the value it was shown against"}, + {"its level", func(p *Proposal) { p.Level = Level{Kind: LevelNode, Node: "g14"} }, "not the ask the person was shown"}, + } { + t.Run(tc.name, func(t *testing.T) { + c, m, r := aProposingMesh(t) + a := propose(t, c, Call{Key: "agent-managed-settings", Value: map[string]any{"a": 1.0}}) + r.choose(t, a.Proposal, OptionApprove, nil) + p := storedProposal(t, m, a.Proposal) + tc.tamper(&p) + _ = m.Put(ProposalsState, p.ID, p) + said, _ := c.Decide() + if len(said) != 1 || !strings.Contains(said[0], tc.want) { + t.Fatalf("decided %v, want %q", said, tc.want) + } + if keys, _ := m.Keys(ValuesState); len(keys) != 0 { + t.Errorf("a changed proposal set %v", keys) + } + }) + } +} + +// **A warrant that is not the answer to the ask is refused** (ADR 0277 §11): another ask's digest, another level, an +// answer after expiry, nobody verified. +func TestAWarrantThatDoesNotAnswerTheAskIsRefused(t *testing.T) { + for _, tc := range []struct { + name string + tweak func(w *asks.Warrant) + }{ + {"another ask's digest", func(w *asks.Warrant) { w.AskDigest = "sha256:00" }}, + {"another level", func(w *asks.Warrant) { w.Level = asks.Acknowledge }}, + {"after its expiry", func(w *asks.Warrant) { w.At = time.Date(2026, 10, 16, 0, 0, 0, 0, time.UTC) }}, + {"an option it did not offer", func(w *asks.Warrant) { w.Option = "force" }}, + {"no person", func(w *asks.Warrant) { w.By = nil }}, + } { + t.Run(tc.name, func(t *testing.T) { + c, m, r := aProposingMesh(t) + a := propose(t, c, Call{Key: "agent-managed-settings", Value: map[string]any{"a": 1.0}}) + r.choose(t, a.Proposal, OptionApprove, tc.tweak) + said, _ := c.Decide() + if len(said) != 1 || !strings.Contains(said[0], "refused") { + t.Fatalf("decided %v", said) + } + if keys, _ := m.Keys(ValuesState); len(keys) != 0 { + t.Errorf("a warrant that does not hold set %v", keys) + } + if len(r.warranted) != 0 { + t.Errorf("a refused warrant was recorded as acted on: %v", r.warranted) + } + }) + } +} + +// **On Decline, at expiry, or on the router's refusal nothing is set** (ADR 0277 §9, §16, §20), and a Decline is a +// warrant the hand-act log holds. +func TestDeclineExpiryAndTheRoutersRefusalSetNothing(t *testing.T) { + c, m, r := aProposingMesh(t) + a := propose(t, c, Call{Key: "agent-managed-settings", Value: map[string]any{"a": 1.0}}) + r.choose(t, a.Proposal, OptionDecline, nil) + if said, _ := c.Decide(); len(said) != 1 || !strings.Contains(said[0], "declined by the operator") { + t.Fatalf("the Decline decided %v", said) + } + if p := storedProposal(t, m, a.Proposal); p.State != ProposalDeclined || p.Approved != "" || !p.Recorded { + t.Errorf("the declined proposal is %+v", p) + } + + // Ended without a choice: the router's word. + a = propose(t, c, Call{Key: "font-size", Value: 12.0, Node: "g14"}) + c.HeardEnd(asks.Warrant{Ask: a.Proposal, Asker: Module, Outcome: asks.OutcomeExpired, At: time.Now()}) + if p := storedProposal(t, m, a.Proposal); p.State != ProposalEnded || !strings.Contains(p.Outcome, "expired") { + t.Errorf("the expired proposal is %+v", p) + } + + // Ended, and the router never recorded it: discarded once its time is past. + r.noRecord = true + a = propose(t, c, Call{Key: "font-size", Value: 13.0, Node: "novox"}) + if !strings.Contains(a.Router, "has not said yet") { + t.Errorf("an ask the router has not taken answered %q", a.Router) + } + later := time.Date(2026, 10, 16, 0, 0, 0, 0, time.UTC) + c.Now = func() time.Time { return later } + if said, _ := c.Decide(); len(said) != 1 || !strings.Contains(said[0], "keeps no record") { + t.Errorf("an ask never recorded decided %v", said) + } + if keys, _ := m.Keys(ValuesState); len(keys) != 0 { + t.Errorf("an unanswered proposal set %v", keys) + } + + // Refused by the router at once: said to the proposer. + c2, m2, r2 := aProposingMesh(t) + r2.refuseWith = "no channel can carry an approval" + _, err := c2.Propose(Call{Action: "propose", Key: "font-size", Value: 12.0, Caller: "g14.node-tools", Why: "w"}) + if err == nil || !strings.Contains(err.Error(), "no channel can carry an approval") { + t.Fatalf("the router's refusal answered %v", err) + } + all, _ := c2.proposals() + if len(all) != 1 || all[0].State != ProposalEnded { + t.Errorf("the refused proposal is %+v", all) + } + if keys, _ := m2.Keys(ValuesState); len(keys) != 0 { + t.Errorf("a refused proposal set %v", keys) + } +} + +// **A proposal is judged as a set before anyone is asked, and refused at once when it cannot be asked** (ADR 0277 §4, +// §19, §21): a value the verb would refuse; nothing that would change; a value shaped like a secret; no channel that +// proves who answered; a publish that fails; the same change twice; a fourth open ask. +func TestAProposalIsJudgedAndRefusedBeforeTheOperatorIsAsked(t *testing.T) { + c, m, r := aProposingMesh(t) + set(t, c, Call{Key: "font-size", Value: 12.0}) + for _, tc := range []struct { + name string + call Call + want string + }{ + {"a key not declared", Call{Key: "colour", Value: "red", Node: "g14", Module: "dunst"}, "it declares:"}, + {"a value failing validation", Call{Key: "font-size", Value: 99.0}, "above the maximum 72"}, + {"a module value without a node", Call{Key: "width", Value: 400.0, Module: "dunst"}, "no module value for the whole mesh"}, + {"nothing that would change", Call{Key: "font-size", Value: 12.0}, "already holds exactly 12"}, + {"a clear of nothing", Call{Key: "font-size", Clear: true, Node: "g14"}, "nothing to clear"}, + {"a clear with a value", Call{Key: "font-size", Clear: true, Value: 1.0}, "a clear carries no value"}, + {"a secret's shape", Call{Key: "agent-managed-settings", Value: map[string]any{"env": map[string]any{ + "API_TOKEN": "ghp_abcdefghijklmnopqrstuvwxyz0123456789"}}}, "leaves the mesh nowhere"}, + {"by", Call{Key: "font-size", Value: 13.0, By: "jochen"}, "by names who gave"}, + {"no why", Call{Key: "font-size", Value: 13.0, Why: " "}, "say why"}, + {"an unproven caller", Call{Key: "font-size", Value: 13.0, Caller: Unproven}, "no proven caller"}, + } { + t.Run(tc.name, func(t *testing.T) { + call := tc.call + call.Action = "propose" + if call.Caller == "" { + call.Caller = "g14.node-tools" + } + if call.Why == "" { + call.Why = "a test" + } + if _, err := c.Propose(call); err == nil || !strings.Contains(err.Error(), tc.want) || + !strings.Contains(err.Error(), "Nothing was proposed") { + t.Fatalf("refused with %v, want %q", err, tc.want) + } + }) + } + if len(r.subjects) != 0 { + t.Fatalf("a refused proposal was asked: %v", r.subjects) + } + + r.kinds = []stdio.KindHeld{{Seat: asks.ChannelSeat, Kind: "desktop", Capabilities: []string{"choice"}}} + if _, err := c.Propose(Call{Action: "propose", Key: "font-size", Value: 13.0, Caller: "g", Why: "w"}); err == nil || + !strings.Contains(err.Error(), "no channel that proves who answered") { + t.Errorf("no approving channel: %v", err) + } + r.kinds = newStandIn().kinds + r.refuse = errors.New("the bus is away") + if _, err := c.Propose(Call{Action: "propose", Key: "font-size", Value: 13.0, Caller: "g", Why: "w"}); err == nil || + !strings.Contains(err.Error(), "could not be asked") { + t.Errorf("a failed publish: %v", err) + } + r.refuse = nil + + first := propose(t, c, Call{Key: "font-size", Value: 14.0, Node: "g14"}) + if _, err := c.Propose(Call{Action: "propose", Key: "font-size", Value: 14.0, Node: "g14", Caller: "g", Why: "w"}); err == nil || + !strings.Contains(err.Error(), "proposed already, as "+first.Proposal) { + t.Errorf("the same change twice: %v", err) + } + // A newer proposal for the same place replaces the older (ADR 0277 §10), and is not counted twice. + newer := propose(t, c, Call{Key: "font-size", Value: 15.0, Node: "g14"}) + if len(newer.Replaced) != 1 || newer.Replaced[0] != first.Proposal || + storedProposal(t, m, first.Proposal).State != ProposalEnded { + t.Errorf("the newer proposal replaced %v", newer.Replaced) + } + propose(t, c, Call{Key: "font-size", Value: 15.0, Node: "novox"}) + propose(t, c, Call{Key: "agent-managed-settings", Value: map[string]any{"a": 1.0}}) + if _, err := c.Propose(Call{Action: "propose", Key: "width", Value: 400.0, Node: "g14", Module: "dunst", + Caller: "g", Why: "w"}); err == nil || !strings.Contains(err.Error(), "3 proposals already wait") { + t.Errorf("a fourth open ask: %v", err) + } + if keys, _ := m.Keys(ValuesState); len(keys) != 1 { + t.Errorf("proposals set %v", keys) + } +} + +// **A clear is proposed and approved the same way**, and a trusted value set or cleared through the verb is refused, +// naming the proposal as the way (ADR 0277 §2). +func TestAClearIsProposedAndATrustedSetNamesTheProposal(t *testing.T) { + c, m, r := aProposingMesh(t) + _, err := c.Change(Call{Action: "set", Key: "agent-managed-settings", Value: map[string]any{}, Caller: "g14.node-tools", + Why: "w"}) + if err == nil || !strings.Contains(err.Error(), "propose it instead") { + t.Errorf("a trusted set through the verb: %v", err) + } + set(t, c, Call{Key: "managed", Value: map[string]any{"a": 1.0}, Node: "g14", Module: "claude-code", + Caller: ControllerCaller, By: "jochen"}) + a := propose(t, c, Call{Key: "managed", Clear: true, Node: "g14", Module: "claude-code"}) + ask := r.asks[a.Proposal] + if !strings.HasPrefix(ask.Headline, "Clear managed for claude-code on g14") || + !strings.Contains(ask.Explanation, `- managed (was: {"a":1})`) { + t.Errorf("the clear asks %q:\n%s", ask.Headline, ask.Explanation) + } + r.choose(t, a.Proposal, OptionApprove, nil) + if said, _ := c.Decide(); len(said) != 1 || !strings.Contains(said[0], "is cleared") { + t.Fatalf("the clear decided %v", said) + } + if _, isSet := storedValue(t, m, "module.g14.claude-code.managed"); isSet { + t.Error("an approved clear left the value") + } + h, _ := c.History(Call{Key: "managed"}) + if last := h[len(h)-1]; last.Action != "clear" || !strings.Contains(last.Approved, "approved by the operator") { + t.Errorf("the history keeps %+v", last) + } +} + +// **The proposals are listed, and one is read whole** with whether the router's ask is the one composed from it. +func TestTheProposalsAreListedAndOneIsReadWhole(t *testing.T) { + c, m, _ := aProposingMesh(t) + a := propose(t, c, Call{Key: "agent-managed-settings", Value: map[string]any{"a": 1.0}}) + b := propose(t, c, Call{Key: "font-size", Value: 12.0, Node: "g14"}) + rows, err := c.Do(Call{Action: "proposals"}) + if err != nil { + t.Fatal(err) + } + if l := rows.([]ProposalRow); len(l) != 2 || l[0].ID != b.Proposal || l[1].State != ProposalWaiting || l[0].Value != nil { + t.Errorf("listed %+v", l) + } + one, err := c.Proposals(Call{Proposal: a.Proposal}) + if err != nil || len(one) != 1 || string(one[0].Value) != `{"a":1}` || one[0].Fingerprint != a.Fingerprint || + !strings.Contains(one[0].Phone, "exactly this proposal") { + t.Fatalf("read whole %+v (%v)", one, err) + } + p := storedProposal(t, m, a.Proposal) + p.Value, p.Digest = json.RawMessage(`{"a":2}`), valueDigest(json.RawMessage(`{"a":2}`), true) + _ = m.Put(ProposalsState, p.ID, p) + if one, _ = c.Proposals(Call{Proposal: a.Proposal}); !strings.Contains(one[0].Phone, "ANOTHER ask") { + t.Errorf("a changed proposal reads %q", one[0].Phone) + } + if _, err := c.Proposals(Call{Proposal: "cfg-none"}); err == nil { + t.Error("an unknown proposal was read") + } +} + +// **The verb's new arguments are read strictly.** +func TestTheProposalArgumentsAreReadStrictly(t *testing.T) { + call, err := callOf(map[string]any{"action": "propose", "key": "k", "clear": true, "proposal": " cfg-1 "}) + if err != nil || !call.Clear || call.Proposal != "cfg-1" { + t.Errorf("read %+v (%v)", call, err) + } + if _, err := callOf(map[string]any{"clear": "yes"}); err == nil { + t.Error("a clear that is not a boolean was read") + } + c, _ := aMesh(t) + if _, err := c.Do(Call{Action: "set", Key: "font-size", Value: 12.0, Clear: true, Caller: "g", Why: "w"}); err == nil { + t.Error("a set took clear") + } + if _, err := c.Do(Call{Action: "propose", Key: "font-size", Value: 12.0, Caller: "g", Why: "w"}); err == nil || + !strings.Contains(err.Error(), "cannot be asked from here") { + t.Errorf("a proposal with no channel: %v", err) + } +} diff --git a/modules/mesh-config/cmd/mesh-config/pulled_in_test.go b/modules/mesh-config/cmd/mesh-config/pulled_in_test.go new file mode 100644 index 00000000..e4120e2c --- /dev/null +++ b/modules/mesh-config/cmd/mesh-config/pulled_in_test.go @@ -0,0 +1,24 @@ +package main + +import "testing" + +// A module only pulled in by another on a node, and handed to the provisioner there, has a value of its own in the +// assignments mirror (mesh-controller internal/inventory AssignmentValues, issue 765): what requires it, its plan, +// and no time it was assigned. Its values resolve there like an assigned module's, so the node's client can fill its +// templates; they go when the value goes. +func TestAModulePulledInHasItsValuesResolvedOnItsNode(t *testing.T) { + c, m := aMesh(t) + if _, err := c.Reconcile(); err != nil { + t.Fatal(err) + } + m.mirror(t, AssignmentsRead, "ace.dunst", `{"node":"ace","module":"dunst","required-by":"desk",`+ + `"provisioning":{"templates":{"source":"x","digest":"y"},"files":[]}}`) + if r, err := c.Reconcile(); err != nil || len(r.Moved) != 4 { + t.Fatalf("a pulled-in module's value moved %+v (%v)", r.Moved, err) + } + expect(t, m, "ace", "10", SourceDefault) + _ = m.Delete(AssignmentsRead, "ace.dunst") + if r, _ := c.Reconcile(); len(r.Moved) != 4 || !r.Moved[0].Gone { + t.Errorf("a module no longer pulled in moved %+v", r.Moved) + } +} diff --git a/modules/mesh-config/cmd/mesh-config/resolve.go b/modules/mesh-config/cmd/mesh-config/resolve.go new file mode 100644 index 00000000..f095f324 --- /dev/null +++ b/modules/mesh-config/cmd/mesh-config/resolve.go @@ -0,0 +1,229 @@ +package main + +import ( + "encoding/json" + "fmt" + "sort" + "strings" + "time" +) + +// The four sources (novox/hq ADR 0348 §2, §3, §9): a module's value on a node resolves, the most specific winning, +// from the default in its manifest, mesh config for the predefined key it falls back to, node config for that key +// on the node, and the module's own value on that node. A value naming no predefined key resolves from its default +// and the module's own value only. + +// The states this module keeps, and those it reads (its manifest's `state` and `reads`). +const ( + ValuesState = "values" + ResolvedState = "resolved" + PredefinedState = "predefined" + ProposalsState = "proposals" + + AssignmentsRead = "mesh-controller.assignments" + DeclaredConfigRead = "mesh-controller.declared-config" + PredefinedRead = "mesh-controller.predefined" + + // HistoryLog is the log of every set and clear, with the proven caller and why (ADR 0348 §9). + HistoryLog = "history" +) + +// The sources a resolved value says it came from. +const ( + SourceDefault = "default" + SourceMesh = "mesh" + SourceNode = "node" + SourceModule = "module" + // SourceUnset is a required value nothing sets: mesh-config answers it unset (ADR 0348 §4). + SourceUnset = "unset" +) + +// The levels a value is set at, and how each is keyed in mesh-config_values (ADR 0348 §9). +const ( + LevelMesh = "mesh" + LevelNode = "node" + LevelModule = "module" +) + +// Level is where a value is set: the mesh, a node, or a module on a node. +type Level struct { + Kind string `json:"level"` + Node string `json:"node,omitempty"` + Module string `json:"module,omitempty"` +} + +// ValueKey is the level's key for a config key in mesh-config_values. +func (l Level) ValueKey(key string) string { + switch l.Kind { + case LevelNode: + return "node." + l.Node + "." + key + case LevelModule: + return "module." + l.Node + "." + l.Module + "." + key + } + return "mesh." + key +} + +func (l Level) String() string { + switch l.Kind { + case LevelNode: + return "node config on " + l.Node + case LevelModule: + return l.Module + "'s own value on " + l.Node + } + return "mesh config" +} + +// parseValueKey is the level and the config key a key of mesh-config_values names. +func parseValueKey(k string) (Level, string, bool) { + parts := strings.SplitN(k, ".", 4) + switch { + case len(parts) >= 2 && parts[0] == LevelMesh: + return Level{Kind: LevelMesh}, strings.Join(parts[1:], "."), true + case len(parts) >= 3 && parts[0] == LevelNode: + return Level{Kind: LevelNode, Node: parts[1]}, strings.Join(parts[2:], "."), true + case len(parts) == 4 && parts[0] == LevelModule: + return Level{Kind: LevelModule, Node: parts[1], Module: parts[2]}, parts[3], true + } + return Level{}, "", false +} + +// Value is one key of mesh-config_values: the value set, who set it and why, and when. +type Value struct { + Value json.RawMessage `json:"value"` + // By is who set it, as proven: the caller, or for a value given at the controller's terminal, who gave it there. + By string `json:"by"` + // Via is the bus account the call came from. + Via string `json:"via"` + Why string `json:"why"` + // Approved is who approved it, for a value set on the operator's warrant (ADR 0348 §8, ADR 0277 §15): told from + // one given at the terminal, which leaves it empty. + Approved string `json:"approved,omitempty"` + At time.Time `json:"at"` +} + +// Resolved is one key of mesh-config_resolved, `..`: the value a module on a node takes for a +// key, and where it came from. Built from the sources alone, so the same sources give the same bytes and a pass +// over unchanged sources writes nothing. +type Resolved struct { + Node string `json:"node"` + Module string `json:"module"` + Key string `json:"key"` + Value json.RawMessage `json:"value,omitempty"` + Source string `json:"source"` + // Predefined is the predefined key the value falls back to, when it names one. + Predefined string `json:"predefined,omitempty"` + // Required says an unset value is the operator's own to set. + Required bool `json:"required,omitempty"` + Trusted bool `json:"trusted,omitempty"` +} + +// ResolvedKey is a resolved value's key in mesh-config_resolved. +func ResolvedKey(node, module, key string) string { return node + "." + module + "." + key } + +// Sources is everything a resolution reads: the assignments, every module's declarations, the published list of +// predefined keys, and every value set. +type Sources struct { + Assignments []Assignment + Declared map[string]ModuleDeclarations + Predefined map[string]Published + Values map[string]Value +} + +// Resolve is every assigned module's every declared value on its node, keyed as mesh-config_resolved is. +func Resolve(s Sources) map[string]Resolved { + out := map[string]Resolved{} + for _, a := range s.Assignments { + decl, ok := s.Declared[a.Module] + if !ok { + continue + } + for _, key := range sortedKeys(decl.Config) { + r := resolveOne(s, a.Node, a.Module, key, decl.Config[key]) + out[ResolvedKey(a.Node, a.Module, key)] = r + } + } + return out +} + +func resolveOne(s Sources, node, module, key string, d Declaration) Resolved { + r := Resolved{Node: node, Module: module, Key: key, Source: SourceUnset, Predefined: d.Predefined, + Required: d.Required, Trusted: d.Trusted} + if d.HasDefault() { + r.Value, r.Source = compact(d.Default), SourceDefault + } + if d.Predefined != "" { + if p, ok := s.Predefined[d.Predefined]; ok && p.Trusted { + r.Trusted = true + } + if v, ok := s.Values[Level{Kind: LevelMesh}.ValueKey(d.Predefined)]; ok { + r.Value, r.Source = compact(v.Value), SourceMesh + } + if v, ok := s.Values[Level{Kind: LevelNode, Node: node}.ValueKey(d.Predefined)]; ok { + r.Value, r.Source = compact(v.Value), SourceNode + } + } + if v, ok := s.Values[Level{Kind: LevelModule, Node: node, Module: module}.ValueKey(key)]; ok { + r.Value, r.Source = compact(v.Value), SourceModule + } + return r +} + +// compact is a JSON value encoded the one way: objects' keys sorted, no spaces. +func compact(raw json.RawMessage) json.RawMessage { + var v any + if err := json.Unmarshal(raw, &v); err != nil { + return raw + } + out, err := json.Marshal(v) + if err != nil { + return raw + } + return out +} + +// Moved is one resolved value that changed: a module on a node whose value for a key moved, from what to what. +type Moved struct { + Node string `json:"node"` + Module string `json:"module"` + Key string `json:"key"` + From json.RawMessage `json:"from,omitempty"` + To json.RawMessage `json:"to,omitempty"` + Source string `json:"source"` + // Gone says the module no longer takes the key there: unassigned, or no longer declared. + Gone bool `json:"gone,omitempty"` +} + +// diff is what moved between the resolved values held and those wanted, in key order. +func diff(held, want map[string]Resolved) []Moved { + var out []Moved + for _, k := range sortedKeys(want) { + w := want[k] + h, had := held[k] + if had && string(encodeResolved(h)) == string(encodeResolved(w)) { + continue + } + m := Moved{Node: w.Node, Module: w.Module, Key: w.Key, To: w.Value, Source: w.Source} + if had { + m.From = h.Value + } + out = append(out, m) + } + for _, k := range sortedKeys(held) { + if _, ok := want[k]; !ok { + h := held[k] + out = append(out, Moved{Node: h.Node, Module: h.Module, Key: h.Key, From: h.Value, Gone: true}) + } + } + sort.SliceStable(out, func(i, j int) bool { + return ResolvedKey(out[i].Node, out[i].Module, out[i].Key) < ResolvedKey(out[j].Node, out[j].Module, out[j].Key) + }) + return out +} + +func encodeResolved(r Resolved) []byte { + raw, err := json.Marshal(r) + if err != nil { + panic(fmt.Sprintf("a resolved value cannot be encoded: %v", err)) + } + return raw +} diff --git a/modules/mesh-config/cmd/mesh-config/sources.go b/modules/mesh-config/cmd/mesh-config/sources.go new file mode 100644 index 00000000..13cd30f1 --- /dev/null +++ b/modules/mesh-config/cmd/mesh-config/sources.go @@ -0,0 +1,252 @@ +package main + +import ( + "encoding/json" + "errors" + "fmt" + "strings" +) + +// Reading the sources, writing the resolved values, and publishing the list of predefined keys. + +// sources reads everything a resolution reads, from the bus as it is now. +func (c *Config) sources() (Sources, error) { + s := Sources{Declared: map[string]ModuleDeclarations{}, Predefined: map[string]Published{}, + Values: map[string]Value{}} + var problems []string + if err := c.each(AssignmentsRead, func(key string, raw []byte) error { + var a Assignment + if err := json.Unmarshal(raw, &a); err != nil || a.Node == "" || a.Module == "" { + return fmt.Errorf("the assignment %s is not one: %s", key, raw) + } + s.Assignments = append(s.Assignments, a) + return nil + }); err != nil { + return s, fmt.Errorf("the assignments cannot be read: %w", err) + } + if err := c.each(DeclaredConfigRead, func(key string, raw []byte) error { + m, err := ParseModuleDeclarations(raw) + if err != nil { + // One module's declarations unread leaves that module unresolved, said; the rest resolve. + problems = append(problems, fmt.Sprintf("%s: %v", key, err)) + return nil + } + s.Declared[m.Module] = m + return nil + }); err != nil { + return s, fmt.Errorf("the modules' declarations cannot be read: %w", err) + } + if err := c.each(PredefinedState, func(key string, raw []byte) error { + var p Published + if err := json.Unmarshal(raw, &p); err != nil { + return fmt.Errorf("the published predefined key %s is not one: %w", key, err) + } + s.Predefined[key] = p + return nil + }); err != nil { + return s, fmt.Errorf("the published list of predefined keys cannot be read: %w", err) + } + if err := c.each(ValuesState, func(key string, raw []byte) error { + var v Value + if err := json.Unmarshal(raw, &v); err != nil { + return fmt.Errorf("the value %s is not one: %w", key, err) + } + s.Values[key] = v + return nil + }); err != nil { + return s, fmt.Errorf("the values cannot be read: %w", err) + } + for _, p := range problems { + c.logf("[mesh-config] a module's declarations are not read, so its values are not resolved: %s", p) + } + return s, nil +} + +// each hands every key of a state to visit. +func (c *Config) each(state string, visit func(key string, raw []byte) error) error { + keys, err := c.Store.Keys(state) + if err != nil { + return err + } + for _, k := range keys { + e, err := c.Store.Get(state, k) + if err != nil { + return err + } + if e == nil { + continue // removed between the listing and the read + } + if err := visit(k, e.Value); err != nil { + return err + } + } + return nil +} + +// held is what mesh-config_resolved holds now. +func (c *Config) held() (map[string]Resolved, error) { + out := map[string]Resolved{} + err := c.each(ResolvedState, func(key string, raw []byte) error { + var r Resolved + if err := json.Unmarshal(raw, &r); err != nil { + // A value nobody can read is written again from the sources. + out[key] = Resolved{Key: key, Source: "unreadable"} + return nil + } + out[key] = r + return nil + }) + return out, err +} + +// writeResolved makes mesh-config_resolved hold what the sources resolve to, writing only what moved, and +// answers what moved. +func (c *Config) writeResolved(s Sources) ([]Moved, error) { + held, err := c.held() + if err != nil { + return nil, fmt.Errorf("the resolved values cannot be read: %w", err) + } + want := Resolve(s) + moved := diff(held, want) + var problems []error + for _, m := range moved { + key := ResolvedKey(m.Node, m.Module, m.Key) + if m.Gone { + if err := c.Store.Delete(ResolvedState, key); err != nil { + problems = append(problems, fmt.Errorf("removing %s: %w", key, err)) + } + continue + } + if err := c.Store.Put(ResolvedState, key, want[key]); err != nil { + problems = append(problems, fmt.Errorf("writing %s: %w", key, err)) + } + } + // A key that is no resolved value at all (written by an older mesh-config, or unreadable) is removed. + for key, h := range held { + if _, ok := want[key]; !ok && h.Source == "unreadable" { + if err := c.Store.Delete(ResolvedState, key); err != nil { + problems = append(problems, fmt.Errorf("removing %s: %w", key, err)) + } + } + } + return moved, errors.Join(problems...) +} + +// Resolution is what one pass of Reconcile did. +type Resolution struct { + Moved []Moved `json:"moved"` + Published string `json:"published,omitempty"` + Withheld string `json:"withheld,omitempty"` +} + +// Reconcile publishes the newest list of predefined keys and brings mesh-config_resolved to what the sources +// resolve to: run at start, and whenever an assignment, a declaration or the list moves. A move it makes raises +// config-changed like a set, with no level, naming what moved. +func (c *Config) Reconcile() (Resolution, error) { + c.mu.Lock() + defer c.mu.Unlock() + var r Resolution + published, err := c.publish() + if err != nil { + c.logf("[mesh-config] the list of predefined keys is not published: %v", err) + } + r.Published, r.Withheld = published, c.withheld + s, err := c.sources() + if err != nil { + return r, err + } + moved, err := c.writeResolved(s) + r.Moved = moved + if len(moved) > 0 { + c.record("resolved", Change{Action: "resolve", Why: "an assignment, a declaration or the list of predefined " + + "keys changed", At: c.now(), Moved: moved}) + } + return r, err +} + +// publish makes mesh-config_predefined hold the list the controller kept, when it is newer than the one published +// (ADR 0348 §7: the newest trunk build wins). **A list dropping a key that still holds a value is withheld**: the +// list published stays, and why is said in every listing and in the log, because a value for a key no longer on +// the list could neither be resolved nor cleared by name (§7: removing a key still holding a value is refused). +// It answers what it published, empty when nothing changed. +func (c *Config) publish() (string, error) { + e, err := c.Store.Get(PredefinedRead, PredefinedMirrorKey) + if err != nil { + return "", fmt.Errorf("the controller's list cannot be read: %w", err) + } + if e == nil { + return "", nil // no list kept yet + } + var list PredefinedList + if err := json.Unmarshal(e.Value, &list); err != nil || list.Keys == nil { + return "", fmt.Errorf("the controller's list is not one: %s", e.Value) + } + current := map[string]Published{} + if err := c.each(PredefinedState, func(key string, raw []byte) error { + var p Published + if err := json.Unmarshal(raw, &p); err != nil { + return err + } + current[key] = p + return nil + }); err != nil { + return "", err + } + generation := 0 + for _, p := range current { + generation = max(generation, p.Generation) + } + if len(current) > 0 && generation >= list.Generation { + c.withheld = "" + return "", nil + } + held := map[string][]string{} + if err := c.each(ValuesState, func(key string, _ []byte) error { + level, k, ok := parseValueKey(key) + if ok && level.Kind != LevelModule { + if _, kept := list.Keys[k]; !kept { + held[k] = append(held[k], level.String()) + } + } + return nil + }); err != nil { + return "", err + } + if len(held) > 0 { + var why []string + for _, k := range sortedKeys(held) { + why = append(why, fmt.Sprintf("%s (still set as %s)", k, strings.Join(held[k], ", "))) + } + c.withheld = fmt.Sprintf("the list at %s (generation %d) is not published: it drops %s; clear those values "+ + "first, or keep the key (novox/hq ADR 0348 §7)", short(list.Commit), list.Generation, strings.Join(why, "; ")) + c.logf("[mesh-config] %s", c.withheld) + return "", nil + } + c.withheld = "" + var problems []error + for _, k := range sortedKeys(list.Keys) { + p := Published{PredefinedKey: list.Keys[k], Generation: list.Generation, Commit: list.Commit} + if err := c.Store.Put(PredefinedState, k, p); err != nil { + problems = append(problems, err) + } + } + for k := range current { + if _, kept := list.Keys[k]; !kept { + if err := c.Store.Delete(PredefinedState, k); err != nil { + problems = append(problems, err) + } + } + } + if err := errors.Join(problems...); err != nil { + return "", err + } + return fmt.Sprintf("the list at %s (generation %d): %s", short(list.Commit), list.Generation, + joinKeys(list.Keys)), nil +} + +func short(commit string) string { + if len(commit) > 8 { + return commit[:8] + } + return commit +} diff --git a/modules/mesh-config/go.mod b/modules/mesh-config/go.mod new file mode 100644 index 00000000..42fbeb9d --- /dev/null +++ b/modules/mesh-config/go.mod @@ -0,0 +1,5 @@ +module mesh-config + +go 1.25.0 + +require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261013235212-c8b07242a009 diff --git a/modules/mesh-config/go.sum b/modules/mesh-config/go.sum new file mode 100644 index 00000000..7e6c029b --- /dev/null +++ b/modules/mesh-config/go.sum @@ -0,0 +1,2 @@ +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261013235212-c8b07242a009 h1:I9rm8HhYRJ0+mzQQz+VAH0kgv2YJWkvctEc67Yd0PX8= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261013235212-c8b07242a009/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY= diff --git a/modules/mesh-config/module.json b/modules/mesh-config/module.json new file mode 100644 index 00000000..9665b54c --- /dev/null +++ b/modules/mesh-config/module.json @@ -0,0 +1,73 @@ +{ + "module": "mesh-config", + "version": "1", + "runs-as": "mesh-config", + "claims": [ + { + "name": "mesh-config", + "scope": "mesh", + "serves": [ + "config" + ] + } + ], + "emits": [ + "config-changed" + ], + "uses": [ + "operator-channel" + ], + "invokes": [ + "seat:mesh-controller.hand-act", + "seat:mesh-controller.warranted" + ], + "reads": [ + "mesh-controller.assignments", + "mesh-controller.declared-config", + "mesh-controller.predefined" + ], + "state": [ + "values", + "resolved", + "predefined", + "proposals" + ], + "logs": [ + "history" + ], + "own-secrets": { + "broker": "${dir:state}/broker" + }, + "secrets-owner": "mesh-config", + "resources": [ + { + "id": "account", + "type": "user", + "name": "mesh-config", + "shell": "/usr/bin/nologin", + "home": "/var/lib/mesh-config" + }, + { + "id": "state", + "type": "directory", + "mode": "0700", + "owner": "mesh-config", + "place": "." + } + ], + "build": { + "artifacts": [ + { + "name": "tools", + "kind": "bundle", + "language": "go", + "system": "arch", + "from": "cmd/mesh-config", + "binary": "mesh-config", + "loads": [ + "mesh-config" + ] + } + ] + } +} diff --git a/modules/mesh-issues/cmd/mesh-issues/manifest_test.go b/modules/mesh-issues/cmd/mesh-issues/manifest_test.go index 1678c03e..15635181 100644 --- a/modules/mesh-issues/cmd/mesh-issues/manifest_test.go +++ b/modules/mesh-issues/cmd/mesh-issues/manifest_test.go @@ -25,9 +25,9 @@ type manifest struct { Logs []string `json:"logs"` Tools []string `json:"tools"` Resources []struct { - ID string `json:"id"` - Path string `json:"path"` - Content string `json:"content"` + ID string `json:"id"` + Path string `json:"path"` + Template string `json:"template"` } `json:"resources"` Build struct { Artifacts []struct { @@ -109,7 +109,8 @@ func TestItDeclaresAndHoldsTheIssueTrackerSeatOnceForTheMesh(t *testing.T) { "gitea.gitea_pull_reviews", "records.records_status", "records.records_list", "records.records_read", "mesh-knowledge.knowledge_highest"}) { t.Fatalf("consumes %v, invokes %v", m.Consumes, m.Invokes) } - if len(m.Build.Artifacts) != 1 || m.Build.Artifacts[0].From != "cmd/mesh-issues" || + // The tools bundle, and the archive the build packs files/ into for the templates (novox/hq ADR 0349 §2). + if len(m.Build.Artifacts) != 2 || m.Build.Artifacts[0].From != "cmd/mesh-issues" || m.Build.Artifacts[1].From != "files" || m.Build.Artifacts[0].Env["MESH_ISSUES_SETTINGS"] == "" || !reflect.DeepEqual(m.Build.Artifacts[0].Loads, []string{"mesh-issues"}) { t.Fatalf("build %+v", m.Build) } @@ -145,7 +146,7 @@ func TestTheSettingsStartTheCounterAndRegisterTheOwners(t *testing.T) { var content string for _, r := range m.Resources { if r.ID == "settings" { - content = r.Content + content = templateText(t, map[string]any{"id": r.ID, "template": r.Template}) } } var s Settings diff --git a/modules/mesh-issues/cmd/mesh-issues/templates_test.go b/modules/mesh-issues/cmd/mesh-issues/templates_test.go new file mode 100644 index 00000000..7d0718b5 --- /dev/null +++ b/modules/mesh-issues/cmd/mesh-issues/templates_test.go @@ -0,0 +1,64 @@ +package main + +import ( + "encoding/json" + "os" + "path/filepath" + "testing" +) + +// The module's directory, where its manifest and its files/ are. +const moduleDir = "../.." + +// templateText is the text a file resource places on the machine: its template's, in the module's files/ +// (novox/hq ADR 0349 §2). A resource that names no template, or a template that is not there, fails the test. +func templateText(t *testing.T, r map[string]any) string { + t.Helper() + name, _ := r["template"].(string) + if name == "" { + t.Fatalf("resource %v names no template", r["id"]) + } + raw, err := os.ReadFile(filepath.Join(moduleDir, filepath.FromSlash(name))) + if err != nil { + t.Fatalf("resource %v names %s: %v", r["id"], name, err) + } + return string(raw) +} + +// No file resource carries its text inline: each names its template in files/, and the build packs files/ +// into the one archive the controller reads the templates back from (novox/hq ADR 0349 §2). +func TestEveryFileTakesItsTextFromATemplate(t *testing.T) { + raw, err := os.ReadFile(filepath.Join(moduleDir, "module.json")) + if err != nil { + t.Fatal(err) + } + var m struct { + Resources []map[string]any `json:"resources"` + Build struct { + Artifacts []map[string]any `json:"artifacts"` + } `json:"build"` + } + if err := json.Unmarshal(raw, &m); err != nil { + t.Fatal(err) + } + for _, r := range m.Resources { + if _, inline := r["content"]; inline { + t.Errorf("resource %v keeps its text inline: keep it in files/ and name it as the template", r["id"]) + } + if r["type"] == "file" { + templateText(t, r) + } + } + packs := 0 + for _, a := range m.Build.Artifacts { + if a["kind"] == "archive" && a["from"] == "files" { + packs++ + if a["name"] != "files" { + t.Errorf("the templates archive is named %v, not files", a["name"]) + } + } + } + if packs != 1 { + t.Errorf("the build packs files/ into %d archives, not one", packs) + } +} diff --git a/modules/mesh-issues/files/settings.json b/modules/mesh-issues/files/settings.json new file mode 100644 index 00000000..2adad465 --- /dev/null +++ b/modules/mesh-issues/files/settings.json @@ -0,0 +1,145 @@ +{ + "first-number": 365, + "owners": [ + "hq", + "mesh-catalog", + "mesh-catalog modules/adwaita", + "mesh-catalog modules/artifact-store-tools", + "mesh-catalog modules/asus-zephyrus-g14", + "mesh-catalog modules/audit-logger", + "mesh-catalog modules/avahi", + "mesh-catalog modules/baserow", + "mesh-catalog modules/betterbird", + "mesh-catalog modules/blueman", + "mesh-catalog modules/bluetooth", + "mesh-catalog modules/build-agent", + "mesh-catalog modules/ca-trust", + "mesh-catalog modules/claude-code", + "mesh-catalog modules/claude-licence-manager", + "mesh-catalog modules/clipmenu", + "mesh-catalog modules/confluence", + "mesh-catalog modules/cups", + "mesh-catalog modules/dbus", + "mesh-catalog modules/desk-channel", + "mesh-catalog modules/disk-load", + "mesh-catalog modules/distribution", + "mesh-catalog modules/dmenu", + "mesh-catalog modules/dnsmasq", + "mesh-catalog modules/docker", + "mesh-catalog modules/docker-compose", + "mesh-catalog modules/dunst", + "mesh-catalog modules/fail2ban", + "mesh-catalog modules/feh", + "mesh-catalog modules/flatpak", + "mesh-catalog modules/fonts", + "mesh-catalog modules/forticlient", + "mesh-catalog modules/gitea", + "mesh-catalog modules/gitlab", + "mesh-catalog modules/gnome-keyring", + "mesh-catalog modules/grafana", + "mesh-catalog modules/hello-web", + "mesh-catalog modules/home-assistant", + "mesh-catalog modules/hostname", + "mesh-catalog modules/i3", + "mesh-catalog modules/i3status-rust", + "mesh-catalog modules/icecast", + "mesh-catalog modules/influxdb", + "mesh-catalog modules/invoicing", + "mesh-catalog modules/jira", + "mesh-catalog modules/keycloak", + "mesh-catalog modules/lab", + "mesh-catalog modules/lemurs", + "mesh-catalog modules/letta", + "mesh-catalog modules/local-model-consumer", + "mesh-catalog modules/localization", + "mesh-catalog modules/logrotate", + "mesh-catalog modules/mailu", + "mesh-catalog modules/marrytts", + "mesh-catalog modules/matrix", + "mesh-catalog modules/memory-pressure", + "mesh-catalog modules/mesh-catalog", + "mesh-catalog modules/mesh-cli", + "mesh-catalog modules/mesh-delivery", + "mesh-catalog modules/mesh-issues", + "mesh-catalog modules/mesh-vault", + "mesh-catalog modules/mesh-watcher", + "mesh-catalog modules/messenger", + "mesh-catalog modules/minio", + "mesh-catalog modules/model-usage", + "mesh-catalog modules/mongodb", + "mesh-catalog modules/mosquitto", + "mesh-catalog modules/mounts", + "mesh-catalog modules/mssql", + "mesh-catalog modules/n8n", + "mesh-catalog modules/nats", + "mesh-catalog modules/netcheck", + "mesh-catalog modules/networkmanager", + "mesh-catalog modules/nextcloud", + "mesh-catalog modules/nextcloud-client", + "mesh-catalog modules/nfs-server", + "mesh-catalog modules/nftables", + "mesh-catalog modules/nm-applet", + "mesh-catalog modules/node-env", + "mesh-catalog modules/nodered", + "mesh-catalog modules/ollama", + "mesh-catalog modules/only-office", + "mesh-catalog modules/openai-consumer", + "mesh-catalog modules/openrazer", + "mesh-catalog modules/pacman", + "mesh-catalog modules/photos-eef", + "mesh-catalog modules/photos-filip", + "mesh-catalog modules/picom", + "mesh-catalog modules/polychromatic", + "mesh-catalog modules/postgres", + "mesh-catalog modules/power", + "mesh-catalog modules/powerlevel10k", + "mesh-catalog modules/records", + "mesh-catalog modules/redis", + "mesh-catalog modules/restic", + "mesh-catalog modules/rofi", + "mesh-catalog modules/route-adapter", + "mesh-catalog modules/route-proxy", + "mesh-catalog modules/screen-lock", + "mesh-catalog modules/searxng", + "mesh-catalog modules/sensors", + "mesh-catalog modules/showcase", + "mesh-catalog modules/slack", + "mesh-catalog modules/snapd", + "mesh-catalog modules/ssh-client", + "mesh-catalog modules/sshd", + "mesh-catalog modules/step-ca", + "mesh-catalog modules/sudo", + "mesh-catalog modules/supabase", + "mesh-catalog modules/systemd", + "mesh-catalog modules/systemd-networkd", + "mesh-catalog modules/systemd-resolved", + "mesh-catalog modules/telegram", + "mesh-catalog modules/time-sync", + "mesh-catalog modules/triggerhappy", + "mesh-catalog modules/umami", + "mesh-catalog modules/unifi", + "mesh-catalog modules/website", + "mesh-catalog modules/xclip", + "mesh-catalog modules/xdg", + "mesh-catalog modules/xorg", + "mesh-catalog modules/xterm", + "mesh-catalog modules/zfs", + "mesh-catalog modules/zsh", + "mesh-catalog modules/zsh-autosuggestions", + "mesh-catalog modules/zsh-syntax-highlighting", + "mesh-controller", + "mesh-host", + "mesh-lab", + "mesh-media-catalog", + "mesh-sdk", + "mesh-tools" + ], + "duplicate-check": "on", + "judge-model": "sonnet", + "forge-owner": "novox", + "stuck-after": "2h", + "worker-stale-after": "24h", + "delivered-not-resolved-after": "1h", + "decision-floor": 293, + "research-floor": 45 +} diff --git a/modules/mesh-issues/module.json b/modules/mesh-issues/module.json index a6312a6a..e911c2b1 100644 --- a/modules/mesh-issues/module.json +++ b/modules/mesh-issues/module.json @@ -81,6 +81,207 @@ "grooming", "reservations" ], + "config": { + "first-number": { + "type": "number", + "description": "The number the first issue the tracker numbers gets, read once when its counter starts and never again; the import reads it as the boundary between the folders hq numbered and the tracker's issues.", + "minimum": 1, + "trusted": true, + "default": 365 + }, + "owners": { + "type": "list", + "description": "The registered places an issue's located-in may name: a repository, or a repository and a folder in it separated by a space.", + "trusted": true, + "default": [ + "hq", + "mesh-catalog", + "mesh-catalog modules/adwaita", + "mesh-catalog modules/artifact-store-tools", + "mesh-catalog modules/asus-zephyrus-g14", + "mesh-catalog modules/audit-logger", + "mesh-catalog modules/avahi", + "mesh-catalog modules/baserow", + "mesh-catalog modules/betterbird", + "mesh-catalog modules/blueman", + "mesh-catalog modules/bluetooth", + "mesh-catalog modules/build-agent", + "mesh-catalog modules/ca-trust", + "mesh-catalog modules/claude-code", + "mesh-catalog modules/claude-licence-manager", + "mesh-catalog modules/clipmenu", + "mesh-catalog modules/confluence", + "mesh-catalog modules/cups", + "mesh-catalog modules/dbus", + "mesh-catalog modules/desk-channel", + "mesh-catalog modules/disk-load", + "mesh-catalog modules/distribution", + "mesh-catalog modules/dmenu", + "mesh-catalog modules/dnsmasq", + "mesh-catalog modules/docker", + "mesh-catalog modules/docker-compose", + "mesh-catalog modules/dunst", + "mesh-catalog modules/fail2ban", + "mesh-catalog modules/feh", + "mesh-catalog modules/flatpak", + "mesh-catalog modules/fonts", + "mesh-catalog modules/forticlient", + "mesh-catalog modules/gitea", + "mesh-catalog modules/gitlab", + "mesh-catalog modules/gnome-keyring", + "mesh-catalog modules/grafana", + "mesh-catalog modules/hello-web", + "mesh-catalog modules/home-assistant", + "mesh-catalog modules/hostname", + "mesh-catalog modules/i3", + "mesh-catalog modules/i3status-rust", + "mesh-catalog modules/icecast", + "mesh-catalog modules/influxdb", + "mesh-catalog modules/invoicing", + "mesh-catalog modules/jira", + "mesh-catalog modules/keycloak", + "mesh-catalog modules/lab", + "mesh-catalog modules/lemurs", + "mesh-catalog modules/letta", + "mesh-catalog modules/local-model-consumer", + "mesh-catalog modules/localization", + "mesh-catalog modules/logrotate", + "mesh-catalog modules/mailu", + "mesh-catalog modules/marrytts", + "mesh-catalog modules/matrix", + "mesh-catalog modules/memory-pressure", + "mesh-catalog modules/mesh-catalog", + "mesh-catalog modules/mesh-cli", + "mesh-catalog modules/mesh-delivery", + "mesh-catalog modules/mesh-issues", + "mesh-catalog modules/mesh-vault", + "mesh-catalog modules/mesh-watcher", + "mesh-catalog modules/messenger", + "mesh-catalog modules/minio", + "mesh-catalog modules/model-usage", + "mesh-catalog modules/mongodb", + "mesh-catalog modules/mosquitto", + "mesh-catalog modules/mounts", + "mesh-catalog modules/mssql", + "mesh-catalog modules/n8n", + "mesh-catalog modules/nats", + "mesh-catalog modules/netcheck", + "mesh-catalog modules/networkmanager", + "mesh-catalog modules/nextcloud", + "mesh-catalog modules/nextcloud-client", + "mesh-catalog modules/nfs-server", + "mesh-catalog modules/nftables", + "mesh-catalog modules/nm-applet", + "mesh-catalog modules/node-env", + "mesh-catalog modules/nodered", + "mesh-catalog modules/ollama", + "mesh-catalog modules/only-office", + "mesh-catalog modules/openai-consumer", + "mesh-catalog modules/openrazer", + "mesh-catalog modules/pacman", + "mesh-catalog modules/photos-eef", + "mesh-catalog modules/photos-filip", + "mesh-catalog modules/picom", + "mesh-catalog modules/polychromatic", + "mesh-catalog modules/postgres", + "mesh-catalog modules/power", + "mesh-catalog modules/powerlevel10k", + "mesh-catalog modules/records", + "mesh-catalog modules/redis", + "mesh-catalog modules/restic", + "mesh-catalog modules/rofi", + "mesh-catalog modules/route-adapter", + "mesh-catalog modules/route-proxy", + "mesh-catalog modules/screen-lock", + "mesh-catalog modules/searxng", + "mesh-catalog modules/sensors", + "mesh-catalog modules/showcase", + "mesh-catalog modules/slack", + "mesh-catalog modules/snapd", + "mesh-catalog modules/ssh-client", + "mesh-catalog modules/sshd", + "mesh-catalog modules/step-ca", + "mesh-catalog modules/sudo", + "mesh-catalog modules/supabase", + "mesh-catalog modules/systemd", + "mesh-catalog modules/systemd-networkd", + "mesh-catalog modules/systemd-resolved", + "mesh-catalog modules/telegram", + "mesh-catalog modules/time-sync", + "mesh-catalog modules/triggerhappy", + "mesh-catalog modules/umami", + "mesh-catalog modules/unifi", + "mesh-catalog modules/website", + "mesh-catalog modules/xclip", + "mesh-catalog modules/xdg", + "mesh-catalog modules/xorg", + "mesh-catalog modules/xterm", + "mesh-catalog modules/zfs", + "mesh-catalog modules/zsh", + "mesh-catalog modules/zsh-autosuggestions", + "mesh-catalog modules/zsh-syntax-highlighting", + "mesh-controller", + "mesh-host", + "mesh-lab", + "mesh-media-catalog", + "mesh-sdk", + "mesh-tools" + ] + }, + "duplicate-check": { + "type": "string", + "description": "Whether a new issue is checked against the open ones for a duplicate: off stops it, anything else is on (novox/hq ADR 0278 §7).", + "trusted": true, + "default": "on" + }, + "judge-model": { + "type": "string", + "description": "The model the duplicate check asks, by the agent's name for it, such as sonnet; empty is the tracker's own default.", + "trusted": true, + "default": "sonnet" + }, + "forge-owner": { + "type": "string", + "description": "The owner on the forge of the repositories an issue's fixed-by names, for asking after a pull request's delivery (novox/hq ADR 0279 §3); empty asks no forge.", + "trusted": true, + "default": "novox" + }, + "stuck-after": { + "type": "string", + "description": "How long a fix may be merged and not delivered before it is said stuck, as a duration such as 2h (novox/hq ADR 0281); empty is the tracker's own default.", + "pattern": "|[0-9]+(\\.[0-9]+)?(ns|us|µs|ms|s|m|h)([0-9]+(\\.[0-9]+)?(ns|us|µs|ms|s|m|h))*", + "trusted": true, + "default": "2h" + }, + "worker-stale-after": { + "type": "string", + "description": "How long an issue's worker stays current with nothing done on it, as a duration such as 24h (novox/hq ADR 0284 §4); empty is the tracker's own default.", + "pattern": "|[0-9]+(\\.[0-9]+)?(ns|us|µs|ms|s|m|h)([0-9]+(\\.[0-9]+)?(ns|us|µs|ms|s|m|h))*", + "trusted": true, + "default": "24h" + }, + "delivered-not-resolved-after": { + "type": "string", + "description": "How long an issue may stay in fixing once every fix it names is delivered before the operator is warned, as a duration such as 1h (novox/hq issue 483); empty is the tracker's own default.", + "pattern": "|[0-9]+(\\.[0-9]+)?(ns|us|µs|ms|s|m|h)([0-9]+(\\.[0-9]+)?(ns|us|µs|ms|s|m|h))*", + "trusted": true, + "default": "1h" + }, + "decision-floor": { + "type": "number", + "description": "The highest decision number already taken somewhere the tracker cannot read, such as an open branch of hq; the decision sequence's first number is one above it, read once when the counter starts (novox/hq issue 424).", + "minimum": 0, + "trusted": true, + "default": 293 + }, + "research-floor": { + "type": "number", + "description": "The highest research number already taken somewhere the tracker cannot read; the research sequence's first number is one above it, read once when the counter starts (novox/hq issue 424).", + "minimum": 0, + "trusted": true, + "default": 45 + } + }, "resources": [ { "id": "state", @@ -94,8 +295,9 @@ "type": "file", "path": "${dir:state}/settings.json", "mode": "0600", + "takes-config": ["first-number", "owners", "duplicate-check", "judge-model", "forge-owner", "stuck-after", "worker-stale-after", "delivered-not-resolved-after", "decision-floor", "research-floor"], "merge": "json", - "content": "{\n \"first-number\": 365,\n \"owners\": [\n \"hq\",\n \"mesh-catalog\",\n \"mesh-catalog modules/adwaita\",\n \"mesh-catalog modules/artifact-store-tools\",\n \"mesh-catalog modules/asus-zephyrus-g14\",\n \"mesh-catalog modules/audit-logger\",\n \"mesh-catalog modules/avahi\",\n \"mesh-catalog modules/baserow\",\n \"mesh-catalog modules/betterbird\",\n \"mesh-catalog modules/blueman\",\n \"mesh-catalog modules/bluetooth\",\n \"mesh-catalog modules/build-agent\",\n \"mesh-catalog modules/ca-trust\",\n \"mesh-catalog modules/claude-code\",\n \"mesh-catalog modules/claude-licence-manager\",\n \"mesh-catalog modules/clipmenu\",\n \"mesh-catalog modules/confluence\",\n \"mesh-catalog modules/cups\",\n \"mesh-catalog modules/dbus\",\n \"mesh-catalog modules/de-spiegel\",\n \"mesh-catalog modules/desk-channel\",\n \"mesh-catalog modules/disk-load\",\n \"mesh-catalog modules/distribution\",\n \"mesh-catalog modules/dmenu\",\n \"mesh-catalog modules/dnsmasq\",\n \"mesh-catalog modules/docker\",\n \"mesh-catalog modules/docker-compose\",\n \"mesh-catalog modules/dunst\",\n \"mesh-catalog modules/fail2ban\",\n \"mesh-catalog modules/feh\",\n \"mesh-catalog modules/flatpak\",\n \"mesh-catalog modules/fonts\",\n \"mesh-catalog modules/forticlient\",\n \"mesh-catalog modules/gitea\",\n \"mesh-catalog modules/gitlab\",\n \"mesh-catalog modules/gnome-keyring\",\n \"mesh-catalog modules/grafana\",\n \"mesh-catalog modules/hello-web\",\n \"mesh-catalog modules/home-assistant\",\n \"mesh-catalog modules/hostname\",\n \"mesh-catalog modules/i3\",\n \"mesh-catalog modules/i3status-rust\",\n \"mesh-catalog modules/icecast\",\n \"mesh-catalog modules/influxdb\",\n \"mesh-catalog modules/invoicing\",\n \"mesh-catalog modules/jira\",\n \"mesh-catalog modules/keycloak\",\n \"mesh-catalog modules/lab\",\n \"mesh-catalog modules/lemurs\",\n \"mesh-catalog modules/letta\",\n \"mesh-catalog modules/local-model-consumer\",\n \"mesh-catalog modules/localization\",\n \"mesh-catalog modules/logrotate\",\n \"mesh-catalog modules/mailu\",\n \"mesh-catalog modules/marrytts\",\n \"mesh-catalog modules/matrix\",\n \"mesh-catalog modules/memory-pressure\",\n \"mesh-catalog modules/mesh-catalog\",\n \"mesh-catalog modules/mesh-cli\",\n \"mesh-catalog modules/mesh-delivery\",\n \"mesh-catalog modules/mesh-issues\",\n \"mesh-catalog modules/mesh-vault\",\n \"mesh-catalog modules/mesh-watcher\",\n \"mesh-catalog modules/messenger\",\n \"mesh-catalog modules/minio\",\n \"mesh-catalog modules/model-usage\",\n \"mesh-catalog modules/mongodb\",\n \"mesh-catalog modules/mosquitto\",\n \"mesh-catalog modules/mounts\",\n \"mesh-catalog modules/mssql\",\n \"mesh-catalog modules/n8n\",\n \"mesh-catalog modules/nats\",\n \"mesh-catalog modules/netcheck\",\n \"mesh-catalog modules/networkmanager\",\n \"mesh-catalog modules/nextcloud\",\n \"mesh-catalog modules/nextcloud-client\",\n \"mesh-catalog modules/nfs-server\",\n \"mesh-catalog modules/nftables\",\n \"mesh-catalog modules/nm-applet\",\n \"mesh-catalog modules/node-env\",\n \"mesh-catalog modules/nodered\",\n \"mesh-catalog modules/ollama\",\n \"mesh-catalog modules/only-office\",\n \"mesh-catalog modules/openai-consumer\",\n \"mesh-catalog modules/openrazer\",\n \"mesh-catalog modules/pacman\",\n \"mesh-catalog modules/photos-eef\",\n \"mesh-catalog modules/photos-filip\",\n \"mesh-catalog modules/picom\",\n \"mesh-catalog modules/polychromatic\",\n \"mesh-catalog modules/postgres\",\n \"mesh-catalog modules/power\",\n \"mesh-catalog modules/powerlevel10k\",\n \"mesh-catalog modules/records\",\n \"mesh-catalog modules/redis\",\n \"mesh-catalog modules/restic\",\n \"mesh-catalog modules/rofi\",\n \"mesh-catalog modules/route-adapter\",\n \"mesh-catalog modules/route-proxy\",\n \"mesh-catalog modules/screen-lock\",\n \"mesh-catalog modules/searxng\",\n \"mesh-catalog modules/sensors\",\n \"mesh-catalog modules/showcase\",\n \"mesh-catalog modules/slack\",\n \"mesh-catalog modules/snapd\",\n \"mesh-catalog modules/ssh-client\",\n \"mesh-catalog modules/sshd\",\n \"mesh-catalog modules/step-ca\",\n \"mesh-catalog modules/sudo\",\n \"mesh-catalog modules/supabase\",\n \"mesh-catalog modules/systemd\",\n \"mesh-catalog modules/systemd-networkd\",\n \"mesh-catalog modules/systemd-resolved\",\n \"mesh-catalog modules/telegram\",\n \"mesh-catalog modules/time-sync\",\n \"mesh-catalog modules/triggerhappy\",\n \"mesh-catalog modules/umami\",\n \"mesh-catalog modules/unifi\",\n \"mesh-catalog modules/website\",\n \"mesh-catalog modules/xclip\",\n \"mesh-catalog modules/xdg\",\n \"mesh-catalog modules/xorg\",\n \"mesh-catalog modules/xterm\",\n \"mesh-catalog modules/zfs\",\n \"mesh-catalog modules/zsh\",\n \"mesh-catalog modules/zsh-autosuggestions\",\n \"mesh-catalog modules/zsh-syntax-highlighting\",\n \"mesh-controller\",\n \"mesh-host\",\n \"mesh-lab\",\n \"mesh-media-catalog\",\n \"mesh-sdk\",\n \"mesh-tools\"\n ],\n \"duplicate-check\": \"on\",\n \"judge-model\": \"sonnet\",\n \"forge-owner\": \"novox\",\n \"stuck-after\": \"2h\",\n \"worker-stale-after\": \"24h\",\n \"delivered-not-resolved-after\": \"1h\",\n \"decision-floor\": 293,\n \"research-floor\": 45\n}\n" + "template": "files/settings.json" } ], "build": { @@ -113,6 +315,11 @@ "env": { "MESH_ISSUES_SETTINGS": "${dir:state}/settings.json" } + }, + { + "name": "files", + "kind": "archive", + "from": "files" } ] } diff --git a/modules/mesh-knowledge/cmd/mesh-knowledge/manifest_test.go b/modules/mesh-knowledge/cmd/mesh-knowledge/manifest_test.go index 6f09d217..46649ab7 100644 --- a/modules/mesh-knowledge/cmd/mesh-knowledge/manifest_test.go +++ b/modules/mesh-knowledge/cmd/mesh-knowledge/manifest_test.go @@ -89,7 +89,8 @@ func TestItDeclaresAndHoldsTheKnowledgeBaseSeatOnceForTheMesh(t *testing.T) { t.Errorf("the manifest invokes %s, which the code never calls", inv) } } - if len(m.Build.Artifacts) != 1 || m.Build.Artifacts[0].From != "cmd/mesh-knowledge" || m.Build.Artifacts[0].Binary != Module || + // The tools bundle, and the archive the build packs files/ into for the templates (novox/hq ADR 0349 §2). + if len(m.Build.Artifacts) != 2 || m.Build.Artifacts[1].From != "files" || m.Build.Artifacts[0].From != "cmd/mesh-knowledge" || m.Build.Artifacts[0].Binary != Module || m.Build.Artifacts[0].Env["DATABASE_URL_FILE"] == "" || m.Build.Artifacts[0].Env["MESH_KNOWLEDGE_SETTINGS"] == "" || m.Build.Artifacts[0].Env["MESH_KNOWLEDGE_STATUS"] == "" || !reflect.DeepEqual(m.Build.Artifacts[0].Loads, []string{Module}) { t.Fatalf("build %+v", m.Build) @@ -138,7 +139,7 @@ func TestItsProcessesAreThePreparationTheActAndTheJudge(t *testing.T) { } settings := byID["settings"] var s Settings - if err := json.Unmarshal([]byte(settings["content"].(string)), &s); err != nil || s.JudgeModel == "" || s.ReservedFor != "168h" { + if err := json.Unmarshal([]byte(templateText(t, settings)), &s); err != nil || s.JudgeModel == "" || s.ReservedFor != "168h" { t.Fatalf("the settings the manifest writes are not the module's: %v %+v", err, s) } if d, err := time.ParseDuration(s.ReservedFor); err != nil || d != DefaultReservedFor { diff --git a/modules/mesh-knowledge/cmd/mesh-knowledge/templates_test.go b/modules/mesh-knowledge/cmd/mesh-knowledge/templates_test.go new file mode 100644 index 00000000..7d0718b5 --- /dev/null +++ b/modules/mesh-knowledge/cmd/mesh-knowledge/templates_test.go @@ -0,0 +1,64 @@ +package main + +import ( + "encoding/json" + "os" + "path/filepath" + "testing" +) + +// The module's directory, where its manifest and its files/ are. +const moduleDir = "../.." + +// templateText is the text a file resource places on the machine: its template's, in the module's files/ +// (novox/hq ADR 0349 §2). A resource that names no template, or a template that is not there, fails the test. +func templateText(t *testing.T, r map[string]any) string { + t.Helper() + name, _ := r["template"].(string) + if name == "" { + t.Fatalf("resource %v names no template", r["id"]) + } + raw, err := os.ReadFile(filepath.Join(moduleDir, filepath.FromSlash(name))) + if err != nil { + t.Fatalf("resource %v names %s: %v", r["id"], name, err) + } + return string(raw) +} + +// No file resource carries its text inline: each names its template in files/, and the build packs files/ +// into the one archive the controller reads the templates back from (novox/hq ADR 0349 §2). +func TestEveryFileTakesItsTextFromATemplate(t *testing.T) { + raw, err := os.ReadFile(filepath.Join(moduleDir, "module.json")) + if err != nil { + t.Fatal(err) + } + var m struct { + Resources []map[string]any `json:"resources"` + Build struct { + Artifacts []map[string]any `json:"artifacts"` + } `json:"build"` + } + if err := json.Unmarshal(raw, &m); err != nil { + t.Fatal(err) + } + for _, r := range m.Resources { + if _, inline := r["content"]; inline { + t.Errorf("resource %v keeps its text inline: keep it in files/ and name it as the template", r["id"]) + } + if r["type"] == "file" { + templateText(t, r) + } + } + packs := 0 + for _, a := range m.Build.Artifacts { + if a["kind"] == "archive" && a["from"] == "files" { + packs++ + if a["name"] != "files" { + t.Errorf("the templates archive is named %v, not files", a["name"]) + } + } + } + if packs != 1 { + t.Errorf("the build packs files/ into %d archives, not one", packs) + } +} diff --git a/modules/mesh-knowledge/files/database.url b/modules/mesh-knowledge/files/database.url new file mode 100644 index 00000000..dc59da9a --- /dev/null +++ b/modules/mesh-knowledge/files/database.url @@ -0,0 +1 @@ +postgresql://${bound:postgres-database:as}:${secret:postgres-database}@${bound:postgres-database:at}:${bound:postgres-database:port}/${bound:postgres-database:as} diff --git a/modules/mesh-knowledge/files/settings.json b/modules/mesh-knowledge/files/settings.json new file mode 100644 index 00000000..6f682fed --- /dev/null +++ b/modules/mesh-knowledge/files/settings.json @@ -0,0 +1,5 @@ +{ + "judge-model": "sonnet", + "drafting": "on", + "reserved-for": "168h" +} diff --git a/modules/mesh-knowledge/module.json b/modules/mesh-knowledge/module.json index 6b914427..261662be 100644 --- a/modules/mesh-knowledge/module.json +++ b/modules/mesh-knowledge/module.json @@ -77,6 +77,24 @@ "records.records_read", "seat:operator-channel.notify" ], + "config": { + "judge-model": { + "type": "string", + "description": "The model the knowledge base's drafting and confirming judgements ask, by the agent's name for it, such as sonnet; empty is sonnet.", + "default": "sonnet" + }, + "drafting": { + "type": "string", + "description": "Whether the knowledge base drafts and confirms clauses itself: off stops it, anything else is on.", + "default": "on" + }, + "reserved-for": { + "type": "string", + "description": "How long a reserved number waits for its proposal before it is withdrawn, as a duration such as 168h; empty is the knowledge base's own default.", + "pattern": "|[0-9]+(\\.[0-9]+)?(ns|us|µs|ms|s|m|h)([0-9]+(\\.[0-9]+)?(ns|us|µs|ms|s|m|h))*", + "default": "168h" + } + }, "resources": [ { "id": "state", @@ -89,7 +107,7 @@ "type": "file", "path": "${dir:state}/database.url", "mode": "0600", - "content": "postgresql://${bound:postgres-database:as}:${secret:postgres-database}@${bound:postgres-database:at}:${bound:postgres-database:port}/${bound:postgres-database:as}\n" + "template": "files/database.url" }, { "id": "settings", @@ -97,8 +115,9 @@ "type": "file", "path": "${dir:state}/settings.json", "mode": "0600", + "takes-config": ["judge-model", "drafting", "reserved-for"], "merge": "json", - "content": "{\n \"judge-model\": \"sonnet\",\n \"drafting\": \"on\",\n \"reserved-for\": \"168h\"\n}\n" + "template": "files/settings.json" }, { "id": "prepare", @@ -181,6 +200,11 @@ "MESH_KNOWLEDGE_SETTINGS": "${dir:state}/settings.json", "MESH_KNOWLEDGE_STATUS": "${dir:state}/health.json" } + }, + { + "name": "files", + "kind": "archive", + "from": "files" } ] } diff --git a/modules/mesh-mail/cmd/mesh-mail/manifest_test.go b/modules/mesh-mail/cmd/mesh-mail/manifest_test.go index 3ccfc83c..87b01d30 100644 --- a/modules/mesh-mail/cmd/mesh-mail/manifest_test.go +++ b/modules/mesh-mail/cmd/mesh-mail/manifest_test.go @@ -129,18 +129,25 @@ func TestTheManifestSaysWhatTheCodeDoes(t *testing.T) { // The two settings are trusted, and their defaults are an empty account list and no grant. for id, want := range map[string]string{"accounts": `{"accounts":{}}`, "grants": `{"grants":{}}`} { r := byID[id] - if r == nil || r["trusted"] != true || r["merge"] != "json" || !reflect.DeepEqual(r["takes-settings"], []any{id}) { + if r == nil || r["trusted"] != true || r["merge"] != "json" || !reflect.DeepEqual(r["takes-config"], []any{id}) { t.Fatalf("%s: %v", id, r) } - if strings.Join(strings.Fields(r["content"].(string)), "") != want { - t.Fatalf("%s's default is %q", id, r["content"]) - } - if _, err := ParseAccounts([]byte(byID["accounts"]["content"].(string))); err != nil { + // The file's text is kept once, in its template under files/ (novox/hq ADR 0349 §2). + raw, err := os.ReadFile(filepath.Join("../..", r["template"].(string))) + if err != nil { t.Fatal(err) } + if strings.Join(strings.Fields(string(raw)), "") != want { + t.Fatalf("%s's default is %q", id, raw) + } + if id == "accounts" { + if _, err := ParseAccounts(raw); err != nil { + t.Fatal(err) + } + } } a := m.Build.Artifacts - if len(a) != 1 || a[0].From != "cmd/mesh-mail" || a[0].Binary != Module || !reflect.DeepEqual(a[0].Loads, []string{Module}) { + if len(a) != 2 || a[1].From != "files" || a[0].From != "cmd/mesh-mail" || a[0].Binary != Module || !reflect.DeepEqual(a[0].Loads, []string{Module}) { t.Fatalf("build %+v", m.Build) } for k, v := range map[string]string{"MESH_MAIL_ACCOUNTS": "accounts", "MESH_MAIL_GRANTS": "grants", "MESH_MAIL_STORE": "store", "DATABASE_URL_FILE": "database-url"} { diff --git a/modules/mesh-mail/files/accounts.json b/modules/mesh-mail/files/accounts.json new file mode 100644 index 00000000..e182cc3d --- /dev/null +++ b/modules/mesh-mail/files/accounts.json @@ -0,0 +1,3 @@ +{ + "accounts": {} +} diff --git a/modules/mesh-mail/files/database-url b/modules/mesh-mail/files/database-url new file mode 100644 index 00000000..dc59da9a --- /dev/null +++ b/modules/mesh-mail/files/database-url @@ -0,0 +1 @@ +postgresql://${bound:postgres-database:as}:${secret:postgres-database}@${bound:postgres-database:at}:${bound:postgres-database:port}/${bound:postgres-database:as} diff --git a/modules/mesh-mail/files/grants.json b/modules/mesh-mail/files/grants.json new file mode 100644 index 00000000..458eabba --- /dev/null +++ b/modules/mesh-mail/files/grants.json @@ -0,0 +1,3 @@ +{ + "grants": {} +} diff --git a/modules/mesh-mail/files/store.json b/modules/mesh-mail/files/store.json new file mode 100644 index 00000000..8c2075da --- /dev/null +++ b/modules/mesh-mail/files/store.json @@ -0,0 +1,9 @@ +{ + "scheme": "${bound:s3-bucket:scheme}", + "at": "${bound:s3-bucket:at}", + "port": "${bound:s3-bucket:port}", + "bucket": "${bound:s3-bucket:bucket}", + "region": "${bound:s3-bucket:region}", + "key": "${bound:s3-bucket:as}", + "secret": "${secret:s3-bucket}" +} diff --git a/modules/mesh-mail/module.json b/modules/mesh-mail/module.json index 0cb00ea0..ae445990 100644 --- a/modules/mesh-mail/module.json +++ b/modules/mesh-mail/module.json @@ -51,6 +51,20 @@ "prompt": "the password of the mail account {member}" } }, + "config": { + "accounts": { + "type": "object", + "description": "The mail accounts, each by a lower-case name that names it on the bus and its secret: its kind (imap), address, incoming and outgoing servers (host, port, security tls or starttls), and optionally the folders watched and the user it signs in as; no password, which is the own secret mail-credential- (novox/hq ADR 0359 §5).", + "trusted": true, + "default": {} + }, + "grants": { + "type": "object", + "description": "Per account, who may read its mail and who may send from it: a module's name (on every machine), ., or person.. Nobody by default; it restricts the stream MAIL and the tools mail_search, mail_read and mail_send alike (novox/hq ADR 0359 §4).", + "trusted": true, + "default": {} + } + }, "tools": [ "mail_accounts", "mail_search", @@ -96,14 +110,14 @@ "type": "file", "path": "${dir:state}/database.url", "mode": "0600", - "content": "postgresql://${bound:postgres-database:as}:${secret:postgres-database}@${bound:postgres-database:at}:${bound:postgres-database:port}/${bound:postgres-database:as}\n" + "template": "files/database-url" }, { "id": "store", "type": "file", "path": "${dir:state}/store.json", "mode": "0600", - "content": "{\n \"scheme\": \"${bound:s3-bucket:scheme}\",\n \"at\": \"${bound:s3-bucket:at}\",\n \"port\": \"${bound:s3-bucket:port}\",\n \"bucket\": \"${bound:s3-bucket:bucket}\",\n \"region\": \"${bound:s3-bucket:region}\",\n \"key\": \"${bound:s3-bucket:as}\",\n \"secret\": \"${secret:s3-bucket}\"\n}\n" + "template": "files/store.json" }, { "id": "accounts", @@ -112,10 +126,10 @@ "path": "${dir:state}/accounts.json", "mode": "0600", "merge": "json", - "takes-settings": [ + "template": "files/accounts.json", + "takes-config": [ "accounts" - ], - "content": "{\n \"accounts\": {}\n}\n" + ] }, { "id": "grants", @@ -124,10 +138,10 @@ "path": "${dir:state}/grants.json", "mode": "0600", "merge": "json", - "takes-settings": [ + "template": "files/grants.json", + "takes-config": [ "grants" - ], - "content": "{\n \"grants\": {}\n}\n" + ] }, { "id": "prepare", @@ -221,6 +235,11 @@ "MESH_MAIL_STORE": "${dir:state}/store.json", "MESH_MAIL_STATUS": "${dir:state}/health.json" } + }, + { + "name": "files", + "kind": "archive", + "from": "files" } ] } diff --git a/modules/mesh-provisioner-client/README.md b/modules/mesh-provisioner-client/README.md new file mode 100644 index 00000000..62c1ec4c --- /dev/null +++ b/modules/mesh-provisioner-client/README.md @@ -0,0 +1,88 @@ +# mesh-provisioner-client + +Holds the `node-provisioner` seat on every node: the provisioner's client (novox/hq ADR 0349 clauses 3 to 5, design 57 +P5). Go, one binary (`cmd/mesh-provisioner-client`), run as a service of its own under the account +`mesh-provisioner-client`, outside the tool runner, on a bus account of its own, `provisioner.`, delivered as +its own secret `provisioner-broker` (mesh-controller `internal/broker/provisioner.go`). The bus grants that account +its own machine's keys and, to read alone, the whole assignments mirror, where a provider's consumers on any +machine are bound to it (ADR 0373); nothing else. + +## What it does + +For each module the controller handed to the provisioner on its node — an assignment in +`mesh-controller_assignments` (`.`) whose value carries a `provisioning` plan: the templates archive +by reference and digest, the files it fills (path, template, `owner-at-runtime`, how its program takes a change) +and the placement values its templates name — the client: + +- fetches the templates archive and uses it only when it hashes to the plan's digest, keeping it by digest; +- records each placeholder as a need in `mesh-provisioner_needs`, `...` (a character a + key cannot hold written `=`); +- watches each owner's keys for its node: placement in the plan, config in `mesh-config_resolved` + (`..`), both named by `mesh-provisioner_registry`; +- asks the vault (`mesh-vault.get`, named by the registry) for each of the module's own secrets its plan names + (`provisioning.secrets`), in its own name on the subject naming its account, with the public half of a key it + made at start and keeps in memory alone; the vault answers the secret sealed to that key (ADR 0206's box), so only + the box crosses the bus. A secret is asked once per run, and again when the vault says `secret-changed` for it, + so a rotation reaches its file without a new declaration. A secret with a `path` is written to that file alone, + as an `owned` file. A secret the vault does not hold yet, or a vault that does not answer, is waited for; a + refusal fails the module with the vault's reason and is asked again after ten minutes. A value is never written + to a bucket, a report or a log; +- asks the vault the same way for each credential of what the module requires that its plan names + (a `provisioning.provisions` entry with a `name`, issue 757; every entry names the provider serving that + provision, issue 760, and its offer's credential form, issue 768, which the vault acts on): the value a provider's software issued for this module on this machine, + `provision////` (the forge's read token for claude-code, issue 737), filled + wherever `${secret:}` names it and written to its `path` alone, and asked again on its `secret-changed`; +- fills each item the module contributes to a collection (`provisioning.items`: the collection and its template) + from the same owners as a file, and records it in `mesh-provisioner_collect`, `..` + (a character a key's token cannot hold written `=`: `shell:zsh:normal` is `shell=3Azsh=3Anormal`). An item + naming a secret fails: an item is kept in a bucket, and no secret is; +- fills a collector's `${collected:}` from every item of that collection recorded for its node, in the + contributors' order, each introduced by a line naming its contributor in the collection's comment + (`provisioning.collections`: the collections the module collects, each with its comment). The three old + namespaces read as collections of the same name: `${shell:zsh:normal}` is the collection `shell:zsh:normal`, + `${environment:posix}` is `environment:posix`, `${contribution:node-hotkeys:trigger}` is + `contribution:node-hotkeys:trigger`. A collector naming a collection its plan does not fails. A collection with no + item is filled empty, and filled again whenever an item of it is recorded or goes; +- fills a module's files only once every value is there; otherwise it writes nothing and reports `waiting` with what + it waits for; +- has the node-engine write each changed file through its local write socket (`/run/nox-mesh-host/fill.sock`, + mesh-host `internal/filling`), which writes it as the declaration says (an `owned` file whole, a `seed` only when + absent, a `managed` file's own keys or block) and restarts or reloads its program as the declaration says; +- writes every file again every minute, so the node-engine puts back an `owned` file changed on disk; +- reports `prov.applied..` and keeps the same report in `mesh-provisioner_applied` (`.`): + `applied` with each file's action and digest, `waiting`, `failed` with the reason, or `removed`; +- when a module is taken off the node, removes its needs, its items and its report (the node-engine removes the files its + declaration no longer names); and when the bus comes back, or the client starts, fills from the current values; +- fills a provider's `${consumers:}` (novox/hq ADR 0373, issue 679) with one entry per consumer bound to + it for that provision, in the order of the assignments' keys and, within a plan, of its `provisions` entries: a + JSON list, each entry `{consumer, module, local, credential, value}`, the credential its own, + `pair////[/]`, asked of the vault in the client's own name as a secret + is. Who binds a provider is read from every machine's plans, so the client watches the whole assignments mirror + (it carries no secret's value) and reads only its own machine's plans as its own. A consumer whose credential the + vault does not hold yet, or will not hand, is left out and said in the log, so the others are served; the + provider's helper waits for it (mesh-sdk `go/provider` ConsumersFile). It is filled again when a consumer binds or + unbinds and when the vault says one of the credentials changed. The catalogue holds the file to mode 0600 and + owned, and refuses the placeholder in a module not providing that provision (mesh-controller internal/catalogue + consumers.go); an item naming it fails; +- fills a consumer's `${bound::}` from its provider's entry (mesh-sdk `go/provider`, issue 678): the + bucket `_bound` of the provider its plan names, key `..[.]`, as + the registry says, watched for its own machine's keys once a plan names it; `as`, `user` or one of the served + values. An entry not written yet, or a bucket not there yet, is waited for; one the provider refused fails the + module with the provider's reason. + +It fills `config`, `secret` (design 57 V4, issue 647), the collections (`collected`, and `shell`, `environment` +and `contribution` read as collections: design 57 L1, issue 684), `consumers` and `bound` (issue 679) and the +placement kinds (`dir`, `access`, `port`, `seat`, `machine`). Any other kind fails with the reason. + +## A module's code asks at runtime + +A module whose code runs in its machine's tool runner reads its config values through the SDK (ADR 0349 clause 14 +as ADR 0370 changes it, issue 685). The tool runner asks this client on `prov.ask.`, which only the same +machine's tool runner may publish, with `{module, kind, key}`; the module is the one whose bundle asks, named by the +tool runner and never by the module's code. The client answers `{value}`, the value as `mesh-config_resolved` holds +it, only for a module assigned to its machine and a key mesh-config resolved for it (what its manifest declares), or +`{error}` with the reason. It never answers a secret: the vault hands a secret only to a caller the bus proves, so a +module in the tool runner reads its secrets from its files. A binding's non-secret values (`bound`) are filled +into files (issue 679) and not yet answered on an ask. It says on `prov.changed..` which value moved (`{kind, key}`, every +key of the kind when `key` is absent, never the value), when the client starts and on every change, so the tool +runner asks again for the watches it holds. diff --git a/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/asks.go b/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/asks.go new file mode 100644 index 00000000..d4753a25 --- /dev/null +++ b/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/asks.go @@ -0,0 +1,119 @@ +package main + +import ( + "bytes" + "encoding/json" + "fmt" + "strings" + + "github.com/nats-io/nats.go" +) + +// A module's code asks at runtime (novox/hq ADR 0349 clause 14 as ADR 0370 changes it, design 57 S1, issue 685). +// +// While a module's code runs in its machine's tool runner, its asks reach the bus under the tool runner's account, +// and the bus proves only that account. So the client is asked on AskSubject, which the controller grants its own +// machine's tool runner alone to publish (ADR 0370 §3), and the module is the one the tool runner names on the ask, +// taken from the bundle that asks (§4). The client answers that module's config values, for what its manifest +// declares — mesh-config resolves a module's declared keys and no other — and only when the composition places the +// module on this machine, where the tool runner carries every module assigned to it. It never answers a secret on +// the tool runner's word (§2): the vault hands a secret only to a caller the bus proves (ADR 0349 clause 11), and a +// module in the tool runner reads its secrets from its files. A binding's non-secret values are its provider's +// (ADR 0349 clause 12); this client fills none yet, so it answers none yet either, and says so. +// +// A watch is the tool runner's: the client says on ChangedSubject which value of a module moved, never the value, +// and the tool runner asks again for the watches it holds. + +// AskSubject is where this machine's tool runner asks (mesh-controller internal/broker ProvisionerAskSubject). +func AskSubject(node string) string { return "prov.ask." + node } + +// ChangedSubject is where the client says a value of one module moved (mesh-controller internal/broker +// ProvisionerChangedSubject). +func ChangedSubject(node, module string) string { return "prov.changed." + node + "." + module } + +// The kinds of value an ask names. +const ( + AskConfig = "config" + AskBound = "bound" + AskSecret = "secret" +) + +// Ask is what the tool runner asks: the module it names, from the bundle that asks; the kind; and the key, a +// config key, or `:` of a binding. +type Ask struct { + Module string `json:"module"` + Kind string `json:"kind"` + Key string `json:"key"` +} + +// Answered is the client's answer: the value as mesh-config resolved it (JSON), or why it is refused. +type Answered struct { + Value json.RawMessage `json:"value,omitempty"` + Error string `json:"error,omitempty"` +} + +// Changed is what the client says moved of one module: a kind, and the key, or every key of the kind when empty. +type Changed struct { + Kind string `json:"kind"` + Key string `json:"key,omitempty"` +} + +// answer is the answer to one ask from the state the client holds. +func (c *Client) answer(s *state, body []byte) Answered { + var a Ask + dec := json.NewDecoder(bytes.NewReader(body)) + dec.DisallowUnknownFields() + if err := dec.Decode(&a); err != nil || a.Module == "" || a.Kind == "" || a.Key == "" { + return Answered{Error: "an ask names its module, its kind and its key: {module, kind, key}"} + } + switch a.Kind { + case AskSecret: + return Answered{Error: fmt.Sprintf("%s's secret %s is not answered on the tool runner's word: the vault hands "+ + "a secret only to a caller the bus proves (novox/hq ADR 0370 §2, ADR 0349 §11); while the module's code "+ + "runs in the tool runner, its secrets reach it through its files", a.Module, a.Key)} + case AskConfig, AskBound: + default: + return Answered{Error: fmt.Sprintf("%q is not a kind the client answers: config or bound", a.Kind)} + } + if !s.assigned[a.Module] { + return Answered{Error: fmt.Sprintf("%s is not placed on %s, so this machine's tool runner carries no code of "+ + "it to answer (novox/hq ADR 0370 §3)", a.Module, c.Node)} + } + if a.Kind == AskBound { + return Answered{Error: fmt.Sprintf("%s's bound value %s is not answered yet: a binding's values come from its "+ + "provider (novox/hq ADR 0349 §12), which this client fills into the module's files (issue 679) and does not "+ + "answer on an ask yet", a.Module, a.Key)} + } + v, ok := s.config[a.Module][a.Key] + if !ok { + return Answered{Error: fmt.Sprintf("%s declares no config value %s, or mesh-config holds none for it on %s", + a.Module, a.Key, c.Node)} + } + return Answered{Value: v} +} + +// respond answers one ask on its reply. +func (c *Client) respond(s *state, m *nats.Msg) { + if m.Reply == "" { + return + } + a := c.answer(s, m.Data) + body, err := json.Marshal(a) + if err != nil { + return + } + if err := m.Respond(body); err != nil { + logf("[provisioner-client] an ask could not be answered: %v", err) + } +} + +// tell says a value of a module moved, so its code's watches are asked again. +func (c *Client) tell(module string, ch Changed) { + if module == "" || strings.ContainsAny(module, "*> ") { + return + } + body, _ := json.Marshal(ch) + if err := c.Conn.Publish(ChangedSubject(c.Node, module), body); err != nil { + logf("[provisioner-client] what moved of %s could not be said: %v", module, err) + } +} diff --git a/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/asks_test.go b/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/asks_test.go new file mode 100644 index 00000000..f9908a01 --- /dev/null +++ b/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/asks_test.go @@ -0,0 +1,154 @@ +package main + +import ( + "encoding/json" + "fmt" + "strings" + "testing" + "time" + + "github.com/nats-io/nats.go" +) + +// A module's code asks at runtime (novox/hq ADR 0349 clause 14, ADR 0370, design 57 S1, issue 685), against a real +// bus whose accounts are the client's and its machine's tool runner's as mesh-controller composes them: the tool +// runner asks for a carried module's config value and is answered; a change is said, so a watch asks again; a key +// the module does not declare, a module not placed on the machine, a secret and a malformed ask are refused, each +// with its reason. + +// toolRunner is g14's tool runner on the bus, asking as its account may. +type toolRunner struct { + t *testing.T + nc *nats.Conn +} + +func aToolRunner(t *testing.T, b *aBus) *toolRunner { + t.Helper() + nc, err := nats.Connect(b.url, nats.UserInfo(node+".node-tools", "x"), + nats.CustomInboxPrefix("_INBOX."+node+".node-tools"), + nats.ErrorHandler(func(_ *nats.Conn, _ *nats.Subscription, err error) { + t.Errorf("the bus refused the tool runner: %v", err) + })) + if err != nil { + t.Fatal(err) + } + t.Cleanup(nc.Close) + return &toolRunner{t: t, nc: nc} +} + +func (r *toolRunner) ask(body string) Answered { + r.t.Helper() + var a Answered + deadline := time.Now().Add(10 * time.Second) + for { + msg, err := r.nc.Request(AskSubject(node), []byte(body), 2*time.Second) + if err == nil { + if err := json.Unmarshal(msg.Data, &a); err != nil { + r.t.Fatalf("the client answered %s", msg.Data) + } + return a + } + if time.Now().After(deadline) { + r.t.Fatalf("the client did not answer %s: %v", body, err) + } + time.Sleep(50 * time.Millisecond) // not listening yet + } +} + +func (r *toolRunner) askConfig(module, key string) Answered { + r.t.Helper() + return r.ask(fmt.Sprintf(`{"module":%q,"kind":"config","key":%q}`, module, key)) +} + +// hear collects what the client says moved on this machine. +func (r *toolRunner) hear() chan string { + r.t.Helper() + heard := make(chan string, 64) + if _, err := r.nc.Subscribe("prov.changed."+node+".>", func(m *nats.Msg) { + heard <- strings.TrimPrefix(m.Subject, "prov.changed."+node+".") + " " + string(m.Data) + }); err != nil { + r.t.Fatal(err) + } + _ = r.nc.Flush() + return heard +} + +// placed writes an assignment of a module the controller has not handed to the provisioner: placed, no plan. +func (b *aBus) placed(t *testing.T, module string) { + t.Helper() + b.put(t, AssignmentsBucket, node+"."+module, fmt.Sprintf(`{"node":%q,"module":%q,"assigned":"2026-10-14T20:00:00Z"}`, + node, module)) +} + +func until(t *testing.T, heard chan string, want string) { + t.Helper() + deadline := time.After(10 * time.Second) + for { + select { + case got := <-heard: + if got == want { + return + } + case <-deadline: + t.Fatalf("the client did not say %s", want) + } + } +} + +func TestNatsAModulesCodeReadsItsConfigAndIsToldOfAChange(t *testing.T) { + b := startBus(t) + e := startEngine(t, nil) + b.placed(t, "claude-code") + b.config(t, "claude-code", "managed_settings", `{"spinnerTipsEnabled":false}`) + runner := aToolRunner(t, b) + heard := runner.hear() + aClient(t, b, e, t.TempDir(), time.Hour) + + // The value as mesh-config resolved it. + if a := runner.askConfig("claude-code", "managed_settings"); a.Error != "" || string(a.Value) != `{"spinnerTipsEnabled":false}` { + t.Fatalf("answered %+v", a) + } + // Coming up, it says every module's config moved, so a watch held across its absence asks again. + until(t, heard, `claude-code {"kind":"config"}`) + + // A change is said, which value and never the value; asked again, the new value. + b.config(t, "claude-code", "managed_settings", `{"spinnerTipsEnabled":true}`) + until(t, heard, `claude-code {"kind":"config","key":"managed_settings"}`) + if a := runner.askConfig("claude-code", "managed_settings"); string(a.Value) != `{"spinnerTipsEnabled":true}` { + t.Fatalf("after the change, answered %+v", a) + } + + // Taken off the machine: said, and no longer answered. + b.del(t, AssignmentsBucket, node+".claude-code") + until(t, heard, `claude-code {"kind":"config"}`) + if a := runner.askConfig("claude-code", "managed_settings"); !strings.Contains(a.Error, "not placed on g14") { + t.Fatalf("a module taken off the machine was answered %+v", a) + } +} + +func TestNatsWhatAModulesCodeMayNotReadIsRefusedWithTheReason(t *testing.T) { + b := startBus(t) + e := startEngine(t, nil) + b.placed(t, "claude-code") + b.config(t, "claude-code", "managed_settings", `{}`) + // Resolved for a module the composition does not place on this machine. + b.config(t, "dunst", "font-size", "13") + runner := aToolRunner(t, b) + aClient(t, b, e, t.TempDir(), time.Hour) + + for _, c := range []struct{ ask, refused string }{ + {`{"module":"claude-code","kind":"config","key":"no_such_key"}`, "declares no config value no_such_key"}, + {`{"module":"dunst","kind":"config","key":"font-size"}`, "dunst is not placed on g14"}, + {`{"module":"claude-code","kind":"secret","key":"gitea-read"}`, "not answered on the tool runner's word"}, + {`{"module":"claude-code","kind":"bound","key":"mcp-endpoint:url"}`, "not answered yet"}, + {`{"module":"claude-code","kind":"setting","key":"role"}`, "not a kind the client answers"}, + {`{"module":"claude-code","kind":"config"}`, "names its module, its kind and its key"}, + {`{"module":"claude-code","kind":"config","key":"managed_settings","as":"dunst"}`, "names its module, its kind and its key"}, + {`not json`, "names its module, its kind and its key"}, + } { + a := runner.ask(c.ask) + if len(a.Value) != 0 || !strings.Contains(a.Error, c.refused) { + t.Errorf("%s: answered %+v, want refused with %q", c.ask, a, c.refused) + } + } +} diff --git a/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/bars_golden_test.go b/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/bars_golden_test.go new file mode 100644 index 00000000..8fb7cccb --- /dev/null +++ b/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/bars_golden_test.go @@ -0,0 +1,89 @@ +package main + +import ( + "encoding/json" + "os" + "path/filepath" + "testing" + "time" +) + +// The bar files i3status-rust's bars come to, filled by this client from power's items of the collection +// contribution:node-bar:block, are byte for byte what the old shaped contribution made (design 57 L3, issue 781). +// The module files are the catalogue's own; testdata/bars holds the bars as composed before the move, for a machine +// with neither capability, the battery, the power meter and both. The controller's side of the same bars, not handed, +// is pinned by mesh-controller internal/catalogue TestTheBarsOfTheCatalogueAreTheSameHandedOrNot. + +func TestNatsTheBarsFilledFromPowersItemsAreWhatTheOldContributionMade(t *testing.T) { + modules := filepath.Join("..", "..", "..") + raw, err := os.ReadFile(filepath.Join(modules, "power", "module.json")) + if err != nil { + t.Fatal(err) + } + var power struct { + Items []struct { + Collection string `json:"collection"` + Name string `json:"name"` + IfCapability string `json:"if-capability"` + Attributes map[string]string `json:"attributes"` + Template string `json:"template"` + } `json:"items"` + } + if err := json.Unmarshal(raw, &power); err != nil { + t.Fatal(err) + } + for name, caps := range map[string][]string{"none": nil, "battery": {"battery"}, "power-meter": {"power-meter"}, + "both": {"battery", "power-meter"}} { + t.Run(name, func(t *testing.T) { + has := map[string]bool{} + for _, c := range caps { + has[c] = true + } + bar := func(which string) string { + b, err := os.ReadFile(filepath.Join(modules, "i3status-rust", "files", which+"-bar.toml")) + if err != nil { + t.Fatal(err) + } + return string(b) + } + barTpl, _ := templates(t, map[string]string{"bottom": bar("bottom"), "top": bar("top")}) + bars := Plan{Templates: barTpl, Files: []PlannedFile{ + {ID: "bottom", Path: "/bars/bottom.toml", Template: "bottom", OwnerAtRuntime: "owned"}, + {ID: "top", Path: "/bars/top.toml", Template: "top", OwnerAtRuntime: "owned"}}, + Collections: []PlannedCollection{{Name: "contribution:node-bar:block", Comment: "#"}}, + Placement: map[string]string{"machine:name": "workstation"}} + texts := map[string]string{} + var items []PlannedItem + for _, it := range power.Items { + if it.IfCapability != "" && !has[it.IfCapability] { + continue // the controller leaves it out of the plan + } + text, err := os.ReadFile(filepath.Join(modules, "power", it.Template)) + if err != nil { + t.Fatal(err) + } + id := filepath.Base(it.Template) + texts[id] = string(text) + items = append(items, PlannedItem{Collection: it.Collection, Name: it.Name, Template: id, + Attributes: it.Attributes}) + } + itemTpl, _ := templates(t, texts) + contrib := Plan{Templates: itemTpl, Items: items, Placement: map[string]string{"machine:name": "workstation"}} + + b := startBus(t) + e := startEngine(t, map[string]string{"/bars/bottom.toml": "owned", "/bars/top.toml": "owned"}) + b.assign(t, "i3status-rust", bars) + b.assign(t, "power", contrib) + aClient(t, b, e, t.TempDir(), time.Hour) + for _, which := range []string{"bottom", "top"} { + golden, err := os.ReadFile(filepath.Join("testdata", "bars", name+"."+which+"-bar.toml")) + if err != nil { + t.Fatal(err) + } + eventually(t, "the "+which+" bar is what it was", func() bool { + return e.read(t, "/bars/"+which+".toml") == string(golden) + }) + } + }) + } +} diff --git a/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/bound_gitea_test.go b/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/bound_gitea_test.go new file mode 100644 index 00000000..23664ee4 --- /dev/null +++ b/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/bound_gitea_test.go @@ -0,0 +1,146 @@ +package main + +import ( + "context" + "os" + "strings" + "testing" + "time" + + "github.com/nats-io/nats.go/jetstream" +) + +// A handed gitea's server.env (issue 788): its template is the catalogue's own, and its machine's client fills it from +// its provider's entry as the provider's helper now writes it (`as`, and `at` and `from` among the values) and from +// the vault's rows, to the text the controller composed for the same values before it was handed. +const giteaEnvPath = "/etc/gitea/server.env" + +func giteaTemplate(t *testing.T) string { + t.Helper() + raw, err := os.ReadFile("../../../gitea/files/server.env") + if err != nil { + t.Fatal(err) + } + return string(raw) +} + +// giteaBoundExpected is that file as the controller rendered it: the database as the binding file told gitea. +func giteaBoundExpected(internalToken, password string) string { + return "GITEA__security__INTERNAL_TOKEN=" + internalToken + "\n" + + "GITEA__database__DB_TYPE=postgres\n" + + "GITEA__database__HOST=10.7.0.1:5432\n" + + "GITEA__database__NAME=mesh_g14_gitea\n" + + "GITEA__database__USER=mesh_g14_gitea\n" + + "GITEA__database__PASSWD=" + password + "\n" + + "GITEA__webhook__ALLOWED_HOST_LIST=172.17.0.1\n" +} + +func giteaRun(t *testing.T, entry string, extra string) (*engine, *aBus, *reports) { + t.Helper() + b := startBus(t) + registryOf679(t, b) + vault := startVault(t, b) + vault.keep("own/g14/gitea/internal-token", "tok-never-on-the-bus", false) + vault.keep("pair/novox/postgres-database/g14/gitea", "pw-never-on-the-bus", false) + heard := b.listen(t) + e := startEngine(t, map[string]string{giteaEnvPath: "owned"}) + tpl, _ := templates(t, map[string]string{"server.env": giteaTemplate(t) + extra}) + b.assign(t, "gitea", Plan{Templates: tpl, Files: []PlannedFile{{ID: "env", Path: giteaEnvPath, + Template: "server.env", OwnerAtRuntime: "owned"}}, + Secrets: []PlannedSecret{{Name: "internal-token"}}, + Provisions: []PlannedProvision{{Name: "postgres-database", Provision: "postgres-database", Provider: "novox", + ProviderModule: "postgres", CredentialForm: "pair"}}}) + kv, err := b.domain.CreateKeyValue(context.Background(), jetstream.KeyValueConfig{Bucket: "postgres_bound"}) + if err != nil { + t.Fatal(err) + } + b.kv["postgres_bound"] = kv + b.put(t, "postgres_bound", node+".gitea.postgres-database", entry) + aClient(t, b, e, t.TempDir(), time.Hour) + return e, b, heard +} + +func TestNatsAHandedGiteasServerEnvIsFilledAsTheControllerComposedIt(t *testing.T) { + e, _, _ := giteaRun(t, `{"provider":"novox","provider-module":"postgres","provision":"postgres-database",`+ + `"consumer":"g14","module":"gitea","as":"mesh_g14_gitea","user":"mesh_g14_gitea",`+ + `"values":{"at":"10.7.0.1","from":"novox","port":5432},"credential":"pair/novox/postgres-database/g14/gitea"}`, + "FROM=${bound:postgres-database:from}\n") + eventually(t, "server.env is filled to the controller's text", func() bool { + return e.read(t, giteaEnvPath) == giteaBoundExpected("tok-never-on-the-bus", "pw-never-on-the-bus")+"FROM=novox\n" + }) +} + +// An entry without `at`, or without `from`, fills nothing: the module fails naming the key, never a file with a hole. +func TestNatsAnEntryWithoutAtOrFromLeavesGiteasServerEnvUnfilled(t *testing.T) { + for key, values := range map[string]string{"at": `{"from":"novox","port":5432}`, "from": `{"at":"10.7.0.1","port":5432}`} { + e, _, heard := giteaRun(t, `{"as":"mesh_g14_gitea","user":"mesh_g14_gitea","values":`+values+`}`, + "FROM=${bound:postgres-database:from}\n") + eventually(t, "the module fails for the missing "+key, func() bool { + r, _ := heard.last("gitea") + return r.Status == StatusFailed && strings.Contains(r.Reason, `no "`+key+`"`) + }) + if got := e.read(t, giteaEnvPath); strings.Contains(got, "GITEA__database__HOST") { + t.Fatalf("without %s the file was filled: %q", key, got) + } + } +} + +// A handed gitea's origins file (issue 792): filled by its machine's client from the route provider's entry, which +// carries the public name the controller composed for gitea, to the text the controller wrote for the binding file. +func TestNatsAHandedGiteasOriginsAreFilledFromTheRouteProvidersEntry(t *testing.T) { + const originsPath = "/etc/gitea/origins" + raw, err := os.ReadFile("../../../gitea/files/origins") + if err != nil { + t.Fatal(err) + } + b := startBus(t) + registryOf679(t, b) + startVault(t, b) + e := startEngine(t, map[string]string{originsPath: "owned"}) + tpl, _ := templates(t, map[string]string{"origins": string(raw)}) + b.assign(t, "gitea", Plan{Templates: tpl, Files: []PlannedFile{{ID: "origins", Path: originsPath, + Template: "origins", OwnerAtRuntime: "owned"}}, + Provisions: []PlannedProvision{{Provision: "route", Provider: "novox", ProviderModule: "route-adapter"}}}) + kv, err := b.domain.CreateKeyValue(context.Background(), jetstream.KeyValueConfig{Bucket: "route-adapter_bound"}) + if err != nil { + t.Fatal(err) + } + b.kv["route-adapter_bound"] = kv + b.put(t, "route-adapter_bound", node+".gitea.route", `{"provider":"novox","provider-module":"route-adapter",`+ + `"provision":"route","consumer":"g14","module":"gitea","as":"mesh_g14_gitea",`+ + `"values":{"name":"git.example.org","internal-name":"git.novox.internal","name-web":"git.example.org"}}`) + aClient(t, b, e, t.TempDir(), time.Hour) + eventually(t, "origins is filled to the controller's text", func() bool { + return e.read(t, originsPath) == "https://git.example.org\n" + }) +} + +// The same where route-proxy, not route-adapter, answers the route, as on novox (issue 792): filled by its machine's client from the route provider's entry, which +// carries the public name the controller composed for gitea, to the text the controller wrote for the binding file. +func TestNatsAHandedGiteasOriginsAreFilledFromRouteProxysEntryOnNovox(t *testing.T) { + const originsPath = "/etc/gitea/origins" + raw, err := os.ReadFile("../../../gitea/files/origins") + if err != nil { + t.Fatal(err) + } + b := startBus(t) + registryOf679(t, b) + startVault(t, b) + e := startEngine(t, map[string]string{originsPath: "owned"}) + tpl, _ := templates(t, map[string]string{"origins": string(raw)}) + b.assign(t, "gitea", Plan{Templates: tpl, Files: []PlannedFile{{ID: "origins", Path: originsPath, + Template: "origins", OwnerAtRuntime: "owned"}}, + Provisions: []PlannedProvision{{Provision: "route", Provider: "novox", ProviderModule: "route-proxy"}}}) + kv, err := b.domain.CreateKeyValue(context.Background(), jetstream.KeyValueConfig{Bucket: "route-proxy_bound"}) + if err != nil { + t.Fatal(err) + } + b.kv["route-proxy_bound"] = kv + b.put(t, "route-proxy_bound", node+".gitea.route", `{"provider":"novox","provider-module":"route-proxy",`+ + `"provision":"route","consumer":"g14","module":"gitea","as":"mesh_g14_gitea",`+ + `"values":{"name":"git.example.org","internal-name":"git.novox.internal","name-web":"git.example.org"}}`) + aClient(t, b, e, t.TempDir(), time.Hour) + eventually(t, "origins is filled to the controller's text", func() bool { + return e.read(t, originsPath) == "https://git.example.org\n" + }) +} diff --git a/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/client.go b/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/client.go new file mode 100644 index 00000000..42b87570 --- /dev/null +++ b/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/client.go @@ -0,0 +1,1021 @@ +package main + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/json" + "errors" + "fmt" + "slices" + "sort" + "strings" + "time" + + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" +) + +// The provisioner's client on one node (novox/hq ADR 0349 clauses 4 and 5, design 57 P5, issue 671). +// +// For each module the controller handed to the provisioner on this node, the client reads its templates' +// placeholders, records them as the module's needs, watches each owner's keys for this node (a watch delivers the +// current values first), fills the templates only once every value is there — otherwise it keeps the files there +// are and reports what it waits for — has the node-engine write each changed file through its narrow local verb, +// which makes the program take the change as the file declares, and reports `prov.applied` (applied, or failed +// with the reason). It does so again on every change it watches, writes every file again on a drift pass so the +// node-engine puts an owned file back, and when its node comes back it starts from the current values. A module +// taken off the node loses its needs and its report; the node-engine removes the files the declaration no longer +// names. +// +// The owners it fills from are placement (the controller's, in the module's assignment), config +// (mesh-config_resolved) and the vault, asked for each secret the module's plan names (secrets.go), and for each +// consumer's credential a provider here lists (consumers.go); and each provider's entries for a consumer here. Every +// key it writes is its own machine's, and every key it reads but the assignments mirror's, which it reads whole for +// who binds a provider here (ADR 0373): its bus account is granted no other. + +// The buckets the client reads and writes, and the subject it reports on. +const ( + AssignmentsBucket = "mesh-controller_assignments" + ConfigBucket = "mesh-config_resolved" + RegistryBucket = "mesh-provisioner_registry" + NeedsBucket = "mesh-provisioner_needs" + AppliedBucket = "mesh-provisioner_applied" +) + +// AppliedSubject is where the client says what it applied for one module (ADR 0349 clauses 4 and 15). +func AppliedSubject(node, module string) string { return "prov.applied." + node + "." + module } + +// What a report says of a module. +const ( + StatusApplied = "applied" + StatusWaiting = "waiting" + StatusFailed = "failed" + StatusRemoved = "removed" +) + +// Report is what the client says of one module on its node, on prov.applied and in mesh-provisioner_applied. +type Report struct { + Node string `json:"node"` + Module string `json:"module"` + Status string `json:"status"` + // Reason is why it failed. + Reason string `json:"reason,omitempty"` + // WaitsFor is each value it waits for, `:`, and why. + WaitsFor []string `json:"waits-for,omitempty"` + // Templates is the digest of the templates it filled from: which build of the module this is. + Templates string `json:"templates,omitempty"` + Files []FileApplied `json:"files,omitempty"` + // Items are the collections the module's items are recorded in on this node (design 57 L1). + Items []string `json:"items,omitempty"` + At time.Time `json:"at"` +} + +// FileApplied is one file as the node-engine wrote it. +type FileApplied struct { + Path string `json:"path"` + Action string `json:"action,omitempty"` + Digest string `json:"digest,omitempty"` + Takes string `json:"takes,omitempty"` +} + +// NeedRecord is one need a module's templates name, as mesh-provisioner_needs keeps it at +// `...`: the key's name in a form the bus takes, and the name as written here. +type NeedRecord struct { + Node string `json:"node"` + Module string `json:"module"` + Kind string `json:"kind"` + Name string `json:"name"` + Files []string `json:"files"` +} + +// Answer is a registry's entry, as mesh-provisioner keeps it: which domain answers a kind, and where. +type Answer struct { + Kind string `json:"kind"` + Domain string `json:"domain"` + From string `json:"from"` + Bucket string `json:"bucket,omitempty"` + Key string `json:"key,omitempty"` + // Ask is a seat's verb asked on request and reply, `.`: the vault's get for a secret. + Ask string `json:"ask,omitempty"` +} + +// resolved is one value of mesh-config_resolved, as far as filling reads it. +type resolved struct { + Value json.RawMessage `json:"value,omitempty"` +} + +// Client is one node's provisioner client. +type Client struct { + Node string + JS jetstream.JetStream + // Conn is where the reports are published. + Conn *nats.Conn + Fill Filler + // Secrets hands the module's secrets from the vault (secrets.go); nil fails every secret placeholder. + Secrets Secrets + // Templates fetches and keeps the templates archives. + Templates *templateStore + // Drift is how often every applied module's files are written again, so an owned file changed on disk is put + // back; zero is a minute. + Drift time.Duration + // Now dates the reports; nil is time.Now. + Now func() time.Time + + // watchBound watches a provider's bucket of entries for this machine's keys, once per run (consumers.go); set by + // Run. + watchBound func(bucket string) error + + // What it last had written, by path: the content's digest. Kept across runs of Run, so a node coming back + // writes only what moved — and the drift pass writes everything again anyway. + written map[string][32]byte +} + +// state is what one run holds, from its watches. +type state struct { + registry map[string]Answer + plans map[string]Plan + refused map[string]string // a module whose plan cannot be acted on, and why + config map[string]map[string]json.RawMessage + needs map[string][]byte // key → value recorded + applied map[string]bool // module → a report is kept + reported map[string][]byte // module → the last report said, undated + secrets map[string]heldSecret // the vault's name → what this run holds of it; never written anywhere + // items is every item recorded for this node, by collection and contributor: what a collector is filled from. + items map[string]map[string]itemValue + // recorded is each item's key and its value as kept: what this client compares before it records again. + recorded map[string][]byte + assigned map[string]bool // every module assigned to this node, with a plan or not (asks.go) + // binds is every machine's assignment by its key, as far as whom it binds (consumers.go): what a provider here is + // filled with its consumers from. + binds map[string]binding + // providing is each module here whose templates name its consumers (consumers.go), and leftOut what was last said + // of the consumers left out of one of its files. + providing map[string]bool + leftOut map[string]string + // bound is each provider's bucket of entries this client watches, by key: this machine's consumers' entries alone; + // boundReady is a bucket whose current entries are all read, and boundUsers the modules filled from each. + bound map[string]map[string][]byte + boundReady map[string]bool + boundUsers map[string]map[string]bool +} + +type update struct { + bucket string + entry jetstream.KeyValueEntry // nil: the watch delivered every current value + // changed is what the vault said of a secret that changed (secret-changed): never a value. Set, entry is nil. + changed []byte + // asked is the tool runner asking for a module's value (asks.go). Set, entry is nil. + asked *nats.Msg +} + +// Run serves until ctx ends or a watch stops, which it returns: run again, it starts from the current values. +func (c *Client) Run(ctx context.Context) error { + if c.written == nil { + c.written = map[string][32]byte{} + } + ctx, cancel := context.WithCancel(ctx) + defer cancel() + kvs := map[string]jetstream.KeyValue{} + for _, b := range []string{AssignmentsBucket, ConfigBucket, RegistryBucket, NeedsBucket, AppliedBucket, + CollectBucket} { + kv, err := c.JS.KeyValue(ctx, b) + if err != nil { + return fmt.Errorf("the bucket %s: %w", b, err) + } + kvs[b] = kv + } + updates := make(chan update, 256) + stopped := make(chan error, 64) + watch := func(bucket, keys string) error { + w, err := kvs[bucket].Watch(ctx, keys) + if err != nil { + return fmt.Errorf("watching %s %s: %w", bucket, keys, err) + } + go func() { + defer func() { _ = w.Stop() }() + for { + select { + case <-ctx.Done(): + return + case e, ok := <-w.Updates(): + if !ok { + stopped <- fmt.Errorf("the watch of %s stopped", bucket) + return + } + select { + case updates <- update{bucket: bucket, entry: e}: + case <-ctx.Done(): + return + } + } + } + }() + return nil + } + mine := c.Node + ".>" + // The registry whole, and every machine's assignment: a provider here is bound by consumers on any machine, in + // their plans (ADR 0373). Every other bucket, this machine's keys alone. + for _, b := range []string{RegistryBucket, AssignmentsBucket} { + if err := watch(b, ">"); err != nil { + return err + } + } + for _, b := range []string{ConfigBucket, NeedsBucket, AppliedBucket, CollectBucket} { + if err := watch(b, mine); err != nil { + return err + } + } + // A provider's bucket of entries, watched once a plan here names that provider (consumers.go): this machine's + // consumers' keys alone. A bucket not there yet is tried again at the next fill. + watchedBound := map[string]bool{} + c.watchBound = func(bucket string) error { + if watchedBound[bucket] { + return nil + } + kv, err := c.JS.KeyValue(ctx, bucket) + if err != nil { + return err + } + kvs[bucket] = kv + if err := watch(bucket, mine); err != nil { + return err + } + watchedBound[bucket] = true + return nil + } + defer func() { c.watchBound = nil }() + // What the vault says changed: a secret this run holds is asked for again, so a rotation reaches its file + // without a new declaration. + said, err := c.Conn.Subscribe(SecretChangedSubject, func(m *nats.Msg) { + select { + case updates <- update{bucket: SecretChangedSubject, changed: m.Data}: + case <-ctx.Done(): + } + }) + if err != nil { + return fmt.Errorf("listening for %s: %w", SecretChangedSubject, err) + } + defer func() { _ = said.Unsubscribe() }() + // This machine's tool runner asking for a module's config value (asks.go): answered from this run's state, in + // its loop, so an answer is never read from half an update. + asks, err := c.Conn.Subscribe(AskSubject(c.Node), func(m *nats.Msg) { + select { + case updates <- update{asked: m}: + case <-ctx.Done(): + } + }) + if err != nil { + return fmt.Errorf("listening for %s: %w", AskSubject(c.Node), err) + } + defer func() { _ = asks.Unsubscribe() }() + + s := &state{registry: map[string]Answer{}, plans: map[string]Plan{}, refused: map[string]string{}, + config: map[string]map[string]json.RawMessage{}, needs: map[string][]byte{}, applied: map[string]bool{}, + reported: map[string][]byte{}, secrets: map[string]heldSecret{}, items: map[string]map[string]itemValue{}, + recorded: map[string][]byte{}, assigned: map[string]bool{}, binds: map[string]binding{}, + providing: map[string]bool{}, leftOut: map[string]string{}, bound: map[string]map[string][]byte{}, + boundReady: map[string]bool{}, boundUsers: map[string]map[string]bool{}} + // Every current value first: nothing is filled from half of them. + current := map[string]bool{} + var early []*nats.Msg + for len(current) < 6 { + select { + case <-ctx.Done(): + return ctx.Err() + case err := <-stopped: + return err + case u := <-updates: + if u.asked != nil { + // Answered once every current value is in: never from half of them. + early = append(early, u.asked) + continue + } + if u.changed != nil { + // Nothing is held yet: every secret is asked for when it is first filled. + continue + } + if u.entry == nil { + current[u.bucket] = true + continue + } + c.take(s, u) + } + } + // A module that left while this client was away: its needs and its report go. + for _, m := range c.leftModules(s) { + c.leave(ctx, kvs, s, m) + } + // Contributors first, so a collector starting with its node is filled from the items there are now. + for _, m := range contributorsFirst(s.plans) { + c.reconcile(ctx, kvs, s, m, false) + } + for _, m := range sortedKeys(s.refused) { + c.report(ctx, kvs, s, Report{Module: m, Status: StatusFailed, Reason: s.refused[m]}) + } + // What was asked meanwhile, and every module's config said to have moved: a watch held across this client's + // absence is asked again from the current values. + for _, m := range early { + c.respond(s, m) + } + for _, m := range sortedKeys(s.config) { + c.tell(m, Changed{Kind: AskConfig}) + } + + drift := c.Drift + if drift <= 0 { + drift = time.Minute + } + tick := time.NewTicker(drift) + defer tick.Stop() + for { + select { + case <-ctx.Done(): + return ctx.Err() + case err := <-stopped: + return err + case <-tick.C: + for _, m := range sortedKeys(s.plans) { + c.reconcile(ctx, kvs, s, m, true) + } + case u := <-updates: + // What moved together is filled together: every update waiting is taken before anything is filled. + dirty := map[string]bool{} + for { + switch { + case u.asked != nil: + c.respond(s, u.asked) + case u.changed != nil: + for _, m := range c.secretChanged(s, u.changed) { + dirty[m] = true + } + case u.entry != nil: + for _, m := range c.take(s, u) { + dirty[m] = true + } + c.told(u) + case strings.HasSuffix(u.bucket, boundSuffix): + // A provider's bucket's current entries are all read: what waited on them is filled. + s.boundReady[u.bucket] = true + for m := range s.boundUsers[u.bucket] { + dirty[m] = true + } + } + select { + case u = <-updates: + continue + case <-time.After(50 * time.Millisecond): + } + break + } + for _, m := range contributorsFirst(onlyThese(s.plans, dirty)) { + switch { + case s.refused[m] != "": + c.report(ctx, kvs, s, Report{Module: m, Status: StatusFailed, Reason: s.refused[m]}) + case hasPlan(s, m): + c.reconcile(ctx, kvs, s, m, false) + default: + c.leave(ctx, kvs, s, m) + } + } + } + } +} + +func hasPlan(s *state, m string) bool { _, ok := s.plans[m]; return ok } + +// told says what an update moved of a module's config, or of its placement, which decides whether it is answered +// at all (asks.go). +func (c *Client) told(u update) { + rest, ok := strings.CutPrefix(u.entry.Key(), c.Node+".") + if !ok { + return + } + switch u.bucket { + case ConfigBucket: + if module, name, ok := strings.Cut(rest, "."); ok { + c.tell(module, Changed{Kind: AskConfig, Key: name}) + } + case AssignmentsBucket: + c.tell(rest, Changed{Kind: AskConfig}) + } +} + +// onlyThese is the modules of dirty, each with its plan where it has one: what contributorsFirst orders. +func onlyThese(plans map[string]Plan, dirty map[string]bool) map[string]Plan { + out := map[string]Plan{} + for m := range dirty { + out[m] = plans[m] + } + return out +} + +// contributorsFirst is the modules in name order, those contributing an item before the others: a contributor's item +// recorded first is in its collector's file the first time it is filled. +func contributorsFirst(plans map[string]Plan) []string { + var first, then []string + for _, m := range sortedKeys(plans) { + if len(plans[m].Items) > 0 { + first = append(first, m) + } else { + then = append(then, m) + } + } + return append(first, then...) +} + +// take folds one update into the state, and answers the modules it may change. +func (c *Client) take(s *state, u update) []string { + e := u.entry + gone := e.Operation() != jetstream.KeyValuePut + key := e.Key() + switch u.bucket { + case RegistryBucket: + if gone { + delete(s.registry, key) + } else { + var a Answer + if err := json.Unmarshal(e.Value(), &a); err != nil { + logf("[provisioner-client] the registry's %s cannot be read, so the kind is not answered: %v", key, err) + delete(s.registry, key) + } else { + s.registry[key] = a + } + } + // A kind's answer moves what every module waits on. + return sortedKeys(s.plans) + case AssignmentsBucket: + // Whom it binds, whichever machine's it is: a provider here listing its consumers is filled again. + var providers []string + if takeBinds(s, key, e.Value(), gone) { + providers = sortedKeys(s.providing) + } + module, mine := strings.CutPrefix(key, c.Node+".") + if !mine { + return providers + } + delete(s.plans, module) + delete(s.refused, module) + if gone { + delete(s.assigned, module) + return append(providers, module) + } + s.assigned[module] = true + m, p, err := planOf(c.Node, key, e.Value()) + switch { + case errors.Is(err, errNoPlan): + case err != nil: + s.refused[module] = err.Error() + logf("[provisioner-client] %v", err) + default: + s.plans[m] = p + } + return append(providers, module) + case ConfigBucket: + rest := strings.TrimPrefix(key, c.Node+".") + module, name, ok := strings.Cut(rest, ".") + if !ok { + return nil + } + if s.config[module] == nil { + s.config[module] = map[string]json.RawMessage{} + } + var r resolved + if gone || json.Unmarshal(e.Value(), &r) != nil || len(r.Value) == 0 { + delete(s.config[module], name) + } else { + s.config[module][name] = r.Value + } + return []string{module} + case NeedsBucket: + if gone { + delete(s.needs, key) + } else { + s.needs[key] = e.Value() + } + case AppliedBucket: + module := strings.TrimPrefix(key, c.Node+".") + s.applied[module] = !gone + case CollectBucket: + collection, contributor, ok := itemOfKey(c.Node, key) + if !ok { + logf("[provisioner-client] %s in %s is not .., and is not read", key, + CollectBucket) + return nil + } + var it ItemRecord + if gone { + delete(s.recorded, key) + delete(s.items[collection], contributor) + } else if err := json.Unmarshal(e.Value(), &it); err != nil { + logf("[provisioner-client] the item %s cannot be read, so it is left out: %v", key, err) + delete(s.recorded, key) + delete(s.items[collection], contributor) + } else { + s.recorded[key] = e.Value() + if s.items[collection] == nil { + s.items[collection] = map[string]itemValue{} + } + s.items[collection][contributor] = itemValue{text: it.Text, attributes: it.Attributes} + } + // Every collector of it on this node is filled again. + var collectors []string + for _, m := range sortedKeys(s.plans) { + if s.plans[m].collects(collection) { + collectors = append(collectors, m) + } + } + return collectors + } + if strings.HasSuffix(u.bucket, boundSuffix) { + // A provider's entry for one of this machine's consumers: each module filled from that bucket again. + if s.bound[u.bucket] == nil { + s.bound[u.bucket] = map[string][]byte{} + } + if gone { + delete(s.bound[u.bucket], key) + } else { + s.bound[u.bucket][key] = e.Value() + } + return sortedKeys(s.boundUsers[u.bucket]) + } + return nil +} + +// leftModules are the modules with needs or a report kept and no assignment with a plan. +func (c *Client) leftModules(s *state) []string { + left := map[string]bool{} + for key := range s.needs { + if parts := strings.SplitN(key, ".", 3); len(parts) == 3 && !hasPlan(s, parts[1]) && s.refused[parts[1]] == "" { + left[parts[1]] = true + } + } + for m, kept := range s.applied { + if kept && !hasPlan(s, m) && s.refused[m] == "" { + left[m] = true + } + } + for key := range s.recorded { + if _, m, ok := itemOfKey(c.Node, key); ok && !hasPlan(s, m) && s.refused[m] == "" { + left[m] = true + } + } + return sortedKeys(left) +} + +// leave removes a module's needs and its report: it is no longer the provisioner's on this node. +func (c *Client) leave(ctx context.Context, kvs map[string]jetstream.KeyValue, s *state, module string) { + prefix := c.Node + "." + module + "." + removed := 0 + for _, key := range sortedKeys(s.needs) { + if strings.HasPrefix(key, prefix) { + if err := kvs[NeedsBucket].Delete(ctx, key); err != nil { + logf("[provisioner-client] the need %s could not be removed: %v", key, err) + continue + } + delete(s.needs, key) + removed++ + } + } + // Its items go: an item is in its collection only while its contributor is placed here. + removed += c.forgetItems(ctx, kvs, s, module, nil) + if !s.applied[module] && removed == 0 && s.reported[module] == nil { + return + } + if err := kvs[AppliedBucket].Delete(ctx, c.Node+"."+module); err != nil { + logf("[provisioner-client] the report of %s could not be removed: %v", module, err) + } + s.applied[module] = false + delete(s.reported, module) + c.publish(Report{Node: c.Node, Module: module, Status: StatusRemoved, At: c.now()}) + logf("[provisioner-client] %s left %s: its %d need(s) and item(s) and its report are removed", module, c.Node, + removed) +} + +// reconcile fills one module's files from the current values, and says what came of it. force writes every file +// again, whether it moved or not: the drift pass. +func (c *Client) reconcile(ctx context.Context, kvs map[string]jetstream.KeyValue, s *state, module string, + force bool) { + p := s.plans[module] + r := Report{Module: module, Templates: p.Templates.Digest} + texts, err := c.Templates.load(ctx, p.Templates) + if err != nil { + r.Status, r.Reason = StatusFailed, err.Error() + c.report(ctx, kvs, s, r) + return + } + // The needs, file by file: each template, and each file holding one secret alone. + targets, wrong := fillTargets(p, texts) + if wrong != "" { + r.Status, r.Reason = StatusFailed, wrong + c.report(ctx, kvs, s, r) + return + } + items, wrong := itemTargets(p, texts) + if wrong != "" { + r.Status, r.Reason = StatusFailed, wrong + c.report(ctx, kvs, s, r) + return + } + byFile := map[string]string{} + needs := map[Need][]string{} + var order []Need + named := func(id, text string) { + for _, n := range needsOf(text) { + if needs[n] == nil { + order = append(order, n) + } + needs[n] = append(needs[n], id) + } + } + for _, f := range targets { + byFile[f.Path] = f.text + named(f.ID, f.text) + } + for _, it := range items { + named(it.id(), it.text) + } + c.recordNeeds(ctx, kvs, s, module, order, needs) + + values := map[Need]string{} + var failed []string + for _, n := range order { + v, wait, fail := c.valueOf(ctx, s, module, p, n) + switch { + case fail != "": + failed = append(failed, fail) + case wait != "": + r.WaitsFor = append(r.WaitsFor, wait) + default: + values[n] = v + } + } + if len(failed) > 0 { + r.Status, r.Reason = StatusFailed, strings.Join(failed, "; ") + c.report(ctx, kvs, s, r) + return + } + if len(r.WaitsFor) > 0 { + // Every value or none: the files there are stay as they are. + r.Status = StatusWaiting + c.report(ctx, kvs, s, r) + return + } + // The items first: recorded, they fill the collectors, this module's own among them. + for _, it := range items { + if err := c.record(ctx, kvs, s, module, p, it.PlannedItem, render(it.text, values)); err != nil { + failed = append(failed, fmt.Sprintf("item %s: %v", it.key(), err)) + continue + } + r.Items = append(r.Items, it.key()) + } + keep := map[string]bool{} + for _, it := range p.Items { + keep[it.key()] = true + } + c.forgetItems(ctx, kvs, s, module, keep) + moved := false + for _, f := range targets { + content := []byte(render(byFile[f.Path], values)) + sum := sha256.Sum256(content) + fa := FileApplied{Path: f.Path, Takes: f.Takes} + if !force && c.written[f.Path] == sum { + fa.Action = "unchanged" + r.Files = append(r.Files, fa) + continue + } + a, err := c.Fill.Fill(ctx, f.Path, content) + if err == nil { + err = errFill(a) + } + if err != nil { + delete(c.written, f.Path) + failed = append(failed, fmt.Sprintf("%s: %v", f.Path, err)) + continue + } + c.written[f.Path] = sum + fa.Action, fa.Digest = a.Action, a.Digest + if a.Action != "unchanged" && a.Action != "kept" { + moved = true + logf("[provisioner-client] %s: %s %s (%s)", module, a.Action, f.Path, a.Digest) + } + r.Files = append(r.Files, fa) + } + if len(failed) > 0 { + r.Status, r.Reason = StatusFailed, strings.Join(failed, "; ") + } else { + r.Status = StatusApplied + } + if force && !moved && r.Status == StatusApplied { + // The drift pass found every file as it should be: nothing to say. + return + } + if force || moved { + // Said again, even when it reads the same: a file was put back. + delete(s.reported, module) + } + c.report(ctx, kvs, s, r) +} + +// valueOf is a need's value, or what it waits for, or why it cannot be filled. +func (c *Client) valueOf(ctx context.Context, s *state, module string, p Plan, n Need) (value, wait, fail string) { + answer, known := s.registry[n.Kind] + switch { + case slices.Contains(placementKinds, n.Kind): + if !known { + return "", n.String() + " (no domain answers " + n.Kind + " in the registry yet)", "" + } + v, ok := p.Placement[n.String()] + if !ok { + return "", "", n.String() + ": the controller's plan carries no value for it" + } + return v, "", "" + case n.Kind == "config": + if !known { + return "", n.String() + " (no domain answers config in the registry yet)", "" + } + if answer.Bucket != ConfigBucket { + return "", "", fmt.Sprintf("%s: the registry answers config from %q, which this client does not watch", + n, answer.Bucket) + } + v, ok := s.config[module][n.Name] + if !ok { + return "", n.String() + " (mesh-config holds no value for it on this node)", "" + } + return configText(v), "", "" + case collectionKind(n): + collection, _ := collectionOf(n) + if !known { + return "", n.String() + " (no domain answers " + n.Kind + " in the registry yet)", "" + } + if answer.Bucket != CollectBucket { + return "", "", fmt.Sprintf("%s: the registry answers %s from %q, which this client does not watch", n, + n.Kind, answer.Bucket) + } + col, ok := p.collectionNamed(collection) + if !ok { + return "", "", fmt.Sprintf("%s: the plan names no collection %s; a module is filled only from the "+ + "collections its plan says it collects", n, collection) + } + if renderer, is := environmentRenderer(n); is { + text, err := environmentText(s.items[collection], renderer) + if err != nil { + return "", "", fmt.Sprintf("%s: %v", n, err) + } + return text, "", "" + } + where, err := selectorOfNeed(n) + if err != nil { + return "", "", fmt.Sprintf("%s: %v", n, err) + } + return collectionText(s.items[collection], col.Comment, where), "", "" + case n.Kind == "secret": + if !known { + return "", n.String() + " (no domain answers secret in the registry yet)", "" + } + if answer.Ask != VaultSeat+"."+GetVerb { + return "", "", fmt.Sprintf("%s: the registry answers secret from %q, which this client does not ask", n, + answer.Ask) + } + return c.secretOf(ctx, s, module, p, n) + case n.Kind == ConsumersKind: + // A provider's consumers' credentials, each the vault's (ADR 0373). + if !known { + return "", n.String() + " (no domain answers consumers in the registry yet)", "" + } + if answer.Ask != VaultSeat+"."+GetVerb { + return "", "", fmt.Sprintf("%s: the registry answers consumers from %q, which this client does not ask", + n, answer.Ask) + } + return c.consumersOf(ctx, s, module, n) + case n.Kind == BoundKind: + // A consumer's values, in its provider's entry (ADR 0349 clause 12). + if !known { + return "", n.String() + " (no domain answers bound in the registry yet)", "" + } + return c.boundOf(s, module, p, n, answer) + } + return "", "", fmt.Sprintf("%s: this client does not fill %s placeholders yet", n, n.Kind) +} + +// recordNeeds makes the module's needs in mesh-provisioner_needs what its templates name: each need written when it +// is new or names other files, each one no longer named removed. +func (c *Client) recordNeeds(ctx context.Context, kvs map[string]jetstream.KeyValue, s *state, module string, + order []Need, files map[Need][]string) { + want := map[string][]byte{} + for _, n := range order { + v, _ := json.Marshal(NeedRecord{Node: c.Node, Module: module, Kind: n.Kind, Name: n.Name, Files: files[n]}) + want[NeedKey(c.Node, module, n)] = v + } + for _, key := range sortedKeys(want) { + if bytes.Equal(s.needs[key], want[key]) { + continue + } + if _, err := kvs[NeedsBucket].Put(ctx, key, want[key]); err != nil { + logf("[provisioner-client] the need %s could not be recorded: %v", key, err) + continue + } + s.needs[key] = want[key] + } + prefix := c.Node + "." + module + "." + for _, key := range sortedKeys(s.needs) { + if _, still := want[key]; strings.HasPrefix(key, prefix) && !still { + if err := kvs[NeedsBucket].Delete(ctx, key); err != nil { + logf("[provisioner-client] the need %s could not be removed: %v", key, err) + continue + } + delete(s.needs, key) + } + } +} + +// NeedKey is a need's key in mesh-provisioner_needs, `...`: the name with every +// character a key cannot hold, and `=`, written `=`, so `${seat:mesh-broker:4222}` is `seat.mesh-broker=3A4222`. +func NeedKey(node, module string, n Need) string { + var b strings.Builder + for _, r := range []byte(n.Name) { + switch { + case r >= 'a' && r <= 'z', r >= 'A' && r <= 'Z', r >= '0' && r <= '9', r == '-', r == '_', r == '/', r == '.': + b.WriteByte(r) + default: + fmt.Fprintf(&b, "=%02X", r) + } + } + return node + "." + module + "." + n.Kind + "." + b.String() +} + +// report says a module's report on prov.applied and keeps it in mesh-provisioner_applied, when it says something +// other than the last. +func (c *Client) report(ctx context.Context, kvs map[string]jetstream.KeyValue, s *state, r Report) { + r.Node = c.Node + undated, _ := json.Marshal(r) + if bytes.Equal(s.reported[r.Module], undated) { + return + } + r.At = c.now() + body, err := json.Marshal(r) + if err != nil { + logf("[provisioner-client] the report of %s: %v", r.Module, err) + return + } + if _, err := kvs[AppliedBucket].Put(ctx, c.Node+"."+r.Module, body); err != nil { + logf("[provisioner-client] the report of %s could not be kept: %v", r.Module, err) + } else { + s.applied[r.Module] = true + } + c.publish(r) + s.reported[r.Module] = undated + switch r.Status { + case StatusFailed: + logf("[provisioner-client] %s failed: %s", r.Module, r.Reason) + case StatusWaiting: + logf("[provisioner-client] %s waits for %s", r.Module, strings.Join(r.WaitsFor, ", ")) + } +} + +func (c *Client) publish(r Report) { + body, err := json.Marshal(r) + if err != nil { + return + } + if err := c.Conn.Publish(AppliedSubject(c.Node, r.Module), body); err != nil { + logf("[provisioner-client] prov.applied for %s could not be said: %v", r.Module, err) + } +} + +func (c *Client) now() time.Time { + if c.Now != nil { + return c.Now().UTC() + } + return time.Now().UTC() +} + +// ItemKey is an item among its module's: the collection, and `/` for a named item. +func ItemKey(collection, name string) string { + if name == "" { + return collection + } + return collection + "/" + name +} + +func sortedKeys[V any](m map[string]V) []string { + out := make([]string, 0, len(m)) + for k := range m { + out = append(out, k) + } + sort.Strings(out) + return out +} + +// fillTarget is one file the client fills for a module: a template's, or one holding a secret alone. +type fillTarget struct { + PlannedFile + text string +} + +// fillTargets is every file a plan fills, in its order: each template file, then each file of a secret of the +// module's own or of a credential it requires (issue 757), whose text is that secret and nothing else. Or why one +// cannot be filled. +func fillTargets(p Plan, texts map[string]string) ([]fillTarget, string) { + var out []fillTarget + for _, f := range p.Files { + text, ok := texts[f.Template] + if !ok { + return nil, fmt.Sprintf("%s: the templates %s hold no %s", f.ID, p.Templates.Digest, f.Template) + } + out = append(out, fillTarget{PlannedFile: f, text: text}) + } + for _, sec := range p.Secrets { + if sec.Path == "" { + continue + } + out = append(out, fillTarget{PlannedFile: PlannedFile{ID: "secret." + sec.Name, Path: sec.Path, + OwnerAtRuntime: "owned"}, text: "${secret:" + sec.Name + "}"}) + } + for _, pr := range p.Provisions { + if pr.Path == "" { + continue + } + out = append(out, fillTarget{PlannedFile: PlannedFile{ID: "secret." + pr.Name, Path: pr.Path, + OwnerAtRuntime: "owned"}, text: "${secret:" + pr.Name + "}"}) + } + return out, "" +} + +func collectionKind(n Need) bool { _, ok := collectionOf(n); return ok } + +// itemTarget is one item a module contributes, with its template's text. +type itemTarget struct { + PlannedItem + text string +} + +func (it itemTarget) id() string { return "item." + it.key() } + +// itemTargets is every item a plan contributes, with its template's text, or why one cannot be filled. An item is +// kept in a bucket, so a template naming a secret is refused: no secret is in a bucket (ADR 0349 clause 7). +func itemTargets(p Plan, texts map[string]string) ([]itemTarget, string) { + var out []itemTarget + for _, it := range p.Items { + text, ok := texts[it.Template] + if !ok { + return nil, fmt.Sprintf("item %s: the templates %s hold no %s", it.key(), p.Templates.Digest, + it.Template) + } + for _, n := range needsOf(text) { + if n.Kind == "secret" || n.Kind == ConsumersKind { + return nil, fmt.Sprintf("item %s: its template %s names %s, and an item is kept in a bucket, where no "+ + "secret is (novox/hq ADR 0349 clause 7)", it.key(), it.Template, n) + } + } + out = append(out, itemTarget{PlannedItem: it, text: text}) + } + return out, "" +} + +// record keeps one item of the module's in mesh-provisioner_collect, when it differs from what is kept. +func (c *Client) record(ctx context.Context, kvs map[string]jetstream.KeyValue, s *state, module string, p Plan, + planned PlannedItem, text string) error { + collection, id := planned.Collection, itemID(module, planned.Name) + key := CollectKey(c.Node, collection, id) + value, err := json.Marshal(ItemRecord{Node: c.Node, Collection: collection, Contributor: module, Text: text, + Name: planned.Name, Attributes: planned.Attributes, Templates: p.Templates.Digest}) + if err != nil { + return err + } + if bytes.Equal(s.recorded[key], value) { + return nil + } + if _, err := kvs[CollectBucket].Put(ctx, key, value); err != nil { + return fmt.Errorf("it could not be recorded: %w", err) + } + // Taken now, not only when the watch brings it back: the module's own templates may collect it. + s.recorded[key] = value + if s.items[collection] == nil { + s.items[collection] = map[string]itemValue{} + } + s.items[collection][id] = itemValue{text: text, attributes: planned.Attributes} + logf("[provisioner-client] %s: its item %s is recorded", module, planned.key()) + return nil +} + +// forgetItems removes the module's items of every collection keep does not name (nil: every one), and answers how +// many. +func (c *Client) forgetItems(ctx context.Context, kvs map[string]jetstream.KeyValue, s *state, module string, + keep map[string]bool) int { + removed := 0 + for _, key := range sortedKeys(s.recorded) { + collection, id, ok := itemOfKey(c.Node, key) + if owner, name := splitID(id); !ok || owner != module || keep[ItemKey(collection, name)] { + continue + } + if err := kvs[CollectBucket].Delete(ctx, key); err != nil { + logf("[provisioner-client] the item %s could not be removed: %v", key, err) + continue + } + delete(s.recorded, key) + delete(s.items[collection], id) + removed++ + } + return removed +} diff --git a/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/client_test.go b/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/client_test.go new file mode 100644 index 00000000..c4c69730 --- /dev/null +++ b/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/client_test.go @@ -0,0 +1,649 @@ +package main + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "context" + "encoding/json" + "errors" + "fmt" + "net" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "github.com/nats-io/nats-server/v2/server" + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" +) + +// The client's test list of novox/hq ADR 0349 ("How it is checked"), against a real bus (design 57 P5, issue 671): +// a template fills from each owner's keys; a missing value keeps the old file and reports what it waits for; a change +// refills, writes once and triggers the declared reaction; an owned file changed on disk is written back, a seed +// file is left, a managed file keeps the program's other keys; a module taken off the node loses its files and +// needs; a node coming back fills from current values. +// +// The bus is a real server, its accounts the client's as mesh-controller composes them (testdata/accounts.conf), so +// every read and write the client makes is one its grants allow. The node-engine is a stand-in on a real unix +// socket speaking its format (mesh-host internal/filling), writing each file as its ownership says — the +// node-engine's own tests prove its writes; these prove the client asks for the right ones — and counting each +// write that moved a file, which is what the node-engine reloads or restarts the program for. + +const node = "g14" + +// aBus is a server of this test alone with the client's buckets, the registry written as mesh-provisioner writes it. +type aBus struct { + url string + domain jetstream.JetStream + conn *nats.Conn + kv map[string]jetstream.KeyValue +} + +func startBus(t *testing.T) *aBus { + t.Helper() + accounts, err := os.ReadFile("testdata/accounts.conf") + if err != nil { + t.Fatal(err) + } + conf := filepath.Join(t.TempDir(), "bus.conf") + if err := os.WriteFile(conf, []byte("listen: 127.0.0.1:-1\njetstream { store_dir: \""+t.TempDir()+"\" }\n"+ + string(accounts)), 0o600); err != nil { + t.Fatal(err) + } + opts, err := server.ProcessConfigFile(conf) + if err != nil { + t.Fatal(err) + } + opts.NoLog, opts.NoSigs = true, true + s, err := server.NewServer(opts) + if err != nil { + t.Fatal(err) + } + go s.Start() + if !s.ReadyForConnections(10 * time.Second) { + t.Fatal("the bus did not come up") + } + t.Cleanup(s.Shutdown) + nc, err := nats.Connect(s.ClientURL(), nats.UserInfo("domain", "x")) + if err != nil { + t.Fatal(err) + } + t.Cleanup(nc.Close) + js, err := jetstream.New(nc) + if err != nil { + t.Fatal(err) + } + b := &aBus{url: s.ClientURL(), domain: js, conn: nc, kv: map[string]jetstream.KeyValue{}} + for _, name := range []string{AssignmentsBucket, ConfigBucket, RegistryBucket, NeedsBucket, AppliedBucket, + CollectBucket} { + kv, err := js.CreateKeyValue(context.Background(), jetstream.KeyValueConfig{Bucket: name}) + if err != nil { + t.Fatal(err) + } + b.kv[name] = kv + } + b.put(t, RegistryBucket, "config", `{"kind":"config","domain":"mesh-config","from":"bucket",`+ + `"bucket":"mesh-config_resolved","key":".."}`) + b.put(t, RegistryBucket, "secret", `{"kind":"secret","domain":"mesh-vault","from":"ask","ask":"mesh-vault.get"}`) + for _, kind := range append([]string{CollectedKind}, oldCollectionKinds...) { + b.put(t, RegistryBucket, kind, `{"kind":"`+kind+`","domain":"mesh-provisioner","from":"bucket",`+ + `"bucket":"mesh-provisioner_collect","key":".."}`) + } + for _, kind := range placementKinds { + b.put(t, RegistryBucket, kind, `{"kind":"`+kind+`","domain":"mesh-controller","from":"bucket",`+ + `"bucket":"mesh-controller_assignments","key":"."}`) + } + return b +} + +func (b *aBus) put(t *testing.T, bucket, key, value string) { + t.Helper() + if _, err := b.kv[bucket].PutString(context.Background(), key, value); err != nil { + t.Fatalf("writing %s in %s: %v", key, bucket, err) + } +} + +func (b *aBus) del(t *testing.T, bucket, key string) { + t.Helper() + if err := b.kv[bucket].Delete(context.Background(), key); err != nil { + t.Fatal(err) + } +} + +// config writes a resolved value as mesh-config does. +func (b *aBus) config(t *testing.T, module, key, value string) { + t.Helper() + b.put(t, ConfigBucket, node+"."+module+"."+key, fmt.Sprintf(`{"node":%q,"module":%q,"key":%q,"value":%s,`+ + `"source":"node"}`, node, module, key, value)) +} + +// assign writes a module's assignment with its plan, as the controller does. +func (b *aBus) assign(t *testing.T, module string, plan Plan) { + t.Helper() + p, err := json.Marshal(plan) + if err != nil { + t.Fatal(err) + } + b.put(t, AssignmentsBucket, node+"."+module, fmt.Sprintf(`{"node":%q,"module":%q,"assigned":"2026-10-14T20:00:00Z",`+ + `"provisioning":%s}`, node, module, p)) +} + +// keys is the bucket's keys under prefix. +func (b *aBus) keys(t *testing.T, bucket, prefix string) []string { + t.Helper() + all, err := b.kv[bucket].Keys(context.Background()) + if err != nil && !errors.Is(err, jetstream.ErrNoKeysFound) { + t.Fatal(err) + } + var out []string + for _, k := range all { + if strings.HasPrefix(k, prefix) { + out = append(out, k) + } + } + return out +} + +// reports collects what is said on prov.applied. +type reports struct { + mu sync.Mutex + all []Report +} + +func (b *aBus) listen(t *testing.T) *reports { + t.Helper() + r := &reports{} + sub, err := b.conn.Subscribe("prov.applied."+node+".>", func(m *nats.Msg) { + var rep Report + if err := json.Unmarshal(m.Data, &rep); err == nil { + r.mu.Lock() + r.all = append(r.all, rep) + r.mu.Unlock() + } + }) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = sub.Unsubscribe() }) + _ = b.conn.Flush() + return r +} + +func (r *reports) last(module string) (Report, int) { + r.mu.Lock() + defer r.mu.Unlock() + n := 0 + var out Report + for _, rep := range r.all { + if rep.Module == module { + out, n = rep, n+1 + } + } + return out, n +} + +// engine stands in for the node-engine's write socket: each path written under root, as its ownership says. +type engine struct { + root string + socket string + mu sync.Mutex + owner map[string]string // path → owned, seed or managed + moved map[string]int // writes that changed the file: what the program is reloaded for + asked map[string]int + wrote map[string]string // what it last wrote +} + +func startEngine(t *testing.T, owners map[string]string) *engine { + t.Helper() + // A unix socket's path is short: under /tmp, not the test's own directory. + dir, err := os.MkdirTemp("/tmp", "p5-") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { os.RemoveAll(dir) }) + e := &engine{root: t.TempDir(), socket: filepath.Join(dir, "fill.sock"), owner: owners, + moved: map[string]int{}, asked: map[string]int{}, wrote: map[string]string{}} + l, err := net.Listen("unix", e.socket) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { l.Close() }) + go func() { + for { + conn, err := l.Accept() + if err != nil { + return + } + go func() { + defer conn.Close() + var req FillRequest + dec := json.NewDecoder(conn) + dec.DisallowUnknownFields() + if err := dec.Decode(&req); err != nil { + _ = json.NewEncoder(conn).Encode(FillAnswer{Refused: "unreadable: " + err.Error()}) + return + } + _ = json.NewEncoder(conn).Encode(e.write(req)) + }() + } + }() + return e +} + +func (e *engine) at(path string) string { return filepath.Join(e.root, path) } + +func (e *engine) write(req FillRequest) FillAnswer { + e.mu.Lock() + defer e.mu.Unlock() + e.asked[req.Path]++ + owner, declared := e.owner[req.Path] + if !declared { + return FillAnswer{Path: req.Path, Refused: req.Path + " is not a file this node's declaration names as filled"} + } + at := e.at(req.Path) + have, err := os.ReadFile(at) + exists := err == nil + want := string(req.Content) + switch owner { + case "seed": + if exists { + return FillAnswer{Path: req.Path, Action: "kept"} + } + case "managed": + // The mesh's block between its marks; the program's lines around it kept. + const begin, end = "# >>> mesh\n", "# <<< mesh\n" + block := begin + want + end + if exists { + text := string(have) + if i, j := strings.Index(text, begin), strings.Index(text, end); i >= 0 && j > i { + block = text[:i] + block + text[j+len(end):] + } else { + block = text + block + } + } + want = block + } + action := "created" + switch { + case exists && string(have) == want: + return FillAnswer{Path: req.Path, Action: "unchanged", Digest: digestOf(have)} + case exists && string(have) != e.wrote[req.Path]: + action = "corrected" + case exists: + action = "updated" + } + _ = os.MkdirAll(filepath.Dir(at), 0o755) + if err := os.WriteFile(at, []byte(want), 0o644); err != nil { + return FillAnswer{Path: req.Path, Failed: err.Error()} + } + e.wrote[req.Path] = want + e.moved[req.Path]++ + return FillAnswer{Path: req.Path, Action: action, Digest: digestOf([]byte(want))} +} + +func (e *engine) counts(path string) (moved, asked int) { + e.mu.Lock() + defer e.mu.Unlock() + return e.moved[path], e.asked[path] +} + +func (e *engine) read(t *testing.T, path string) string { + t.Helper() + b, err := os.ReadFile(e.at(path)) + if err != nil { + return "" + } + return string(b) +} + +// templates is an artifact store holding one archive per name, and the plan's Templates for it. +func templates(t *testing.T, files map[string]string) (Templates, *httptest.Server) { + t.Helper() + var buf bytes.Buffer + z := gzip.NewWriter(&buf) + w := tar.NewWriter(z) + for _, name := range sortedKeys(files) { + if err := w.WriteHeader(&tar.Header{Name: name, Mode: 0o644, Size: int64(len(files[name])), + Typeflag: tar.TypeReg}); err != nil { + t.Fatal(err) + } + if _, err := w.Write([]byte(files[name])); err != nil { + t.Fatal(err) + } + } + if err := w.Close(); err != nil { + t.Fatal(err) + } + if err := z.Close(); err != nil { + t.Fatal(err) + } + body := buf.Bytes() + srv := httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, _ *http.Request) { _, _ = rw.Write(body) })) + t.Cleanup(srv.Close) + return Templates{Source: srv.URL + "/files.tar.gz", Digest: digestOf(body)}, srv +} + +// aClient runs a client on g14's account until the returned stop is called. +func aClient(t *testing.T, b *aBus, e *engine, state string, drift time.Duration) (*Client, func()) { + t.Helper() + nc, err := nats.Connect(b.url, nats.UserInfo("provisioner."+node, "x"), + nats.CustomInboxPrefix("_INBOX.provisioner."+node), + nats.ErrorHandler(func(_ *nats.Conn, _ *nats.Subscription, err error) { + t.Errorf("the bus refused the client: %v", err) + })) + if err != nil { + t.Fatal(err) + } + js, err := jetstream.New(nc) + if err != nil { + t.Fatal(err) + } + secrets, err := NewVaultSecrets(nc, "provisioner."+node) + if err != nil { + t.Fatal(err) + } + secrets.Within = 2 * time.Second + c := &Client{Node: node, JS: js, Conn: nc, Drift: drift, Secrets: secrets, + Fill: SocketFiller{Socket: e.socket, Busy: 10 * time.Millisecond, Tries: 3}, + Templates: &templateStore{dir: state, fetch: HTTPFetch}} + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan error, 1) + go func() { done <- c.Run(ctx) }() + stopped := false + stop := func() { + if stopped { + return + } + stopped = true + cancel() + if err := <-done; err != nil && !errors.Is(err, context.Canceled) { + t.Errorf("the client stopped with %v", err) + } + nc.Close() + } + t.Cleanup(stop) + return c, stop +} + +func eventually(t *testing.T, what string, ok func() bool) { + t.Helper() + deadline := time.Now().Add(10 * time.Second) + for !ok() { + if time.Now().After(deadline) { + t.Fatalf("not within 10s: %s", what) + } + time.Sleep(20 * time.Millisecond) + } +} + +const dunstrc = "/home/nox/.config/dunst/dunstrc" + +// dunst is a module with one owned file naming a config value and a placement value. +func dunst(t *testing.T) Plan { + tpl, _ := templates(t, map[string]string{ + "dunstrc": "[global]\nfont = Monospace ${config:font-size}\nicons = ${machine:account-home}/.icons\n"}) + return Plan{Templates: tpl, Files: []PlannedFile{{ID: "dunstrc", Path: dunstrc, Template: "dunstrc", + OwnerAtRuntime: "owned", Takes: "reload-on"}}, + Placement: map[string]string{"machine:account-home": "/home/nox"}} +} + +func TestNatsATemplateFillsFromEachOwnersKeys(t *testing.T) { + b := startBus(t) + heard := b.listen(t) + e := startEngine(t, map[string]string{dunstrc: "owned"}) + b.config(t, "dunst", "font-size", "13") + b.assign(t, "dunst", dunst(t)) + aClient(t, b, e, t.TempDir(), time.Hour) + + want := "[global]\nfont = Monospace 13\nicons = /home/nox/.icons\n" + eventually(t, "dunstrc is filled", func() bool { return e.read(t, dunstrc) == want }) + eventually(t, "prov.applied says applied", func() bool { + r, _ := heard.last("dunst") + return r.Status == StatusApplied + }) + r, _ := heard.last("dunst") + if r.Templates == "" || len(r.Files) != 1 || r.Files[0].Action != "created" || r.Files[0].Takes != "reload-on" { + t.Errorf("prov.applied said %+v", r) + } + // The needs, one per placeholder, under the node's and the module's name. + eventually(t, "the needs are recorded", func() bool { + got := b.keys(t, NeedsBucket, node+".dunst.") + return len(got) == 2 && got[0] == "g14.dunst.config.font-size" && got[1] == "g14.dunst.machine.account-home" + }) + kept, err := b.kv[AppliedBucket].Get(context.Background(), node+".dunst") + if err != nil || !strings.Contains(string(kept.Value()), `"status":"applied"`) { + t.Errorf("mesh-provisioner_applied keeps %v (%v)", kept, err) + } +} + +func TestNatsAMissingValueKeepsTheOldFileAndSaysWhatItWaitsFor(t *testing.T) { + b := startBus(t) + heard := b.listen(t) + e := startEngine(t, map[string]string{dunstrc: "owned"}) + b.assign(t, "dunst", dunst(t)) + aClient(t, b, e, t.TempDir(), time.Hour) + + // No value yet: nothing written, and the wait said. + eventually(t, "prov.applied says it waits", func() bool { + r, _ := heard.last("dunst") + return r.Status == StatusWaiting + }) + r, _ := heard.last("dunst") + if len(r.WaitsFor) != 1 || !strings.HasPrefix(r.WaitsFor[0], "config:font-size") { + t.Errorf("it waits for %v", r.WaitsFor) + } + if _, asked := e.counts(dunstrc); asked != 0 { + t.Errorf("a file waiting for a value was written %d times", asked) + } + // Filled once it is there; and a value gone keeps the file there is. + b.config(t, "dunst", "font-size", "13") + eventually(t, "dunstrc is filled", func() bool { return strings.Contains(e.read(t, dunstrc), "Monospace 13") }) + b.del(t, ConfigBucket, node+".dunst.font-size") + eventually(t, "prov.applied says it waits again", func() bool { + r, _ := heard.last("dunst") + return r.Status == StatusWaiting + }) + if !strings.Contains(e.read(t, dunstrc), "Monospace 13") { + t.Errorf("a value gone changed the file there was: %q", e.read(t, dunstrc)) + } + if _, asked := e.counts(dunstrc); asked != 1 { + t.Errorf("dunstrc was asked for %d times, want 1", asked) + } +} + +func TestNatsAChangeRefillsWritesOnceAndTheProgramTakesIt(t *testing.T) { + b := startBus(t) + heard := b.listen(t) + e := startEngine(t, map[string]string{dunstrc: "owned"}) + b.config(t, "dunst", "font-size", "13") + b.assign(t, "dunst", dunst(t)) + aClient(t, b, e, t.TempDir(), time.Hour) + eventually(t, "dunstrc is filled", func() bool { return strings.Contains(e.read(t, dunstrc), "Monospace 13") }) + + b.config(t, "dunst", "font-size", "15") + eventually(t, "dunstrc takes the change", func() bool { return strings.Contains(e.read(t, dunstrc), "Monospace 15") }) + eventually(t, "prov.applied says it", func() bool { + r, _ := heard.last("dunst") + return r.Status == StatusApplied && len(r.Files) == 1 && r.Files[0].Action == "updated" + }) + // The same value again, and a key the template does not name: nothing written. + b.config(t, "dunst", "font-size", "15") + b.config(t, "dunst", "geometry", `"300x5"`) + time.Sleep(300 * time.Millisecond) + if moved, asked := e.counts(dunstrc); moved != 2 || asked != 2 { + t.Errorf("two values wrote dunstrc %d times (asked %d); want 2 and 2: the program is reloaded once a change", + moved, asked) + } +} + +func TestNatsAnOwnedFileIsWrittenBackASeedLeftAndAManagedFileKeepsTheProgramsLines(t *testing.T) { + const seed, managed = "/home/nox/.config/app/first.conf", "/home/nox/.config/app/settings.ini" + b := startBus(t) + heard := b.listen(t) + e := startEngine(t, map[string]string{dunstrc: "owned", seed: "seed", managed: "managed"}) + tpl, _ := templates(t, map[string]string{"dunstrc": "font = ${config:font-size}\n", + "first.conf": "start = ${config:font-size}\n", "settings.ini": "size = ${config:font-size}\n"}) + b.config(t, "app", "font-size", "13") + b.assign(t, "app", Plan{Templates: tpl, Files: []PlannedFile{ + {ID: "rc", Path: dunstrc, Template: "dunstrc", OwnerAtRuntime: "owned"}, + {ID: "first", Path: seed, Template: "first.conf", OwnerAtRuntime: "seed"}, + {ID: "settings", Path: managed, Template: "settings.ini", OwnerAtRuntime: "managed", Takes: "read-on-use"}}}) + aClient(t, b, e, t.TempDir(), 200*time.Millisecond) + eventually(t, "all three are placed", func() bool { + r, _ := heard.last("app") + return r.Status == StatusApplied + }) + + // The program changes each on disk. + for path, text := range map[string]string{dunstrc: "font = 99\n", seed: "start = mine\n", + managed: "theme = dark\n" + e.read(t, managed) + "recent = a,b\n"} { + if err := os.WriteFile(e.at(path), []byte(text), 0o644); err != nil { + t.Fatal(err) + } + } + eventually(t, "the owned file is written back", func() bool { return e.read(t, dunstrc) == "font = 13\n" }) + eventually(t, "prov.applied says it was put back", func() bool { + r, _ := heard.last("app") + return r.Status == StatusApplied && len(r.Files) == 3 && r.Files[0].Action == "corrected" + }) + time.Sleep(500 * time.Millisecond) + if got := e.read(t, seed); got != "start = mine\n" { + t.Errorf("the seed file is %q: written once, then the program's", got) + } + if got := e.read(t, managed); !strings.Contains(got, "theme = dark\n") || !strings.Contains(got, "recent = a,b\n") || + !strings.Contains(got, "size = 13\n") { + t.Errorf("the managed file is %q: the mesh's block and the program's own lines", got) + } + // The drift pass that finds every file as it should be says nothing. + _, before := heard.last("app") + time.Sleep(600 * time.Millisecond) + if _, after := heard.last("app"); after != before { + t.Errorf("drift passes that moved nothing said %d reports", after-before) + } +} + +func TestNatsAModuleTakenOffTheNodeLosesItsNeedsAndItsReport(t *testing.T) { + b := startBus(t) + heard := b.listen(t) + e := startEngine(t, map[string]string{dunstrc: "owned"}) + b.config(t, "dunst", "font-size", "13") + b.assign(t, "dunst", dunst(t)) + aClient(t, b, e, t.TempDir(), time.Hour) + eventually(t, "the needs are recorded", func() bool { return len(b.keys(t, NeedsBucket, node+".dunst.")) == 2 }) + + b.del(t, AssignmentsBucket, node+".dunst") + eventually(t, "its needs go", func() bool { return len(b.keys(t, NeedsBucket, node+".dunst.")) == 0 }) + eventually(t, "its report goes", func() bool { return len(b.keys(t, AppliedBucket, node+".dunst")) == 0 }) + eventually(t, "prov.applied says it is removed", func() bool { + r, _ := heard.last("dunst") + return r.Status == StatusRemoved + }) +} + +func TestNatsANodeComingBackFillsFromTheCurrentValues(t *testing.T) { + b := startBus(t) + const zshrc = "/home/nox/.zshrc" + e := startEngine(t, map[string]string{dunstrc: "owned", zshrc: "owned"}) + b.config(t, "dunst", "font-size", "13") + b.assign(t, "dunst", dunst(t)) + b.config(t, "zsh", "prompt", `"%"`) + tpl, _ := templates(t, map[string]string{"zshrc": "PS1=${config:prompt}\n"}) + b.assign(t, "zsh", Plan{Templates: tpl, Files: []PlannedFile{{ID: "zshrc", Path: zshrc, Template: "zshrc", + OwnerAtRuntime: "owned"}}}) + state := t.TempDir() + _, stop := aClient(t, b, e, state, time.Hour) + eventually(t, "both are filled", func() bool { + return strings.Contains(e.read(t, dunstrc), "Monospace 13") && e.read(t, zshrc) == "PS1=%\n" + }) + + // Away: a value changes and a module leaves, and the artifact store is gone. + stop() + b.config(t, "dunst", "font-size", "17") + b.del(t, AssignmentsBucket, node+".zsh") + + heard := b.listen(t) + aClient(t, b, e, state, time.Hour) + eventually(t, "dunstrc has the value set while it was away", func() bool { + return strings.Contains(e.read(t, dunstrc), "Monospace 17") + }) + eventually(t, "the module that left lost its needs", func() bool { return len(b.keys(t, NeedsBucket, node+".zsh.")) == 0 }) + eventually(t, "and its report", func() bool { + r, _ := heard.last("zsh") + return r.Status == StatusRemoved && len(b.keys(t, AppliedBucket, node+".zsh")) == 0 + }) +} + +// A plan that cannot be acted on, templates that are not the ones named, and a secret the plan does not name each +// fail, with the reason; nothing is written. +func TestNatsWhatCannotBeFilledFailsWithTheReason(t *testing.T) { + b := startBus(t) + heard := b.listen(t) + e := startEngine(t, map[string]string{dunstrc: "owned", "/etc/app.conf": "owned"}) + aClient(t, b, e, t.TempDir(), time.Hour) + + b.put(t, AssignmentsBucket, node+".broken", `{"node":"g14","module":"broken","provisioning":{"templates":`+ + `{"source":"x","digest":"sha256:00"},"files":[],"surprise":1}}`) + p := dunst(t) + p.Templates.Digest = "sha256:" + strings.Repeat("0", 64) + b.assign(t, "forged", p) + tpl, _ := templates(t, map[string]string{"app.conf": "token = ${secret:app-token}\n"}) + b.assign(t, "secretive", Plan{Templates: tpl, Files: []PlannedFile{{ID: "conf", Path: "/etc/app.conf", + Template: "app.conf", OwnerAtRuntime: "owned"}}}) + + for module, why := range map[string]string{"broken": "surprise", "forged": "and the plan names", + "secretive": "the plan names no secret app-token"} { + eventually(t, module+" fails", func() bool { + r, _ := heard.last(module) + return r.Status == StatusFailed && strings.Contains(r.Reason, why) + }) + } + if _, asked := e.counts("/etc/app.conf"); asked != 0 { + t.Errorf("a file that cannot be filled was written") + } +} + +func TestANeedsKeyHoldsAnyName(t *testing.T) { + for n, want := range map[Need]string{ + {Kind: "config", Name: "font-size"}: "g14.dunst.config.font-size", + {Kind: "seat", Name: "mesh-broker:4222"}: "g14.dunst.seat.mesh-broker=3A4222", + {Kind: "config", Name: "a=b c"}: "g14.dunst.config.a=3Db=20c", + {Kind: "machine", Name: "account-home"}: "g14.dunst.machine.account-home", + } { + if got := NeedKey(node, "dunst", n); got != want { + t.Errorf("%v: %s, want %s", n, got, want) + } + } +} + +// TestNatsResticsFilesFillFromTheBackupPlacement is the backup lines as placement (issue 794): restic's two files +// are the plan's `backup:lines` and `backup:data`, each exactly the text the controller composed, trailing newline +// and all, and a changed plan refills them. +func TestNatsResticsFilesFillFromTheBackupPlacement(t *testing.T) { + const backups, data = "/var/lib/restic/backups.conf", "/var/lib/restic/data.conf" + b := startBus(t) + e := startEngine(t, map[string]string{backups: "owned", data: "owned"}) + tpl, _ := templates(t, map[string]string{ + "backups.conf": "# What the modules on this machine back up, composed by the mesh. Do not edit.\n${backup:lines}", + "data.conf": "# The data the modules on this machine declare, composed by the mesh. Do not edit.\n${backup:data}"}) + plan := func(lines, declared string) Plan { + return Plan{Templates: tpl, Files: []PlannedFile{ + {ID: "declared", Path: backups, Template: "backups.conf", OwnerAtRuntime: "owned"}, + {ID: "data-declared", Path: data, Template: "data.conf", OwnerAtRuntime: "owned"}}, + Placement: map[string]string{"backup:lines": lines, "backup:data": declared}} + } + b.assign(t, "restic", plan("# a\n/srv/a\n# b\n/srv/b\n", "# a\nstate\n")) + aClient(t, b, e, t.TempDir(), time.Hour) + head := "# What the modules on this machine back up, composed by the mesh. Do not edit.\n" + eventually(t, "backups.conf is filled", func() bool { return e.read(t, backups) == head+"# a\n/srv/a\n# b\n/srv/b\n" }) + eventually(t, "data.conf is filled", func() bool { + return e.read(t, data) == "# The data the modules on this machine declare, composed by the mesh. Do not edit.\n# a\nstate\n" + }) + b.assign(t, "restic", plan("# a\n/srv/a\n", "# a\nstate\n")) + eventually(t, "a module taken away refills backups.conf", func() bool { return e.read(t, backups) == head+"# a\n/srv/a\n" }) +} diff --git a/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/collect.go b/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/collect.go new file mode 100644 index 00000000..b197e070 --- /dev/null +++ b/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/collect.go @@ -0,0 +1,256 @@ +package main + +import ( + "cmp" + "fmt" + "slices" + "strconv" + "strings" +) + +// Collections (novox/hq ADR 0349 clause 13, design 57 L1, issue 684): a provision with many providers. A collector, +// a module taking additions from others, names a collection in its template; each contributor ships its item as a +// template of its own. The client of a node fills each contributor's item placed there, from the same owners as a +// file, and records it in mesh-provisioner_collect at `..`; it fills a collector's +// template from every item of the collection recorded for its node, in the contributors' order, each introduced by +// a line naming its contributor. An item goes when its contributor leaves the node, and the collector is filled +// again without it. +// +// **The three old namespaces read as collections.** `${shell::}`, `${environment:}` and +// `${contribution::}`, which the controller fills for a module not handed to the provisioner, name in a +// handed collector's template the collection of the same name: `${shell:zsh:normal}` is `shell:zsh:normal`, the +// collection a contributor's item for zsh's normal slot is recorded in. + +// CollectBucket keeps the items of every collection, `..`. +const CollectBucket = "mesh-provisioner_collect" + +// CollectedKind is the namespace of a collection's placeholder, `${collected:}`. +const CollectedKind = "collected" + +// oldCollectionKinds are the namespaces read as collections of their own name. +var oldCollectionKinds = []string{"shell", "environment", "contribution"} + +// collectionOf is the collection a need names, without its selector, and whether it names one. +func collectionOf(n Need) (string, bool) { + name := "" + if _, is := environmentRenderer(n); is { + return EnvironmentCollection, true + } + switch { + case n.Kind == CollectedKind: + name = n.Name + case slices.Contains(oldCollectionKinds, n.Kind): + name = n.Kind + ":" + n.Name + default: + return "", false + } + collection, _ := splitSelector(name) + return collection, true +} + +// selectorOfNeed is the attributes a need's selector asks for (`${collected:c:bar=top,place=status}`), none for a +// need without one, or why the selector is none (design 57 L3, issue 777). +func selectorOfNeed(n Need) (map[string]string, error) { + name := n.Name + if n.Kind != CollectedKind { + name = n.Kind + ":" + n.Name + } + _, selector := splitSelector(name) + out := map[string]string{} + if selector == "" { + return out, nil + } + for _, pair := range strings.Split(selector, ",") { + k, v, ok := strings.Cut(pair, "=") + switch { + case !ok || !attributeKeyShape.MatchString(k) || !attributeValueShape.MatchString(v): + return nil, fmt.Errorf("%q in %q is not =", pair, selector) + case out[k] != "": + return nil, fmt.Errorf("%q names %s twice", selector, k) + } + out[k] = v + } + return out, nil +} + +// splitSelector is a placeholder's name without its selector, and the selector: `blocks:bar=top,place=left` is +// `blocks` and `bar=top,place=left`. A collection's name holds no `=`, so the selector begins at the last colon +// before the first one (mesh-controller internal/catalogue collections.go). +func splitSelector(name string) (collection, selector string) { + eq := strings.Index(name, "=") + if eq < 0 { + return name, "" + } + colon := strings.LastIndex(name[:eq], ":") + if colon < 0 { + return name, "" + } + return name[:colon], name[colon+1:] +} + +// ItemRecord is one contributor's item, as mesh-provisioner_collect keeps it: filled, never holding a secret. +type ItemRecord struct { + Node string `json:"node"` + Collection string `json:"collection"` + Contributor string `json:"contributor"` + Text string `json:"text"` + // Name tells the item from the contributor's others of the collection; Attributes are what a placeholder + // selecting by them is matched against, `order` among them (design 57 L3, issue 777). + Name string `json:"name,omitempty"` + Attributes map[string]string `json:"attributes,omitempty"` + // Templates is the digest of the contributor's templates it was filled from. + Templates string `json:"templates,omitempty"` +} + +// itemID is an item's name among its contributor's: the module, and `/` for a named item. It is the +// contributor part of the item's key. +func itemID(module, name string) string { + if name == "" { + return module + } + return module + "/" + name +} + +// splitID is an item's module and name. +func splitID(id string) (module, name string) { + module, name, _ = strings.Cut(id, "/") + return module, name +} + +// itemValue is an item as the client holds it: its filled text and its attributes. +type itemValue struct { + text string + attributes map[string]string +} + +// order is the item's place among the items of one placeholder: its `order` attribute (0 to 99), else 50. +func (v itemValue) order() int { + if n, err := strconv.Atoi(v.attributes[OrderAttribute]); err == nil && n >= 0 && n <= 99 { + return n + } + return OrderDefault +} + +// OrderAttribute orders the items of a placeholder, lowest first; OrderDefault is the order of an item with none. +const ( + OrderAttribute = "order" + OrderDefault = 50 +) + +// CollectKey is an item's key, `..`: the collection with every character a key's +// token cannot hold written `=`, so `shell:zsh:normal` is `shell=3Azsh=3Anormal`. +func CollectKey(node, collection, contributor string) string { + return node + "." + keyToken(collection) + "." + contributor +} + +func keyToken(s string) string { + var b strings.Builder + for _, r := range []byte(s) { + switch { + case r >= 'a' && r <= 'z', r >= 'A' && r <= 'Z', r >= '0' && r <= '9', r == '-', r == '_': + b.WriteByte(r) + default: + fmt.Fprintf(&b, "=%02X", r) + } + } + return b.String() +} + +// fromKeyToken reads a token keyToken wrote. +func fromKeyToken(s string) (string, bool) { + var b strings.Builder + for i := 0; i < len(s); i++ { + if s[i] != '=' { + b.WriteByte(s[i]) + continue + } + if i+2 >= len(s) { + return "", false + } + v, err := strconv.ParseUint(s[i+1:i+3], 16, 8) + if err != nil { + return "", false + } + b.WriteByte(byte(v)) + i += 2 + } + return b.String(), true +} + +// itemOfKey is an item key's collection and contributor, for this node; false for a key not in the shape. +func itemOfKey(node, key string) (collection, contributor string, ok bool) { + rest, mine := strings.CutPrefix(key, node+".") + if !mine { + return "", "", false + } + token, contributor, two := strings.Cut(rest, ".") + if !two || token == "" || contributor == "" || strings.Contains(contributor, ".") { + return "", "", false + } + if module, name := splitID(contributor); module == "" || strings.Contains(name, "/") || + (name == "" && strings.Contains(contributor, "/")) { + return "", "", false + } + collection, ok = fromKeyToken(token) + return collection, contributor, ok && collectionShape.MatchString(collection) +} + +// collectionText is a collection's value on this node: every item of it the selector takes (all without one), by +// order, then contributor, then item name, each introduced by a line naming its contributor in the collector's +// comment (none without one), each ending its line. Empty when nothing is contributed: a collection with no item is +// filled, not waited for. +func collectionText(items map[string]itemValue, comment string, where map[string]string) string { + var ids []string + for id, it := range items { + if matches(it, where) { + ids = append(ids, id) + } + } + slices.SortFunc(ids, func(a, b string) int { + if c := cmp.Compare(items[a].order(), items[b].order()); c != 0 { + return c + } + ma, na := splitID(a) + mb, nb := splitID(b) + return cmp.Or(cmp.Compare(ma, mb), cmp.Compare(na, nb)) + }) + var b strings.Builder + for _, id := range ids { + if comment != "" { + module, _ := splitID(id) + fmt.Fprintf(&b, "%s %s\n", comment, module) + } + text := items[id].text + b.WriteString(text) + if !strings.HasSuffix(text, "\n") { + b.WriteString("\n") + } + } + return b.String() +} + +// matches is whether the item carries every attribute the selector asks for. +func matches(it itemValue, where map[string]string) bool { + for k, v := range where { + if got, has := it.attributes[k]; !has || got != v { + return false + } + } + return true +} + +// collectionNamed is the plan's collection of that name. +func (p Plan) collectionNamed(name string) (PlannedCollection, bool) { + for _, c := range p.Collections { + if c.Name == name { + return c, true + } + } + return PlannedCollection{}, false +} + +// collects is whether the plan collects that collection. +func (p Plan) collects(name string) bool { + _, ok := p.collectionNamed(name) + return ok +} diff --git a/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/collect_select_test.go b/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/collect_select_test.go new file mode 100644 index 00000000..3ffe11b6 --- /dev/null +++ b/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/collect_select_test.go @@ -0,0 +1,131 @@ +package main + +import ( + "strings" + "testing" + "time" +) + +// An item may carry attributes and a module may give several named items to one collection; a placeholder selecting +// by attributes takes only the items that carry them, in the order of the old forms: order, contributor, item name +// (design 57 L3, issue 777). Against a real bus, on the client's own account. + +const barsConf = "/etc/bars.conf" + +// barCollector fills one file from the bottom bar's status blocks, the top bar's, and all of them. +func barCollector(t *testing.T) Plan { + tpl, _ := templates(t, map[string]string{"bars": "[bottom]\n${collected:blocks:bar=bottom,place=status}" + + "[top]\n${collected:blocks:bar=top,place=status}[all]\n${collected:blocks}"}) + return Plan{Templates: tpl, Files: []PlannedFile{{ID: "conf", Path: barsConf, Template: "bars", + OwnerAtRuntime: "owned"}}, Collections: []PlannedCollection{{Name: "blocks", Comment: "#"}}} +} + +func blocksOf(t *testing.T, items map[string]PlannedItem, texts map[string]string) Plan { + tpl, _ := templates(t, texts) + p := Plan{Templates: tpl} + for _, name := range sortedKeys(items) { + p.Items = append(p.Items, items[name]) + } + return p +} + +func TestNatsASelectingPlaceholderTakesOnlyTheItemsCarryingItsAttributesInOrder(t *testing.T) { + b := startBus(t) + e := startEngine(t, map[string]string{barsConf: "owned"}) + b.assign(t, "bars", barCollector(t)) + b.assign(t, "power", blocksOf(t, map[string]PlannedItem{ + "battery": {Collection: "blocks", Name: "battery", Template: "battery", + Attributes: map[string]string{"bar": "bottom", "place": "status"}}, + "cpu": {Collection: "blocks", Name: "cpu", Template: "cpu", + Attributes: map[string]string{"bar": "bottom", "place": "status", "order": "20"}}, + "clock": {Collection: "blocks", Name: "clock", Template: "clock", + Attributes: map[string]string{"bar": "top", "place": "status"}}, + }, map[string]string{"battery": "battery\n", "cpu": "cpu\n", "clock": "clock\n"})) + b.assign(t, "another", blocksOf(t, map[string]PlannedItem{ + "disk": {Collection: "blocks", Template: "disk", Attributes: map[string]string{"bar": "bottom", "place": "status", "order": "20"}}, + }, map[string]string{"disk": "disk\n"})) + aClient(t, b, e, t.TempDir(), time.Hour) + + want := "[bottom]\n# another\ndisk\n# power\ncpu\n# power\nbattery\n" + + "[top]\n# power\nclock\n" + + "[all]\n# another\ndisk\n# power\ncpu\n# power\nbattery\n# power\nclock\n" + eventually(t, "the file holds each placeholder's items", func() bool { return e.read(t, barsConf) == want }) + eventually(t, "the named items are recorded under their module and name", func() bool { + got := b.keys(t, CollectBucket, node+".blocks.") + return len(got) == 4 && got[0] == "g14.blocks.another" && got[1] == "g14.blocks.power/battery" && + got[2] == "g14.blocks.power/clock" && got[3] == "g14.blocks.power/cpu" + }) + + // An item no longer contributed goes, and only that one. + b.assign(t, "power", blocksOf(t, map[string]PlannedItem{ + "clock": {Collection: "blocks", Name: "clock", Template: "clock", Attributes: map[string]string{"bar": "top", "place": "status"}}, + }, map[string]string{"clock": "clock\n"})) + eventually(t, "the items of the module's others go", func() bool { + return strings.Contains(e.read(t, barsConf), "[bottom]\n# another\ndisk\n[top]") + }) + if got := b.keys(t, CollectBucket, node+".blocks."); len(got) != 2 { + t.Errorf("kept %v", got) + } +} + +func TestASelectorIsReadFromAPlaceholderOfAnyNamespace(t *testing.T) { + for n, want := range map[Need]string{ + {Kind: "collected", Name: "blocks:bar=top,place=status"}: "blocks", + {Kind: "collected", Name: "blocks"}: "blocks", + {Kind: "collected", Name: "a:b:k=v"}: "a:b", + {Kind: "contribution", Name: "node-bar:block:bar=top,place=left"}: "contribution:node-bar:block", + } { + if got, ok := collectionOf(n); !ok || got != want { + t.Errorf("%s is the collection %q (%v), want %q", n, got, ok, want) + } + } + if where, err := selectorOfNeed(Need{Kind: "collected", Name: "blocks:bar=top,place=status"}); err != nil || + len(where) != 2 || where["bar"] != "top" || where["place"] != "status" { + t.Errorf("the selector is %v (%v)", where, err) + } + for _, bad := range []string{"blocks:bar=top,bar=top", "blocks:bar=", "blocks:bar=top,"} { + if _, err := selectorOfNeed(Need{Kind: "collected", Name: bad}); err == nil { + t.Errorf("%s was read as a selector", bad) + } + } +} + +func TestAnItemsKeyHoldsItsName(t *testing.T) { + key := CollectKey(node, "blocks", itemID("power", "battery")) + if key != "g14.blocks.power/battery" { + t.Fatal(key) + } + if c, id, ok := itemOfKey(node, key); !ok || c != "blocks" || id != "power/battery" { + t.Errorf("reads back as %q %q %v", c, id, ok) + } + for _, bad := range []string{"g14.blocks.power/", "g14.blocks./x", "g14.blocks.a/b/c"} { + if _, _, ok := itemOfKey(node, bad); ok { + t.Errorf("%s read as an item's key", bad) + } + } + p := Plan{Templates: Templates{Source: "x", Digest: "sha256:" + strings.Repeat("0", 64)}, + Items: []PlannedItem{{Collection: "a", Name: "n", Template: "i"}, {Collection: "a", Name: "n", Template: "j"}, + {Collection: "a", Name: "Big", Template: "k", Attributes: map[string]string{"order": "100", "Bar": "x"}}}} + got := strings.Join(p.problems(), "\n") + for _, want := range []string{"item a/n is named twice", `"Big" is not a name`, "0 to 99", "attribute Bar"} { + if !strings.Contains(got, want) { + t.Errorf("the problems %q do not say %q", got, want) + } + } +} + +func TestAnItemOfAPlanReadsItsNameAndAttributesAndRefusesAnUnknownField(t *testing.T) { + plan := func(item string) string { + return `{"node":"g14","module":"power","provisioning":{"templates":{"source":"x","digest":"sha256:` + + strings.Repeat("0", 64) + `"},"files":[],"items":[` + item + `]}}` + } + _, p, err := planOf("g14", "g14.power", []byte(plan( + `{"collection":"blocks","name":"battery","attributes":{"bar":"top"},"template":"battery"}`))) + if err != nil || len(p.Items) != 1 || p.Items[0].Name != "battery" || p.Items[0].Attributes["bar"] != "top" { + t.Fatalf("read %+v (%v)", p.Items, err) + } + if _, _, err := planOf("g14", "g14.power", []byte(plan( + `{"collection":"blocks","template":"battery","colour":"red"}`))); err == nil || !strings.Contains(err.Error(), "colour") { + t.Errorf("an unknown item field was accepted: %v", err) + } +} diff --git a/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/collect_test.go b/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/collect_test.go new file mode 100644 index 00000000..f58a7ca3 --- /dev/null +++ b/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/collect_test.go @@ -0,0 +1,179 @@ +package main + +import ( + "context" + "encoding/json" + "strings" + "testing" + "time" +) + +// A collection's test of novox/hq ADR 0349 ("How it is checked", design 57 L1, issue 684): zsh's file holds every +// placed contributor's item and loses one whose contributor leaves. Against a real bus, on the client's own account. + +const zshrc = "/home/nox/.zshrc" + +// zshCollector is zsh as a handed collector: its rc names its normal slot the old way, read as the collection +// shell:zsh:normal, and a collection of the new form. +func zshCollector(t *testing.T) Plan { + tpl, _ := templates(t, map[string]string{"zshrc": "# zsh\n${shell:zsh:normal}# then\n${collected:zsh-widgets}# end\n"}) + return Plan{Templates: tpl, Files: []PlannedFile{{ID: "rc", Path: zshrc, Template: "zshrc", + OwnerAtRuntime: "owned"}}, + Collections: []PlannedCollection{{Name: "shell:zsh:normal", Comment: "#"}, {Name: "zsh-widgets"}}} +} + +// contributor is a module whose only part on the node is its item of collection, from its own template. +func contributor(t *testing.T, collection, text string) Plan { + tpl, _ := templates(t, map[string]string{"item": text}) + return Plan{Templates: tpl, Items: []PlannedItem{{Collection: collection, Template: "item"}}, + Placement: map[string]string{"machine:account-home": "/home/nox"}} +} + +func TestNatsZshsFileHoldsEveryPlacedContributorsItemAndLosesOneWhoseContributorLeaves(t *testing.T) { + b := startBus(t) + heard := b.listen(t) + e := startEngine(t, map[string]string{zshrc: "owned", "/etc/fzf.conf": "owned"}) + b.assign(t, "zsh", zshCollector(t)) + b.assign(t, "powerlevel10k", contributor(t, "shell:zsh:normal", + "source ${machine:account-home}/.p10k.zsh\n")) + b.assign(t, "mesh-cli", contributor(t, "shell:zsh:normal", "alias nox=mesh-cli")) + b.config(t, "fzf", "binding", `"^T"`) + fzf := contributor(t, "zsh-widgets", "bindkey ${config:binding} fzf-file-widget\n") + b.assign(t, "fzf", fzf) + aClient(t, b, e, t.TempDir(), time.Hour) + + every := "# zsh\n# mesh-cli\nalias nox=mesh-cli\n# powerlevel10k\nsource /home/nox/.p10k.zsh\n# then\n" + + "bindkey ^T fzf-file-widget\n# end\n" + eventually(t, "zshrc holds every contributor's item", func() bool { return e.read(t, zshrc) == every }) + // Each item recorded under the node, the collection and its contributor; filled, the contributor's report says so. + eventually(t, "the items are recorded", func() bool { + got := b.keys(t, CollectBucket, node+".") + return len(got) == 3 && got[0] == "g14.shell=3Azsh=3Anormal.mesh-cli" && + got[1] == "g14.shell=3Azsh=3Anormal.powerlevel10k" && got[2] == "g14.zsh-widgets.fzf" + }) + kept, err := b.kv[CollectBucket].Get(context.Background(), "g14.shell=3Azsh=3Anormal.powerlevel10k") + if err != nil { + t.Fatal(err) + } + var it ItemRecord + if err := json.Unmarshal(kept.Value(), &it); err != nil || it.Text != "source /home/nox/.p10k.zsh\n" || + it.Collection != "shell:zsh:normal" || it.Contributor != "powerlevel10k" { + t.Errorf("the item kept is %s (%v)", kept.Value(), err) + } + eventually(t, "powerlevel10k's report says its item", func() bool { + r, _ := heard.last("powerlevel10k") + return r.Status == StatusApplied && len(r.Items) == 1 && r.Items[0] == "shell:zsh:normal" + }) + // zsh records the collections it reads as its needs, like any value. + eventually(t, "zsh's needs are recorded", func() bool { + got := b.keys(t, NeedsBucket, node+".zsh.") + return len(got) == 2 && got[0] == "g14.zsh.collected.zsh-widgets" && got[1] == "g14.zsh.shell.zsh=3Anormal" + }) + + // A contributor's own value changes: its item, and so zshrc, follow. + b.config(t, "fzf", "binding", `"^F"`) + eventually(t, "zshrc follows the item", func() bool { return strings.Contains(e.read(t, zshrc), "bindkey ^F") }) + + // powerlevel10k leaves the node: its item goes, and zshrc without it. + b.del(t, AssignmentsBucket, node+".powerlevel10k") + without := "# zsh\n# mesh-cli\nalias nox=mesh-cli\n# then\nbindkey ^F fzf-file-widget\n# end\n" + eventually(t, "zshrc loses the item of the contributor that left", func() bool { return e.read(t, zshrc) == without }) + if got := b.keys(t, CollectBucket, node+".shell=3Azsh=3Anormal.powerlevel10k"); len(got) != 0 { + t.Errorf("the item of a contributor that left is kept: %v", got) + } + // And a contributor that stops contributing, though it stays: the same. + b.assign(t, "fzf", Plan{Templates: fzf.Templates, Files: []PlannedFile{{ID: "conf", Path: "/etc/fzf.conf", + Template: "item", OwnerAtRuntime: "owned"}}, Placement: fzf.Placement}) + eventually(t, "zshrc loses an item no longer contributed", func() bool { + return e.read(t, zshrc) == "# zsh\n# mesh-cli\nalias nox=mesh-cli\n# then\n# end\n" + }) +} + +func TestNatsACollectorStartingWithItsNodeHoldsTheItemsAlreadyThereAndOnesRecordedWhileAway(t *testing.T) { + b := startBus(t) + e := startEngine(t, map[string]string{zshrc: "owned"}) + b.assign(t, "zsh", zshCollector(t)) + b.assign(t, "mesh-cli", contributor(t, "shell:zsh:normal", "alias nox=mesh-cli\n")) + state := t.TempDir() + _, stop := aClient(t, b, e, state, time.Hour) + eventually(t, "zshrc holds mesh-cli's item", func() bool { return strings.Contains(e.read(t, zshrc), "alias nox") }) + + stop() + b.del(t, AssignmentsBucket, node+".mesh-cli") + b.assign(t, "zsh-autosuggestions", contributor(t, "shell:zsh:normal", "source suggestions.zsh\n")) + aClient(t, b, e, state, time.Hour) + want := "# zsh\n# zsh-autosuggestions\nsource suggestions.zsh\n# then\n# end\n" + eventually(t, "zshrc is filled from the items now", func() bool { return e.read(t, zshrc) == want }) + eventually(t, "the item of the module that left while away is removed", func() bool { + return len(b.keys(t, CollectBucket, node+".shell=3Azsh=3Anormal.mesh-cli")) == 0 + }) +} + +// A collection the plan does not name, and an item naming a secret, fail with the reason; an item is never +// recorded with a secret in it. +func TestNatsACollectionNotPlannedAndAnItemNamingASecretFail(t *testing.T) { + b := startBus(t) + heard := b.listen(t) + e := startEngine(t, map[string]string{zshrc: "owned"}) + p := zshCollector(t) + p.Collections = p.Collections[:1] + b.assign(t, "zsh", p) + b.assign(t, "leaky", contributor(t, "shell:zsh:normal", "export TOKEN=${secret:token}\n")) + aClient(t, b, e, t.TempDir(), time.Hour) + for module, why := range map[string]string{"zsh": "the plan names no collection zsh-widgets", + "leaky": "where no secret is"} { + eventually(t, module+" fails", func() bool { + r, _ := heard.last(module) + return r.Status == StatusFailed && strings.Contains(r.Reason, why) + }) + } + if got := b.keys(t, CollectBucket, node+"."); len(got) != 0 { + t.Errorf("recorded %v", got) + } + if _, asked := e.counts(zshrc); asked != 0 { + t.Errorf("a collector naming a collection its plan does not was written") + } +} + +func TestAnItemsKeyHoldsAnyCollectionAndReadsBack(t *testing.T) { + for collection, want := range map[string]string{ + "zsh-widgets": "g14.zsh-widgets.fzf", + "shell:zsh:normal": "g14.shell=3Azsh=3Anormal.fzf", + "contribution:node-hotkeys:trigger": "g14.contribution=3Anode-hotkeys=3Atrigger.fzf", + } { + key := CollectKey(node, collection, "fzf") + if key != want { + t.Errorf("%s: %s, want %s", collection, key, want) + } + c, who, ok := itemOfKey(node, key) + if !ok || c != collection || who != "fzf" { + t.Errorf("%s reads back as %q %q %v", key, c, who, ok) + } + } + for _, bad := range []string{"g14.x", "ace.zsh-widgets.fzf", "g14.=ZZ.fzf", "g14.a.b.c", "g14.Upper.fzf"} { + if _, _, ok := itemOfKey(node, bad); ok { + t.Errorf("%s read as an item's key", bad) + } + } +} + +func TestAPlansItemsAndCollectionsAreJudged(t *testing.T) { + p := Plan{Templates: Templates{Source: "x", Digest: "sha256:" + strings.Repeat("0", 64)}, + Items: []PlannedItem{{Collection: "a", Template: "item"}, {Collection: "a", Template: "../x"}, + {Collection: "Bad", Template: "y"}}, + Collections: []PlannedCollection{{Name: "c", Comment: "#\n"}, {Name: "c"}}} + got := strings.Join(p.problems(), "\n") + for _, want := range []string{"item a is named twice", `template "../x"`, `item "Bad" is not`, + "collection c: its comment holds a line break", "collection c is named twice"} { + if !strings.Contains(got, want) { + t.Errorf("the problems %q do not say %q", got, want) + } + } + // A plan contributing an item and filling no file is one the client acts on. + if wrong := (Plan{Templates: p.Templates, Items: p.Items[:1]}).problems(); len(wrong) != 0 { + t.Errorf("a contributor's plan: %v", wrong) + } + if wrong := (Plan{Templates: p.Templates}).problems(); len(wrong) != 1 { + t.Errorf("a plan doing nothing: %v", wrong) + } +} diff --git a/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/consumers.go b/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/consumers.go new file mode 100644 index 00000000..61e29d13 --- /dev/null +++ b/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/consumers.go @@ -0,0 +1,257 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + "reflect" + "strconv" + "strings" +) + +// A provider's consumers' credentials, and a consumer's values from its provider (novox/hq ADR 0349 clause 12, ADR +// 0373, issue 679). +// +// **`${consumers:}`: each consumer bound to this provider, with its credential.** A provider in the tool +// runner may not ask the vault (ADR 0349 clause 11, ADR 0370 clause 2), so its machine's client fills its file. A +// consumer binds the provider where its plan, on any machine, names this machine and this module as the provider +// of the provision (issue 760); so the client reads every machine's plans (the whole assignments mirror, which +// carries no secret's value) for who binds, and asks the vault for each consumer's own credential, +// `pair////[/]`, which the vault hands the provider's machine while the +// provider is handed there (mesh-vault pair.go). The file lists one entry per consumer, in the order of the +// assignments' keys and, within one plan, of its provisions entries: a JSON list, each naming the consumer and its +// credential's name and carrying its value, which the provider's helper reads (mesh-sdk go/provider ConsumersFile). +// A consumer whose credential the vault does not hold yet, or will not hand, is left out and said, so the others are +// served; the provider's helper waits for it. An entry with a name is a credential the provider's software issued +// per holder (issue 757), which the provider holds already: not listed; nor is a bind whose offer names another +// credential form than a login per consumer (issue 768): a shared value is the provider's own secret, and an issued +// or given one is not the vault's to make. The file is the provider's alone: the +// catalogue holds it to mode 0600 and owned (mesh-controller internal/catalogue consumers.go). It is filled again +// when a consumer binds or unbinds, and when the vault says one of the credentials changed. +// +// **`${bound::}`: the consumer's entry in its provider's bucket.** The provider's helper writes one +// entry per consumer to its own bucket, `_bound`, keyed `..[.]` +// (mesh-sdk go/provider, issue 678), as the registry says. The client watches its own machine's keys of the bucket +// of each provider its plans name, and fills a key from the entry: `as`, `user`, or one of its served values. An +// entry the provider refused fails the module with the provider's reason; one not written yet is waited for. + +// ConsumersKind is the namespace of a provider's consumers' credentials, `${consumers:}`. +const ConsumersKind = "consumers" + +// BoundKind is the namespace of a consumer's value from its provider, `${bound::}`. +const BoundKind = "bound" + +// boundSuffix ends a provider's bucket of entries: `_bound`. +const boundSuffix = "_bound" + +// bindsOf is as much of any machine's assignment as the client reads for who binds a provider: lenient, since +// another machine's plan is that machine's client's to act on. +type bindsOf struct { + Node string `json:"node"` + Module string `json:"module"` + Provisioning *struct { + Provisions []PlannedProvision `json:"provisions"` + } `json:"provisioning"` +} + +// binding is one assignment's binds: its machine and module, and its plan's provisions in order. +type binding struct { + node, module string + provisions []PlannedProvision +} + +// takeBinds folds one assignment, of any machine, into who binds what; it answers whether that changed. +func takeBinds(s *state, key string, value []byte, gone bool) bool { + var was *binding + if b, ok := s.binds[key]; ok { + was = &b + } + if gone { + delete(s.binds, key) + return was != nil + } + var a bindsOf + if err := json.Unmarshal(value, &a); err != nil || a.Node+"."+a.Module != key { + logf("[provisioner-client] the assignment %s is not the controller's, so whom it binds is not read", key) + delete(s.binds, key) + return was != nil + } + b := binding{node: a.Node, module: a.Module} + if a.Provisioning != nil { + b.provisions = a.Provisioning.Provisions + } + s.binds[key] = b + return was == nil || was.node != b.node || was.module != b.module || !reflect.DeepEqual(was.provisions, b.provisions) +} + +// PairName is one consumer's own credential as the vault names it (mesh-vault pair.go, mesh-sdk go/provider +// PairName). +func PairName(provider, provision, node, module, local string) string { + n := "pair/" + provider + "/" + provision + "/" + node + "/" + module + if local != "" { + n += "/" + local + } + return n +} + +// consumerListed is one entry of a provider's consumers' file (mesh-sdk go/provider ConsumerCredential). +type consumerListed struct { + Consumer string `json:"consumer"` + Module string `json:"module"` + Local string `json:"local,omitempty"` + Credential string `json:"credential"` + Value string `json:"value"` +} + +// consumersOf is the file text of a provider's consumers for one provision, or why it cannot be filled. +func (c *Client) consumersOf(ctx context.Context, s *state, module string, n Need) (value, wait, fail string) { + s.providing[module] = true + list := []consumerListed{} + var left []string + for _, key := range sortedKeys(s.binds) { + b := s.binds[key] + for _, pr := range b.provisions { + if pr.issuedForHolder() || pr.Provider != c.Node || pr.ProviderModule != module || pr.Provision != n.Name || + (pr.CredentialForm != "" && pr.CredentialForm != "pair") { + continue + } + name := PairName(c.Node, n.Name, b.node, b.module, pr.Local) + v, w, f := c.vaultValue(ctx, s, n, name) + switch { + case f != "": + left = append(left, f) + continue + case w != "": + left = append(left, w) + continue + } + list = append(list, consumerListed{Consumer: b.node, Module: b.module, Local: pr.Local, Credential: name, + Value: v}) + } + } + said := strings.Join(left, "; ") + if s.leftOut[module+" "+n.Name] != said { + s.leftOut[module+" "+n.Name] = said + if said != "" { + logf("[provisioner-client] %s: consumers left out of %s until the vault hands their credential: %s", + module, n, said) + } + } + text, err := json.Marshal(list) + if err != nil { + return "", "", fmt.Sprintf("%s: %v", n, err) + } + return string(text), "", "" +} + +// boundEntry is as much of a provider's entry as a consumer's file reads (mesh-sdk go/provider Entry). +type boundEntry struct { + As string `json:"as"` + User string `json:"user"` + Values map[string]any `json:"values"` + Refused string `json:"refused"` +} + +// boundOf is a consumer's value from its provider's entry, or what it waits for, or why it cannot be filled. +func (c *Client) boundOf(s *state, module string, p Plan, n Need, answer Answer) (value, wait, fail string) { + provision, key, ok := strings.Cut(n.Name, ":") + if !ok || provision == "" || key == "" { + return "", "", n.String() + ": a bound value is ${bound::}" + } + key, fallback, defaulted := strings.Cut(key, "|default:") + var pr *PlannedProvision + for i := range p.Provisions { + q := p.Provisions[i] + if q.issuedForHolder() || q.Provision != provision { + continue + } + if pr == nil || q.Local == "" { + pr = &p.Provisions[i] + } + } + switch { + case pr == nil && defaulted: + return fallback, "", "" + case pr == nil: + return "", "", fmt.Sprintf("%s: the plan names no provider of %s for %s here", n, provision, module) + case pr.ProviderModule == "": + return "", "", fmt.Sprintf("%s: the plan names no provider module of %s", n, provision) + } + bucket := strings.Replace(answer.Bucket, "", pr.ProviderModule, 1) + if bucket != pr.ProviderModule+boundSuffix { + return "", "", fmt.Sprintf("%s: the registry answers bound from %q, which this client does not read", n, + answer.Bucket) + } + if s.boundUsers[bucket] == nil { + s.boundUsers[bucket] = map[string]bool{} + } + s.boundUsers[bucket][module] = true + if c.watchBound == nil { + return "", "", n.String() + ": this client watches no provider's bucket" + } + if err := c.watchBound(bucket); err != nil { + return "", fmt.Sprintf("%s (the provider's bucket %s cannot be watched yet: %v)", n, bucket, err), "" + } + if !s.boundReady[bucket] { + return "", fmt.Sprintf("%s (the entries of %s are not read yet)", n, bucket), "" + } + entryKey := c.Node + "." + module + "." + provision + if pr.Local != "" { + entryKey += "." + pr.Local + } + raw, ok := s.bound[bucket][entryKey] + if !ok { + return "", fmt.Sprintf("%s (%s on %s has not answered %s on %s yet)", n, pr.ProviderModule, pr.Provider, + module, c.Node), "" + } + var e boundEntry + if err := json.Unmarshal(raw, &e); err != nil { + return "", "", fmt.Sprintf("%s: the entry %s of %s is not the provider's", n, entryKey, bucket) + } + if e.Refused != "" { + return "", "", fmt.Sprintf("%s: %s on %s refused it: %s", n, pr.ProviderModule, pr.Provider, e.Refused) + } + switch key { + case "as": + if e.As != "" { + return e.As, "", "" + } + case "user": + if e.User != "" { + return e.User, "", "" + } + } + if v, ok := e.Values[key]; ok { + return plainly(v), "", "" + } + have := []string{} + if e.As != "" { + have = append(have, "as") + } + if e.User != "" { + have = append(have, "user") + } + have = append(have, sortedKeys(e.Values)...) + return "", "", fmt.Sprintf("%s: %s on %s serves %s no %q; it serves %s", n, pr.ProviderModule, pr.Provider, module, + key, strings.Join(have, ", ")) +} + +// plainly is a served value as a file takes it: a string as it is, a whole number without a fraction, anything else +// as its JSON (mesh-controller internal/catalogue plainly). +func plainly(v any) string { + switch x := v.(type) { + case string: + return x + case float64: + if x == float64(int64(x)) { + return strconv.FormatInt(int64(x), 10) + } + return strconv.FormatFloat(x, 'f', -1, 64) + case bool: + return strconv.FormatBool(x) + case nil: + return "" + } + raw, _ := json.Marshal(v) + return string(raw) +} diff --git a/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/consumers_test.go b/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/consumers_test.go new file mode 100644 index 00000000..469e4396 --- /dev/null +++ b/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/consumers_test.go @@ -0,0 +1,262 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + "strings" + "testing" + "time" + + "github.com/nats-io/nats.go/jetstream" +) + +// A provider's consumers' credentials, filled by its machine's client (novox/hq ADR 0373, issue 679), against a real +// bus whose accounts are the client's as mesh-controller composes them: a provider template with +// ${consumers:} is filled with one entry per bound consumer, in the plan's provisions order, each carrying +// the credential the vault holds for that pair; a consumer that binds or unbinds, or a credential the vault remakes, +// changes the file within one watch; no value crosses the bus in the clear. And a consumer's ${bound:…} is filled from +// its provider's entry (mesh-sdk go/provider, issue 678). + +const consumersFile = "/var/lib/postgres/consumers.json" + +// registryOf679 writes the registry's two answers this issue adds, as mesh-provisioner writes them. +func registryOf679(t *testing.T, b *aBus) { + t.Helper() + b.put(t, RegistryBucket, "consumers", `{"kind":"consumers","domain":"mesh-vault","from":"ask","ask":"mesh-vault.get"}`) + b.put(t, RegistryBucket, "bound", `{"kind":"bound","domain":"provider","from":"bucket",`+ + `"bucket":"_bound","key":"..[.]"}`) +} + +// bind writes another machine's module's assignment, whose plan binds it to providers, as the controller does. +func (b *aBus) bind(t *testing.T, at, module string, provisions ...PlannedProvision) { + t.Helper() + ps, err := json.Marshal(provisions) + if err != nil { + t.Fatal(err) + } + b.put(t, AssignmentsBucket, at+"."+module, fmt.Sprintf(`{"node":%q,"module":%q,"assigned":"2026-10-14T20:00:00Z",`+ + `"provisioning":{"templates":{"source":"x","digest":"sha256:%s"},"files":[{"id":"c","path":"/etc/c",`+ + `"template":"c","owner-at-runtime":"owned"}],"provisions":%s}}`, at, module, strings.Repeat("0", 64), ps)) +} + +func postgresPlan(t *testing.T, text string) Plan { + tpl, _ := templates(t, map[string]string{"consumers.json": text}) + return Plan{Templates: tpl, Files: []PlannedFile{{ID: "consumers", Path: consumersFile, Template: "consumers.json", + OwnerAtRuntime: "owned"}}} +} + +func fromHere(local string) PlannedProvision { + return PlannedProvision{Provision: "postgres-database", Local: local, Provider: node, ProviderModule: "postgres"} +} + +func pairOf(at, module, local string) string { + n := "pair/" + node + "/postgres-database/" + at + "/" + module + if local != "" { + n += "/" + local + } + return n +} + +type listed struct { + Consumer string `json:"consumer"` + Module string `json:"module"` + Local string `json:"local,omitempty"` + Credential string `json:"credential"` + Value string `json:"value"` +} + +func consumersIn(t *testing.T, e *engine) []listed { + t.Helper() + var out []listed + if err := json.Unmarshal([]byte(e.read(t, consumersFile)), &out); err != nil { + return nil + } + return out +} + +func sameList(have []listed, want ...listed) bool { + if len(have) != len(want) { + return false + } + for i := range want { + if have[i] != want[i] { + return false + } + } + return true +} + +func TestNatsAProvidersConsumersAreFilledFromTheVaultAndFollowEachBindAndRemake(t *testing.T) { + b := startBus(t) + registryOf679(t, b) + heard := b.listen(t) + wire := sniff(t, b) + vault := startVault(t, b) + vault.keep(pairOf("ace", "letta", ""), "letta-pw-never-on-the-bus", false) + vault.keep(pairOf("shanks", "gitea", "cache"), "gitea-pw-never-on-the-bus", false) + // Bound to this provider: ace's letta, and shanks's gitea under a local name. Not: a provision of another + // provider, another provision, and a credential its provider's software issued per holder (issue 757). + b.bind(t, "ace", "letta", fromHere(""), + PlannedProvision{Provision: "postgres-database", Local: "old", Provider: "ace", ProviderModule: "postgres"}) + b.bind(t, "shanks", "gitea", PlannedProvision{Provision: "redis", Provider: node, ProviderModule: "postgres"}, + fromHere("cache"), + PlannedProvision{Name: "token", Provision: "forge-read", Provider: node, ProviderModule: "postgres"}) + // Nor a bind whose offer names another credential form than a login per consumer (issue 768). + b.bind(t, "ace", "shared", PlannedProvision{Provision: "postgres-database", Provider: node, + ProviderModule: "postgres", CredentialForm: "shared"}) + e := startEngine(t, map[string]string{consumersFile: "owned"}) + b.assign(t, "postgres", postgresPlan(t, "${consumers:postgres-database}")) + aClient(t, b, e, t.TempDir(), time.Hour) + + letta := listed{Consumer: "ace", Module: "letta", Credential: pairOf("ace", "letta", ""), + Value: "letta-pw-never-on-the-bus"} + gitea := listed{Consumer: "shanks", Module: "gitea", Local: "cache", Credential: pairOf("shanks", "gitea", "cache"), + Value: "gitea-pw-never-on-the-bus"} + eventually(t, "the file lists each bound consumer with its credential, in the plans' order", func() bool { + return sameList(consumersIn(t, e), letta, gitea) + }) + eventually(t, "prov.applied says applied", func() bool { + r, _ := heard.last("postgres") + return r.Status == StatusApplied + }) + + // A consumer binds, on this machine: listed once the vault holds its credential; until then left out, and the + // others kept. + b.bind(t, node, "photos", fromHere("")) + time.Sleep(300 * time.Millisecond) + if !sameList(consumersIn(t, e), letta, gitea) { + t.Fatalf("a consumer whose credential the vault does not hold yet changed the file: %+v", consumersIn(t, e)) + } + vault.keep(pairOf(node, "photos", ""), "photos-pw-never-on-the-bus", true) + photos := listed{Consumer: node, Module: "photos", Credential: pairOf(node, "photos", ""), + Value: "photos-pw-never-on-the-bus"} + eventually(t, "a consumer that binds is listed when the vault holds its credential, in its assignment's order", + func() bool { return sameList(consumersIn(t, e), letta, photos, gitea) }) + // The vault remakes a credential. + vault.keep(pairOf("ace", "letta", ""), "letta-pw-remade-never-on-the-bus", true) + letta.Value = "letta-pw-remade-never-on-the-bus" + eventually(t, "a credential the vault remakes reaches the file", func() bool { + return sameList(consumersIn(t, e), letta, photos, gitea) + }) + // A consumer unbinds. + b.del(t, AssignmentsBucket, "shanks.gitea") + eventually(t, "a consumer that unbinds leaves the file", func() bool { + return sameList(consumersIn(t, e), letta, photos) + }) + // Another provider's consumer's credential changing is no reason to ask. + before := vault.times(pairOf("ace", "letta", "")) + vault.keep("pair/ace/postgres-database/ace/letta/old", "elsewhere", true) + time.Sleep(300 * time.Millisecond) + if vault.times("pair/ace/postgres-database/ace/letta/old") != 0 || vault.times(pairOf("ace", "letta", "")) != before { + t.Error("a change of another provider's consumer made the client ask") + } + + if wire.heard() == 0 { + t.Fatal("the sniffer heard nothing: it proves nothing") + } + for _, value := range []string{"letta-pw", "gitea-pw", "photos-pw"} { + if said := wire.carrying(value); len(said) > 0 { + t.Errorf("a consumer's credential crossed the bus in the clear: %q", said) + } + } + // The need is recorded as any other, by name: never a value. + if keys := b.keys(t, NeedsBucket, node+".postgres."); len(keys) != 1 || + keys[0] != node+".postgres.consumers.postgres-database" { + t.Errorf("the provider's needs are %v", keys) + } +} + +// A provider with no consumer bound is filled with an empty list, never left waiting. +func TestNatsAProviderWithNoConsumerIsFilledWithAnEmptyList(t *testing.T) { + b := startBus(t) + registryOf679(t, b) + startVault(t, b) + e := startEngine(t, map[string]string{consumersFile: "owned"}) + b.assign(t, "postgres", postgresPlan(t, "${consumers:postgres-database}")) + aClient(t, b, e, t.TempDir(), time.Hour) + eventually(t, "the file holds an empty list", func() bool { return e.read(t, consumersFile) == "[]" }) +} + +// An item is kept in a bucket, where no secret is: an item naming its consumers fails. +func TestAnItemNamingConsumersIsRefused(t *testing.T) { + p := Plan{Items: []PlannedItem{{Collection: "c", Template: "item"}}} + if _, why := itemTargets(p, map[string]string{"item": "${consumers:postgres-database}"}); !strings.Contains(why, + "no secret") { + t.Fatalf("an item naming consumers: %q", why) + } +} + +const appBound = "/etc/app/database.env" + +func appPlan(t *testing.T) Plan { + tpl, _ := templates(t, map[string]string{"database.env": "HOST=${bound:postgres-database:host}\n" + + "PORT=${bound:postgres-database:port}\nUSER=${bound:postgres-database:user}\n"}) + return Plan{Templates: tpl, Files: []PlannedFile{{ID: "db", Path: appBound, Template: "database.env", + OwnerAtRuntime: "owned"}}, + Provisions: []PlannedProvision{{Provision: "postgres-database", Provider: "novox", ProviderModule: "postgres"}}} +} + +func TestNatsAConsumersBoundValuesAreFilledFromItsProvidersEntry(t *testing.T) { + b := startBus(t) + registryOf679(t, b) + heard := b.listen(t) + e := startEngine(t, map[string]string{appBound: "owned"}) + b.assign(t, "app", appPlan(t)) + aClient(t, b, e, t.TempDir(), time.Hour) + + // No bucket of the provider's yet: waited for, and said. + eventually(t, "the consumer waits for its provider's bucket", func() bool { + r, _ := heard.last("app") + return r.Status == StatusWaiting && strings.Contains(strings.Join(r.WaitsFor, " "), "postgres_bound") + }) + kv, err := b.domain.CreateKeyValue(context.Background(), jetstream.KeyValueConfig{Bucket: "postgres_bound"}) + if err != nil { + t.Fatal(err) + } + b.kv["postgres_bound"] = kv + // The provider has not answered it yet: waited for. + b.put(t, "postgres_bound", "ace.app.postgres-database", `{"values":{"host":"elsewhere"}}`) + b.assign(t, "app", appPlan(t)) // a change of its own, so it looks again before the drift pass + eventually(t, "the consumer waits for its own entry", func() bool { + r, _ := heard.last("app") + return r.Status == StatusWaiting && strings.Contains(strings.Join(r.WaitsFor, " "), "postgres on novox") + }) + b.put(t, "postgres_bound", node+".app.postgres-database", `{"provider":"novox","provider-module":"postgres",`+ + `"provision":"postgres-database","consumer":"g14","module":"app","as":"mesh_g14_app","user":"mesh_g14_app",`+ + `"values":{"host":"novox.mesh","port":5432},"credential":"pair/novox/postgres-database/g14/app"}`) + eventually(t, "the file is filled from its provider's entry", func() bool { + return e.read(t, appBound) == "HOST=novox.mesh\nPORT=5432\nUSER=mesh_g14_app\n" + }) + b.put(t, "postgres_bound", node+".app.postgres-database", `{"user":"mesh_g14_app",`+ + `"values":{"host":"novox.lan","port":5432}}`) + eventually(t, "a changed entry refills the file", func() bool { + return e.read(t, appBound) == "HOST=novox.lan\nPORT=5432\nUSER=mesh_g14_app\n" + }) + b.put(t, "postgres_bound", node+".app.postgres-database", `{"refused":"app on g14 keeps several holders"}`) + eventually(t, "an entry the provider refused fails the consumer with its reason", func() bool { + r, _ := heard.last("app") + return r.Status == StatusFailed && strings.Contains(r.Reason, "keeps several holders") + }) +} + +// A consumer whose plan names its login's file (issue 779) is still a bind of its provider: it is listed, where an +// entry naming a value its provider's software issued per holder (issue 757) is not. +func TestNatsAProviderListsAConsumerWhoseEntryNamesItsLoginFile(t *testing.T) { + b := startBus(t) + registryOf679(t, b) + vault := startVault(t, b) + vault.keep(pairOf("ace", "nextcloud", ""), "nc-pw-never-on-the-bus", false) + vault.keep(pairOf("ace", "forge", ""), "forge-pw", false) + b.bind(t, "ace", "nextcloud", PlannedProvision{Name: "database", Provision: "postgres-database", Provider: node, + ProviderModule: "postgres", CredentialForm: "pair", Path: "/s/database.secret"}) + b.bind(t, "ace", "forge", PlannedProvision{Name: "database", Provision: "postgres-database", Provider: node, + ProviderModule: "postgres", CredentialForm: "issued", Path: "/s/forge.secret"}) + e := startEngine(t, map[string]string{consumersFile: "owned"}) + b.assign(t, "postgres", postgresPlan(t, "${consumers:postgres-database}")) + aClient(t, b, e, t.TempDir(), time.Hour) + eventually(t, "the consumer with a login file is listed, the per-holder one is not", func() bool { + return sameList(consumersIn(t, e), listed{Consumer: "ace", Module: "nextcloud", + Credential: pairOf("ace", "nextcloud", ""), Value: "nc-pw-never-on-the-bus"}) + }) +} diff --git a/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/environment.go b/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/environment.go new file mode 100644 index 00000000..2dae079a --- /dev/null +++ b/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/environment.go @@ -0,0 +1,234 @@ +package main + +import ( + "cmp" + "fmt" + "regexp" + "slices" + "strings" +) + +// The session environment as a collection (novox/hq ADR 0203, ADR 0349 clause 13, design 57 L3, issue 795). One +// collection, `environment`, whose items are typed by attributes: an item is a variable (`kind=variable`, +// `name=`) or a PATH entry (`kind=path`, `at=start|end`), and its template's text is the variable's value or +// the entry. The collector names the rendering, not a selector: `${collected:environment:posix}` and +// `${collected:environment:systemd}` (and the old `${environment:posix|systemd}`) are the collection `environment` +// written for a POSIX shell or for environment.d. Both renderings are the ones mesh-controller internal/catalogue +// environment_into.go writes for a collector not handed, byte for byte (its golden files are this client's). + +// Environment collection and renderers. +const ( + EnvironmentCollection = "environment" + EnvironmentPOSIX = "posix" + EnvironmentSystemd = "systemd" +) + +// environmentName is an environment variable's name, which only this collection's `name` attribute may be. +var environmentName = regexp.MustCompile(`^[A-Za-z_][A-Za-z0-9_]*$`) + +// attributeFits is whether an item's attribute is of the shape the collection allows. +func attributeFits(collection, k, v string) bool { + if collection == EnvironmentCollection && k == "name" { + return environmentName.MatchString(v) + } + return attributeKeyShape.MatchString(k) && attributeValueShape.MatchString(v) +} + +// environmentRenderer is the rendering a need names, when it is the environment's: `${collected:environment:posix}` +// or `${environment:posix}`. +func environmentRenderer(n Need) (string, bool) { + name := n.Name + switch { + case n.Kind == "environment": + case n.Kind == CollectedKind: + rest, is := strings.CutPrefix(name, EnvironmentCollection+":") + if !is { + return "", false + } + name = rest + default: + return "", false + } + return name, name == EnvironmentPOSIX || name == EnvironmentSystemd +} + +type envVariable struct{ module, name, value string } + +type envEntry struct { + module, entry string + order int + item string +} + +// literalProblem is why a value cannot be written, unquoted by either reader, as the same string in both formats +// (mesh-controller internal/catalogue environment_into.go). +func literalProblem(v string) string { + switch { + case strings.ContainsAny(v, `'"`): + return "holds a quote" + case strings.Contains(v, `\`): + return "holds a backslash" + case strings.ContainsAny(v, "\n\r"): + return "holds a line break" + case strings.ContainsRune(v, 0): + return "holds a NUL" + case strings.Contains(v, "$"): + return "holds a $, which the two readers expand differently" + } + return "" +} + +// environmentText is the collection's items written for one renderer: modules in name order, each module's variables +// sorted by name under a line naming it, then PATH's entries, each once where the first puts it, as the old form +// wrote them. An empty collection is empty text. +func environmentText(items map[string]itemValue, renderer string) (string, error) { + type owned struct { + id string + itemValue + } + var all []owned + for id, it := range items { + all = append(all, owned{id, it}) + } + slices.SortFunc(all, func(a, b owned) int { + ma, na := splitID(a.id) + mb, nb := splitID(b.id) + return cmp.Or(cmp.Compare(ma, mb), cmp.Compare(na, nb)) + }) + var variables []envVariable + var start, end []envEntry + setter := map[string]string{} + placed := map[string]bool{} + // Items of one module in the order the old form declared them: by order, then name. + slices.SortStableFunc(all, func(a, b owned) int { + ma, _ := splitID(a.id) + mb, _ := splitID(b.id) + if c := cmp.Compare(ma, mb); c != 0 { + return c + } + if c := cmp.Compare(a.order(), b.order()); c != 0 { + return c + } + _, na := splitID(a.id) + _, nb := splitID(b.id) + return cmp.Compare(na, nb) + }) + for _, it := range all { + module, _ := splitID(it.id) + value := strings.TrimSuffix(it.text, "\n") + if why := literalProblem(value); why != "" { + return "", fmt.Errorf("%s's item %s is %q on this machine, which %s: a value is a literal both readers "+ + "take alike", module, it.id, value, why) + } + switch it.attributes["kind"] { + case "variable": + n := it.attributes["name"] + if n == "" { + return "", fmt.Errorf("%s's item %s is a variable and has no name", module, it.id) + } + if first, taken := setter[n]; taken { + return "", fmt.Errorf("%s and %s both set %s on this machine; the account has one environment, so "+ + "one of them must stop setting it (novox/hq ADR 0203)", first, module, n) + } + setter[n] = module + variables = append(variables, envVariable{module, n, value}) + case "path": + if strings.Contains(value, ":") { + return "", fmt.Errorf("%s puts %q on PATH on this machine, which holds a colon, PATH's own separator", + module, value) + } + if value == "" { + return "", fmt.Errorf("%s's item %s is a PATH entry and says none", module, it.id) + } + if placed[value] { + continue + } + placed[value] = true + e := envEntry{module: module, entry: value, order: it.order(), item: it.id} + switch it.attributes["at"] { + case "start": + start = append(start, e) + case "end": + end = append(end, e) + default: + return "", fmt.Errorf("%s's item %s is a PATH entry and is not at start or end", module, it.id) + } + default: + return "", fmt.Errorf("%s's item %s is of the environment and is not kind=variable or kind=path", module, + it.id) + } + } + // Variables per module, sorted by name within it. + slices.SortStableFunc(variables, func(a, b envVariable) int { + return cmp.Or(cmp.Compare(a.module, b.module), cmp.Compare(a.name, b.name)) + }) + if renderer == EnvironmentSystemd { + return systemdText(variables, start, end), nil + } + return posixText(variables, start, end), nil +} + +func posixText(variables []envVariable, start, end []envEntry) string { + var b strings.Builder + named := "" + for _, v := range variables { + if v.module != named { + fmt.Fprintf(&b, "# %s\n", v.module) + named = v.module + } + fmt.Fprintf(&b, "export %s='%s'\n", v.name, v.value) + } + named = "" + for i := len(start) - 1; i >= 0; i-- { + p := start[i] + if p.module != named { + fmt.Fprintf(&b, "# %s\n", p.module) + named = p.module + } + fmt.Fprintf(&b, "case \":${PATH}:\" in *':%s:'*) ;; *) PATH='%s'\"${PATH:+:${PATH}}\" ;; esac\n", + p.entry, p.entry) + } + named = "" + for _, p := range end { + if p.module != named { + fmt.Fprintf(&b, "# %s\n", p.module) + named = p.module + } + fmt.Fprintf(&b, "case \":${PATH}:\" in *':%s:'*) ;; *) PATH=\"${PATH:+${PATH}:}\"'%s' ;; esac\n", + p.entry, p.entry) + } + if len(start)+len(end) > 0 { + b.WriteString("export PATH\n") + } + return b.String() +} + +func systemdText(variables []envVariable, start, end []envEntry) string { + var b strings.Builder + named := "" + for _, v := range variables { + if v.module != named { + fmt.Fprintf(&b, "# %s\n", v.module) + named = v.module + } + fmt.Fprintf(&b, "%s=%s\n", v.name, v.value) + } + line := func(entries []envEntry, format string) { + if len(entries) == 0 { + return + } + var modules, paths []string + seen := map[string]bool{} + for _, p := range entries { + if !seen[p.module] { + seen[p.module] = true + modules = append(modules, p.module) + } + paths = append(paths, p.entry) + } + fmt.Fprintf(&b, "# %s\n"+format+"\n", strings.Join(modules, ", "), strings.Join(paths, ":")) + } + line(start, "PATH=%s${PATH:+:$PATH}") + line(end, "PATH=${PATH:+$PATH:}%s") + return b.String() +} diff --git a/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/environment_test.go b/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/environment_test.go new file mode 100644 index 00000000..46a7922a --- /dev/null +++ b/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/environment_test.go @@ -0,0 +1,192 @@ +package main + +import ( + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + "time" +) + +// The session environment's collection (design 57 L3, issue 795), filled by this client from typed items, is what the +// controller composed from the old forms before the move. testdata/environment holds that composition for a machine +// with the five contributors and for one with zsh alone; mesh-controller's internal/catalogue +// TestTheEnvironmentOfTheCatalogueIsTheSameHandedOrNot holds the not handed half against the same files. + +type envItem struct{ module, name, kind, attr, value string } + +// theCataloguesEnvironment is the five contributors' environment as the catalogue declared it before the move. +var theCataloguesEnvironment = []envItem{ + {"adwaita", "gtk-theme", "variable", "GTK_THEME", "Adwaita:dark"}, + {"adwaita", "gtk2-rc-files", "variable", "GTK2_RC_FILES", "/usr/share/themes/Adwaita-dark/gtk-2.0/gtkrc"}, + {"adwaita", "qt-qpa-platformtheme", "variable", "QT_QPA_PLATFORMTHEME", "qt6ct"}, + {"adwaita", "qt-style-override", "variable", "QT_STYLE_OVERRIDE", "Fusion"}, + {"adwaita", "qt-select", "variable", "QT_SELECT", "6"}, + {"adwaita", "xcursor-theme", "variable", "XCURSOR_THEME", "Adwaita"}, + {"adwaita", "xcursor-size", "variable", "XCURSOR_SIZE", "24"}, + {"clipmenu", "cm-selections", "variable", "CM_SELECTIONS", "clipboard"}, + {"clipmenu", "cm-max-clips", "variable", "CM_MAX_CLIPS", "500"}, + {"clipmenu", "cm-histlength", "variable", "CM_HISTLENGTH", "15"}, + {"i3", "xdg-current-desktop", "variable", "XDG_CURRENT_DESKTOP", "i3"}, + {"i3", "xdg-session-desktop", "variable", "XDG_SESSION_DESKTOP", "i3"}, + {"xterm", "terminal", "variable", "TERMINAL", "xterm"}, + {"zsh", "editor", "variable", "EDITOR", "vim"}, + {"zsh", "visual", "variable", "VISUAL", "vim"}, + {"zsh", "xdg-config-home", "variable", "XDG_CONFIG_HOME", "${machine:account-home}/.config"}, + {"zsh", "path-1", "path:10", "", "${machine:account-home}/.local/bin"}, + {"zsh", "path-2", "path:20", "", "${machine:account-home}/scripts"}, + {"zsh", "path-3", "path:30", "", "${machine:account-home}/scripts/bin"}, +} + +func envPlans(t *testing.T, items []envItem) (collector Plan, contributors map[string]Plan) { + t.Helper() + posix := "# header\n${collected:environment:posix}" + systemd := "${environment:systemd}" + tpl, _ := templates(t, map[string]string{"posix": posix, "systemd": systemd}) + facts := map[string]string{"machine:account-home": "/home/op"} + collector = Plan{Templates: tpl, Placement: facts, + Files: []PlannedFile{{ID: "posix", Path: "/env/environment.sh", Template: "posix", OwnerAtRuntime: "owned"}, + {ID: "systemd", Path: "/env/50-mesh.conf", Template: "systemd", OwnerAtRuntime: "owned"}}, + Collections: []PlannedCollection{{Name: "environment", Comment: "#"}}} + texts := map[string]map[string]string{} + planned := map[string][]PlannedItem{} + for _, it := range items { + if texts[it.module] == nil { + texts[it.module] = map[string]string{} + } + texts[it.module]["env-"+it.name] = it.value + "\n" + attrs := map[string]string{"kind": "variable", "name": it.attr} + if strings.HasPrefix(it.kind, "path") { + attrs = map[string]string{"kind": "path", "at": "start", "order": strings.TrimPrefix(it.kind, "path:")} + } + planned[it.module] = append(planned[it.module], PlannedItem{Collection: "environment", Name: it.name, + Attributes: attrs, Template: "env-" + it.name}) + } + contributors = map[string]Plan{} + for m := range texts { + tp, _ := templates(t, texts[m]) + contributors[m] = Plan{Templates: tp, Items: planned[m], Placement: facts} + } + return collector, contributors +} + +func TestNatsTheEnvironmentFilledFromTypedItemsIsWhatTheOldContributionMade(t *testing.T) { + for name, only := range map[string]string{"all": "", "zsh": "zsh"} { + t.Run(name, func(t *testing.T) { + var items []envItem + for _, it := range theCataloguesEnvironment { + if only == "" || it.module == only { + items = append(items, it) + } + } + collector, contributors := envPlans(t, items) + b := startBus(t) + e := startEngine(t, map[string]string{"/env/environment.sh": "owned", "/env/50-mesh.conf": "owned"}) + b.assign(t, "node-env", collector) + for m, p := range contributors { + b.assign(t, m, p) + } + aClient(t, b, e, t.TempDir(), time.Hour) + for file, golden := range map[string]string{"/env/environment.sh": "posix", "/env/50-mesh.conf": "systemd"} { + want, err := os.ReadFile(filepath.Join("testdata", "environment", name+"."+golden)) + if err != nil { + t.Fatal(err) + } + prefix := "" + if golden == "posix" { + prefix = "# header\n" + } + eventually(t, file+" is what it was", func() bool { return e.read(t, file) == prefix+string(want) }) + } + }) + } +} + +func TestTheEnvironmentRefusesAVariableTwoModulesSetAndAColonOnPATH(t *testing.T) { + items := map[string]itemValue{ + "a": {text: "1\n", attributes: map[string]string{"kind": "variable", "name": "X"}}, + "b": {text: "2\n", attributes: map[string]string{"kind": "variable", "name": "X"}}, + } + if _, err := environmentText(items, EnvironmentPOSIX); err == nil || !strings.Contains(err.Error(), "both set X") { + t.Fatalf("two modules setting X: %v", err) + } + items = map[string]itemValue{"a": {text: "/x:/y\n", attributes: map[string]string{"kind": "path", "at": "end"}}} + if _, err := environmentText(items, EnvironmentSystemd); err == nil || !strings.Contains(err.Error(), "colon") { + t.Fatalf("a colon on PATH: %v", err) + } + if !attributeFits("environment", "name", "GTK_THEME") || attributeFits("shell:zsh:normal", "name", "GTK_THEME") { + t.Fatal("an upper-case name is the environment's alone") + } +} + +// The catalogue's own modules, as they are declared now: node-env's two files and the five contributors' items, read +// from their module.json and templates, fill to the files the old form composed (issue 795). +func TestNatsTheEnvironmentOfTheCataloguesModulesIsWhatTheOldFormComposed(t *testing.T) { + modules := filepath.Join("..", "..", "..") + type item struct { + Collection string `json:"collection"` + Name string `json:"name"` + Attributes map[string]string `json:"attributes"` + Template string `json:"template"` + } + read := func(path string) []byte { + b, err := os.ReadFile(filepath.Join(modules, path)) + if err != nil { + t.Fatal(err) + } + return b + } + for name, only := range map[string][]string{"all": {"adwaita", "clipmenu", "i3", "xterm", "zsh"}, "zsh": {"zsh"}} { + t.Run(name, func(t *testing.T) { + facts := map[string]string{"machine:account-home": "/home/op"} + tpl, _ := templates(t, map[string]string{"posix": string(read("node-env/files/environment.sh")), + "systemd": string(read("node-env/files/50-mesh.conf"))}) + b := startBus(t) + e := startEngine(t, map[string]string{"/env/environment.sh": "owned", "/env/50-mesh.conf": "owned"}) + b.assign(t, "node-env", Plan{Templates: tpl, Placement: facts, + Files: []PlannedFile{{ID: "posix", Path: "/env/environment.sh", Template: "posix", OwnerAtRuntime: "owned"}, + {ID: "systemd", Path: "/env/50-mesh.conf", Template: "systemd", OwnerAtRuntime: "owned"}}, + Collections: []PlannedCollection{{Name: "environment", Comment: "#"}}}) + for _, m := range only { + var manifest struct { + Moved bool `json:"moved"` + Items []item `json:"items"` + } + if err := json.Unmarshal(read(m+"/module.json"), &manifest); err != nil { + t.Fatal(err) + } + if !manifest.Moved || len(manifest.Items) == 0 { + t.Fatalf("%s is not a moved contributor of items", m) + } + texts := map[string]string{} + var planned []PlannedItem + for _, it := range manifest.Items { + id := filepath.Base(it.Template) + texts[id] = string(read(m + "/" + it.Template)) + planned = append(planned, PlannedItem{Collection: it.Collection, Name: it.Name, + Attributes: it.Attributes, Template: id}) + } + tp, _ := templates(t, texts) + b.assign(t, m, Plan{Templates: tp, Items: planned, Placement: facts}) + } + aClient(t, b, e, t.TempDir(), time.Hour) + for file, golden := range map[string]string{"/env/environment.sh": "posix", "/env/50-mesh.conf": "systemd"} { + want, err := os.ReadFile(filepath.Join("testdata", "environment", name+"."+golden)) + if err != nil { + t.Fatal(err) + } + // node-env's own files hold their header, which is the module's, then the collected text. + header := strings.SplitAfter(string(read("node-env/files/"+map[string]string{"posix": "environment.sh", + "systemd": "50-mesh.conf"}[golden])), "\n") + head := "" + for _, l := range header { + if strings.HasPrefix(l, "#") { + head += l + } + } + eventually(t, file+" is what it was", func() bool { return e.read(t, file) == head+string(want) }) + } + }) + } +} diff --git a/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/fill.go b/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/fill.go new file mode 100644 index 00000000..06eaf17d --- /dev/null +++ b/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/fill.go @@ -0,0 +1,104 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net" + "strings" + "time" +) + +// The node-engine's narrow local write verb (novox/hq ADR 0349 clause 4, design 57 P2; mesh-host internal/filling): +// one unix socket, one JSON request a line — a filled file's path and its content — and one answer. **The client +// fills; the node-engine writes**, as the declared file, with its owner, mode and who owns it at runtime: an owned +// file whole and written back whenever it differs, a seed only when absent, a managed file's own keys or block +// only. And it makes what reads the file take the change (restart-on, reload-on, a container that reads it). So +// the client never writes a machine's file itself and needs no right to. +// +// The request and the answer are copied here, field for field, from mesh-host internal/filling: the client is +// the one other reader of that format, and the node-engine's decoder refuses a field it does not know. + +// DefaultSocket is where the node-engine serves the write. +const DefaultSocket = "/run/nox-mesh-host/fill.sock" + +// FillRequest is one write. +type FillRequest struct { + Path string `json:"path"` + Content []byte `json:"content"` +} + +// FillAnswer is what came of it: Refused is a write not attempted, Failed one attempted that did not complete, +// each with why; otherwise Action is what the write did (created, updated, corrected, unchanged, kept) and Digest +// what the file holds now. +type FillAnswer struct { + Path string `json:"path,omitempty"` + Action string `json:"action,omitempty"` + Digest string `json:"digest,omitempty"` + Refused string `json:"refused,omitempty"` + Failed string `json:"failed,omitempty"` +} + +// Filler has the node-engine write a filled file. +type Filler interface { + Fill(ctx context.Context, path string, content []byte) (FillAnswer, error) +} + +// SocketFiller is the node-engine's socket. +type SocketFiller struct { + Socket string + // Busy is how long a write the node-engine says it is busy for waits before it is asked again, and Tries how + // many times it is asked. + Busy time.Duration + Tries int +} + +// Fill asks the node-engine to write one file, again while it says it is busy. +func (s SocketFiller) Fill(ctx context.Context, path string, content []byte) (FillAnswer, error) { + tries := max(s.Tries, 1) + var a FillAnswer + var err error + for i := 0; i < tries; i++ { + if a, err = s.once(ctx, path, content); err != nil || !strings.HasPrefix(a.Refused, "busy") { + return a, err + } + select { + case <-ctx.Done(): + return a, ctx.Err() + case <-time.After(s.Busy): + } + } + return a, err +} + +func (s SocketFiller) once(ctx context.Context, path string, content []byte) (FillAnswer, error) { + var d net.Dialer + conn, err := d.DialContext(ctx, "unix", s.Socket) + if err != nil { + return FillAnswer{}, fmt.Errorf("the node-engine's write socket %s: %w", s.Socket, err) + } + defer conn.Close() + if deadline, ok := ctx.Deadline(); ok { + _ = conn.SetDeadline(deadline) + } + if err := json.NewEncoder(conn).Encode(FillRequest{Path: path, Content: content}); err != nil { + return FillAnswer{}, fmt.Errorf("asking the node-engine to write %s: %w", path, err) + } + var a FillAnswer + if err := json.NewDecoder(conn).Decode(&a); err != nil { + return FillAnswer{}, fmt.Errorf("the node-engine's answer for %s: %w", path, err) + } + return a, nil +} + +// errFill is a write the node-engine refused or failed, with its word. +func errFill(a FillAnswer) error { + switch { + case a.Refused != "": + return errors.New("the node-engine refused it: " + a.Refused) + case a.Failed != "": + return errors.New("the node-engine's write failed: " + a.Failed) + } + return nil +} diff --git a/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/i3_golden_test.go b/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/i3_golden_test.go new file mode 100644 index 00000000..0097891f --- /dev/null +++ b/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/i3_golden_test.go @@ -0,0 +1,90 @@ +package main + +import ( + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + "time" +) + +// i3's window-manager configuration, filled by this client from the items of the collection +// contribution:node-display-session:config that its contributors declare, is byte for byte what the controller +// composed from their old contributions before the move (design 57 L3, issue 848). The module files are the +// catalogue's own; testdata/i3 holds the configuration as the controller composed it, for a machine with all nine +// contributors (dunst, rofi, i3status-rust, asus-zephyrus-g14, clipmenu, feh, slack, polychromatic and i3 itself) and for +// one with only i3. The golden files were composed once, by a one-off run of mesh-controller's catalogue at the +// feature branch's head over the catalogue as it was before the move; the controller's composition for a collector +// not handed, from the same items, was compared with them in the same way and is byte for byte the same. + +func TestNatsI3sConfigFilledFromItsContributorsItemsIsWhatTheOldContributionsMade(t *testing.T) { + modules := filepath.Join("..", "..", "..") + read := func(path string) string { + b, err := os.ReadFile(filepath.Join(modules, path)) + if err != nil { + t.Fatal(err) + } + return string(b) + } + const collection = "contribution:node-display-session:config" + all := []string{"asus-zephyrus-g14", "clipmenu", "dunst", "feh", "i3status-rust", "polychromatic", "rofi", "slack"} + for name, contributors := range map[string][]string{"all": all, "i3-only": nil} { + t.Run(name, func(t *testing.T) { + facts := map[string]string{"machine:account-home": "/home/op"} + tpl, _ := templates(t, map[string]string{"config": read("i3/files/config")}) + collector := Plan{Templates: tpl, Placement: facts, + Files: []PlannedFile{{ID: "config", Path: "/i3/config", Template: "config", OwnerAtRuntime: "owned"}}, + Collections: []PlannedCollection{{Name: collection, Comment: "#"}}} + b := startBus(t) + e := startEngine(t, map[string]string{"/i3/config": "owned"}) + b.assign(t, "i3", collector) + for _, m := range contributors { + var c struct { + Moved bool `json:"moved"` + Contributions []any `json:"contributions"` + Items []struct { + Collection string `json:"collection"` + Name string `json:"name"` + Attributes map[string]string `json:"attributes"` + Template string `json:"template"` + } `json:"items"` + } + if err := json.Unmarshal([]byte(read(m+"/module.json")), &c); err != nil { + t.Fatal(err) + } + if !c.Moved || len(c.Contributions) != 0 { + t.Fatalf("%s is not moved or still holds contributions", m) + } + texts := map[string]string{} + var planned []PlannedItem + for _, it := range c.Items { + if it.Collection != collection { + continue + } + id := filepath.Base(it.Template) + texts[id] = read(m + "/" + it.Template) + planned = append(planned, PlannedItem{Collection: it.Collection, Name: it.Name, Template: id, + Attributes: it.Attributes}) + } + if len(planned) != 1 { + t.Fatalf("%s declares %d items of the window-manager configuration, want 1", m, len(planned)) + } + tp, _ := templates(t, texts) + b.assign(t, m, Plan{Templates: tp, Items: planned, Placement: facts}) + } + aClient(t, b, e, t.TempDir(), time.Hour) + want, err := os.ReadFile(filepath.Join("testdata", "i3", name+".config")) + if err != nil { + t.Fatal(err) + } + // The golden holds asus-zephyrus-g14's directory as @ASUS_LIB@: the catalogue's path check keeps one module's + // path out of another module's files, and the golden is the one file that holds every contributor's lines. + golden := strings.ReplaceAll(string(want), "@ASUS_LIB@", "/usr/local/lib/asus-zephyrus-g14") + eventually(t, "i3's configuration is what it was", func() bool { return e.read(t, "/i3/config") == golden }) + if t.Failed() { + t.Logf("got:\n%s", e.read(t, "/i3/config")) + } + }) + } +} diff --git a/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/main.go b/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/main.go new file mode 100644 index 00000000..3a127e23 --- /dev/null +++ b/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/main.go @@ -0,0 +1,170 @@ +// mesh-provisioner-client: the provisioner's client, holder of the node-provisioner seat on every node (novox/hq +// ADR 0349 clauses 3 to 5, design 57 P5, issue 671). A service of its own, outside the tool runner, on a bus account +// of its own (`provisioner.`, mesh-controller internal/broker, design 57 P3): the vault hands a secret only to +// an account of its own, and the bus grants this one its machine's keys and nothing else. It fills the templates of +// each module handed to the provisioner on its node and has the node-engine write them (client.go). +package main + +import ( + "context" + "crypto/sha256" + "crypto/tls" + "crypto/x509" + "encoding/hex" + "encoding/json" + "errors" + "flag" + "fmt" + "os" + "os/signal" + "path/filepath" + "syscall" + "time" + + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" +) + +func logf(format string, a ...any) { fmt.Fprintf(os.Stderr, format+"\n", a...) } + +// Credential is the client's account as the controller delivers it, its own secret `provisioner-broker` +// (mesh-controller cmd/mesh-controller issueProvisionerClient): the bus, its certificate by fingerprint, the +// machine, the user and its password. +type Credential struct { + URL string `json:"url"` + Fingerprint string `json:"fingerprint,omitempty"` + Node string `json:"node"` + User string `json:"user"` + Password string `json:"password"` +} + +func readCredential(path string) (Credential, error) { + raw, err := os.ReadFile(path) + if err != nil { + return Credential{}, fmt.Errorf("the client's account cannot be read: %w", err) + } + var c Credential + if err := json.Unmarshal(raw, &c); err != nil { + return Credential{}, errors.New("the client's account is not the JSON the mesh delivers (url, fingerprint, " + + "node, user, password)") + } + if c.URL == "" || c.Node == "" || c.User == "" || c.Password == "" { + return Credential{}, errors.New("the client's account names no bus, machine, user or password") + } + if c.User != "provisioner."+c.Node { + return Credential{}, fmt.Errorf("the account %s is not machine %s's provisioner client's", c.User, c.Node) + } + return c, nil +} + +// pinnedTo accepts exactly the certificate with this SHA-256: the mesh's bus presents one of its own, in no trust +// store (mesh-controller internal/broker PinnedToFingerprint). +func pinnedTo(want string) *tls.Config { + return &tls.Config{ + InsecureSkipVerify: true, //nolint:gosec // replaced by the pin below, which is stricter + MinVersion: tls.VersionTLS12, + VerifyPeerCertificate: func(rawCerts [][]byte, _ [][]*x509.Certificate) error { + if len(rawCerts) == 0 { + return errors.New("the bus presented no certificate") + } + sum := sha256.Sum256(rawCerts[0]) + if got := "sha256:" + hex.EncodeToString(sum[:]); got != want { + return fmt.Errorf("the bus presented a certificate this mesh does not know (%s)", got) + } + return nil + }, + } +} + +func main() { + credential := flag.String("credential", os.Getenv("MESH_PROVISIONER_CREDENTIAL"), + "the client's account, its own secret provisioner-broker (MESH_PROVISIONER_CREDENTIAL)") + socket := flag.String("socket", DefaultSocket, "the node-engine's socket for writing a filled file") + stateDir := flag.String("state", os.Getenv("MESH_PROVISIONER_STATE"), + "where the client keeps the templates it fetched (MESH_PROVISIONER_STATE)") + drift := flag.Duration("drift", time.Minute, "how often every filled file is written again") + flag.Parse() + if *credential == "" || *stateDir == "" { + logf("usage: mesh-provisioner-client -credential -state ") + os.Exit(2) + } + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer stop() + if err := serve(ctx, *credential, *socket, *stateDir, *drift); err != nil && ctx.Err() == nil { + logf("[provisioner-client] %v", err) + os.Exit(1) + } +} + +// serve connects and runs the client until ctx ends. A run that stops — a watch ended, the bus came back — is run +// again from the current values, after a wait that grows to a minute. +func serve(ctx context.Context, credential, socket, stateDir string, drift time.Duration) error { + c, err := readCredential(credential) + if err != nil { + return err + } + back := make(chan struct{}, 1) + opts := []nats.Option{ + nats.UserInfo(c.User, c.Password), + nats.Name("mesh-provisioner-client " + c.Node), + // Answers come to the account's own inbox and no other (novox/hq design 25 §4). + nats.CustomInboxPrefix("_INBOX." + c.User), + nats.MaxReconnects(-1), + nats.ReconnectHandler(func(*nats.Conn) { + select { + case back <- struct{}{}: + default: + } + }), + } + if c.Fingerprint != "" { + opts = append(opts, nats.Secure(pinnedTo(c.Fingerprint))) + } + nc, err := nats.Connect(c.URL, opts...) + if err != nil { + return fmt.Errorf("cannot reach the bus at %s as %s: %w", c.URL, c.User, err) + } + defer nc.Close() + js, err := jetstream.New(nc) + if err != nil { + return err + } + // Its own key, for the vault to seal each secret to: made now, kept in memory alone. + secrets, err := NewVaultSecrets(nc, c.User) + if err != nil { + return err + } + client := &Client{Node: c.Node, JS: js, Conn: nc, Drift: drift, Secrets: secrets, + Fill: SocketFiller{Socket: socket, Busy: time.Second, Tries: 10}, + Templates: &templateStore{dir: filepath.Join(stateDir, "templates"), fetch: HTTPFetch}} + wait := time.Second + for { + run, cancel := context.WithCancel(ctx) + go func() { + select { + case <-back: + logf("[provisioner-client] the bus is back; filling from the current values") + cancel() + case <-run.Done(): + } + }() + started := time.Now() + err := client.Run(run) + cancel() + if ctx.Err() != nil { + return nil + } + if time.Since(started) > time.Minute { + wait = time.Second + } + if err != nil && !errors.Is(err, context.Canceled) { + logf("[provisioner-client] stopped (%v); running again in %s", err, wait) + } + select { + case <-ctx.Done(): + return nil + case <-time.After(wait): + } + wait = min(2*wait, time.Minute) + } +} diff --git a/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/moved_collectors_test.go b/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/moved_collectors_test.go new file mode 100644 index 00000000..91b55c6f --- /dev/null +++ b/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/moved_collectors_test.go @@ -0,0 +1,105 @@ +package main + +import ( + "encoding/json" + "os" + "path/filepath" + "regexp" + "testing" + "time" +) + +// The collectors design 57 L2 moves (issue 681): zsh, power and triggerhappy, each with every contributor the +// catalogue gives it, filled by this client from the modules' real templates, write the file the controller composed +// from the old forms before the move, byte for byte (composedBeforeTheMove). + +// catalogueModules is where the catalogue's modules are, from this package. +const catalogueModules = "../../.." + +// repoManifest is the part of a module's manifest in the catalogue this test reads. +type repoManifest struct { + Module string `json:"module"` + Templates string `json:"templates"` + Moved bool `json:"moved"` + Items []struct { + Collection string `json:"collection"` + Template string `json:"template"` + } `json:"items"` +} + +var oldNamespacePlaceholder = regexp.MustCompile(`\$\{((?:shell|contribution):[^}]*)\}`) + +func TestNatsTheCollectorsL2MovedFillFromTheirContributorsItemsAsTheyWereComposed(t *testing.T) { + collectors := []struct{ module, template, want string }{ + {"zsh", "files/zshrc", "zsh.zshrc"}, + {"triggerhappy", "files/triggers.conf", "triggerhappy.triggers.conf"}, + {"power", "templates/mesh-power/moments/after-wake", "power.after-wake"}, + } + paths, err := filepath.Glob(filepath.Join(catalogueModules, "*", "module.json")) + if err != nil || len(paths) == 0 { + t.Fatalf("no catalogue at %s: %v", catalogueModules, err) + } + var modules []repoManifest + for _, p := range paths { + raw, err := os.ReadFile(p) + if err != nil { + t.Fatal(err) + } + var m repoManifest + if err := json.Unmarshal(raw, &m); err != nil { + t.Fatalf("%s: %v", p, err) + } + modules = append(modules, m) + } + for _, c := range collectors { + t.Run(c.module, func(t *testing.T) { + text, err := os.ReadFile(filepath.Join(catalogueModules, c.module, c.template)) + if err != nil { + t.Fatal(err) + } + want, ok := composedBeforeTheMove[c.want] + if !ok { + t.Fatalf("no file composed before the move for %s", c.want) + } + path := "/etc/" + c.module + "/collected" + b := startBus(t) + e := startEngine(t, map[string]string{path: "owned"}) + tpl, _ := templates(t, map[string]string{"file": string(text)}) + collector := Plan{Templates: tpl, Files: []PlannedFile{{ID: "file", Path: path, Template: "file", + OwnerAtRuntime: "owned"}}} + names := map[string]bool{} + for _, f := range oldNamespacePlaceholder.FindAllStringSubmatch(string(text), -1) { + if !names[f[1]] { + names[f[1]] = true + // The controller plans an old namespace's collection with the comment it wrote: "#" for these. + collector.Collections = append(collector.Collections, PlannedCollection{Name: f[1], Comment: "#"}) + } + } + b.assign(t, c.module, collector) + contributors := 0 + for _, m := range modules { + for _, it := range m.Items { + if !names[it.Collection] { + continue + } + if !m.Moved { + t.Errorf("%s contributes an item and is not moved", m.Module) + } + item, err := os.ReadFile(filepath.Join(catalogueModules, m.Module, it.Template)) + if err != nil { + t.Fatal(err) + } + b.assign(t, m.Module, contributor(t, it.Collection, string(item))) + contributors++ + } + } + aClient(t, b, e, t.TempDir(), time.Hour) + eventually(t, c.module+"'s file as the controller composed it", func() bool { + return e.read(t, path) == want + }) + if t.Failed() { + t.Logf("%d contributors; got:\n%s", contributors, e.read(t, path)) + } + }) + } +} diff --git a/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/moved_collectors_want_test.go b/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/moved_collectors_want_test.go new file mode 100644 index 00000000..f9a1b867 --- /dev/null +++ b/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/moved_collectors_want_test.go @@ -0,0 +1,10 @@ +package main + +// The files the controller composed from the old forms at the feature branch before design 57 L2 moved their +// collectors (issue 681): zsh's rc, triggerhappy's triggers and power's after-wake moment, every contributor placed. +// Kept in a test file, where the paths check (novox/hq ADR 0260) reads no other module's path. +var composedBeforeTheMove = map[string]string{ + "zsh.zshrc": "# The mesh's block (module zsh, novox/hq ADR 0204): the defaults every machine shares, and the code\n# other modules contribute in three slots. It is replaced at every push. Everything below it is\n# yours, kept as you wrote it, and runs after it, so your lines win.\n\n# History, kept across sessions and shared between open shells. Without these zsh saves none:\n# SAVEHIST defaults to 0 (novox/hq to-be 41, found on review 2026-10-04).\nHISTFILE=\"$HOME/.zsh_history\"\nHISTSIZE=100000\nSAVEHIST=100000\nsetopt EXTENDED_HISTORY SHARE_HISTORY HIST_IGNORE_DUPS HIST_IGNORE_SPACE HIST_REDUCE_BLANKS\n\n# The completion system. Nothing initialised it, so only zsh's basic completion ran. Its cache\n# goes under ~/.cache rather than beside ~/.zshrc.\n[[ -d \"$HOME/.cache/zsh\" ]] || mkdir -p \"$HOME/.cache/zsh\"\nautoload -Uz compinit && compinit -d \"$HOME/.cache/zsh/zcompdump\"\nzstyle ':completion:*' menu select\n\n# Terminal title: host, directory, git branch\nfunction set_terminal_title() {\n local git_branch=\"\"\n if git rev-parse --is-inside-work-tree &>/dev/null; then\n git_branch=\" ($(git branch --show-current 2>/dev/null))\"\n fi\n print -Pn \"\\e]2;%m: %~${git_branch}\\a\"\n}\nprecmd_functions+=(set_terminal_title)\n\n# Keybindings\nbindkey \"^[[H\" beginning-of-line # Home\nbindkey \"^[OH\" beginning-of-line # Home\nbindkey \"^A\" beginning-of-line # Ctrl + A\nbindkey \"^[[F\" end-of-line # End\nbindkey \"^[OF\" end-of-line # End\nbindkey \"^E\" end-of-line # Ctrl + E\nbindkey \"^[[3~\" delete-char # Del\n\n# Colour support of ls and friends\nif [ -x /usr/bin/dircolors ]; then\n test -r $HOME/.dircolors && eval \"$(dircolors -b $HOME/.dircolors)\" || eval \"$(dircolors -b)\"\n alias ls='ls --color=auto'\n alias dir='dir --color=auto'\n alias vdir='vdir --color=auto'\n alias grep='grep --color=auto'\n alias fgrep='fgrep --color=auto'\n alias egrep='egrep --color=auto'\nfi\nalias ll='ls -alhF'\nalias la='ls -Ah'\nalias l='ls -CFh'\n\n# A throwaway container, and what takes the space under a directory\nalias drun='docker run -it --rm'\ndisksize() {\n du -h --max-depth=1 \"${1:-.}\" | sort -h\n}\nsudo-disksize() {\n sudo du -h --max-depth=1 \"${1:-.}\" | sort -h\n}\n\n# claude-code\n# The agent's session (module claude-code, novox/hq ADR 0266): where this machine names an account of the\n# agents' own, claude in an interactive zsh is claude-agent, which starts the session as that account. Where\n# agents run as the operator's account the file names that account, and nothing is added. claude-agent runs\n# the real claude: exec, sudo and its sh see no alias.\nif [[ -r /etc/claude-code/agent-account ]]; then\n\t() {\n\t\tlocal agent=$(.` of mesh-controller_assignments +// (mesh-controller internal/inventory Provisioning), which this client reads for its own machine alone. An +// assignment with no plan is a module the controller has not handed to the provisioner: its files are the +// node-engine's, from the declaration, and this client leaves them alone (ADR 0349 clause 17). A module only pulled +// in by another on this node, and handed, has a value of its own there, which names what requires it and no time it +// was assigned (issue 765): read and filled the same way. + +// Assignment is one value of the assignments mirror. **Lenient at the top**, so a field the controller adds to an +// assignment (design 54 §1) does not stop the client; **strict in the plan**, which is the client's to act on. +type Assignment struct { + Node string `json:"node"` + Module string `json:"module"` + Provisioning json.RawMessage `json:"provisioning,omitempty"` +} + +// Plan is one module's plan on this node. +type Plan struct { + Templates Templates `json:"templates"` + Files []PlannedFile `json:"files"` + Placement map[string]string `json:"placement,omitempty"` + // Secrets are the module's own secrets its declaration on this node names (design 57 V4): each one its + // templates name as `${secret:}`, and each it keeps in a file of its own at Path. The vault hands this + // client a secret only when the plan names it (mesh-vault askers.go). + Secrets []PlannedSecret `json:"secrets,omitempty"` + // Items are the module's items of collections (ADR 0349 clause 13, design 57 L1): each filled from its template + // like a file, and recorded in mesh-provisioner_collect for this node instead of written to a file. The collector + // of each, on this node, is filled from every contributor's item there. + Items []PlannedItem `json:"items,omitempty"` + // Collections are the collections the module's templates name, as their collector: `${collected:}`, or + // one of the three old namespaces read as a collection of the same name (`${shell:zsh:normal}` is the collection + // `shell:zsh:normal`). Each with the comment its tool writes a line in, which introduces each contributor's + // item with the contributor's name; none for a tool without one. + Collections []PlannedCollection `json:"collections,omitempty"` + // Provisions are what the module requires on this node, each with the provider serving it (issue 760), which + // the vault acts on and this client only reads. An entry with a name is a credential this client fills from the + // vault (issue 757): the value a provider's software issued for this module on this machine. Its templates name + // one as `${secret:}`, and Path, when set, is a file holding it alone. The vault hands one only when the + // plan names it (mesh-vault askers.go). + Provisions []PlannedProvision `json:"provisions,omitempty"` +} + +// PlannedItem is one item the module contributes: to which collection, and the template it is filled from. +// +// A module may give several items to one collection, each with a Name of its own, and any with Attributes a +// placeholder selecting by them is matched against (design 57 L3, issue 777). An item whose capability the machine +// lacks is not in the plan: the controller judged that. +type PlannedItem struct { + Collection string `json:"collection"` + Name string `json:"name,omitempty"` + Attributes map[string]string `json:"attributes,omitempty"` + Template string `json:"template"` +} + +// key is the item among its module's: the collection, and `/` for a named item. +func (it PlannedItem) key() string { + if it.Name == "" { + return it.Collection + } + return it.Collection + "/" + it.Name +} + +// PlannedCollection is one collection the module collects. +type PlannedCollection struct { + Name string `json:"name"` + Comment string `json:"comment,omitempty"` +} + +// PlannedProvision is one provision the module requires, under its local name (empty for the ordinary one), and +// its provider: the machine and the module serving it. With a name, a credential of it this client fills: Name as +// the module's files name it (`${secret:}`), and the file holding it alone, when there is one. +type PlannedProvision struct { + Name string `json:"name,omitempty"` + Provision string `json:"provision"` + Local string `json:"local,omitempty"` + Provider string `json:"provider"` + ProviderModule string `json:"provider-module,omitempty"` + // CredentialForm is the form the provider's offer declares for its credential (issue 768), which the vault acts + // on and this client only reads: one of credentialForms, or none from a controller before it. + CredentialForm string `json:"credential-form,omitempty"` + Path string `json:"path,omitempty"` + // Contributes is what this module tells the provider about itself for this provision (a database's name, its + // extensions): the consumer manifest's `contributes`, which the provider's helper reads and the client keeps. + Contributes map[string]any `json:"contributes,omitempty"` +} + +// issuedForHolder is whether the entry names a value its provider's software issued for this holder alone (issue +// 757). An entry of the login-per-consumer form that names a file (issue 779) is still the bind it was: its provider +// lists this consumer and its bound values are read from it. +func (p PlannedProvision) issuedForHolder() bool { return p.Name != "" && p.CredentialForm != "pair" } + +// credentialForms are the forms a provider's offer may declare for its credential (mesh-vault pair.go): one value +// every holder shares, a login per consumer, a value its software issues, or one a person gives. +var credentialForms = []string{"shared", "pair", "issued", "given"} + +// PlannedSecret is one secret of the module's own, by its name in the module's manifest; Path, when set, is the +// file holding it alone (the manifest's `own-secrets`), which the client fills with its value and nothing else. +type PlannedSecret struct { + Name string `json:"name"` + Path string `json:"path,omitempty"` +} + +// Templates is the archive the module's build packed its templates directory into: fetched from Source, and used only when it +// hashes to Digest. +type Templates struct { + Source string `json:"source"` + Digest string `json:"digest"` +} + +// PlannedFile is one file the provisioner fills. +type PlannedFile struct { + ID string `json:"id"` + Path string `json:"path"` + Template string `json:"template"` + OwnerAtRuntime string `json:"owner-at-runtime"` + Takes string `json:"takes,omitempty"` +} + +// The placeholder kinds placement answers, as the plan's Placement keys begin (ADR 0349 clause 2). +var placementKinds = []string{"dir", "access", "port", "seat", "machine", "backup"} + +var ( + digestShape = regexp.MustCompile(`^sha256:[0-9a-f]{64}$`) + // secretShape is a secret's name as a manifest writes it and `${secret:…}` names it (mesh-controller + // internal/catalogue secrets_into_files.go). + secretShape = regexp.MustCompile(`^[a-z0-9][a-z0-9-]*$`) + // collectionShape is a collection's name (mesh-controller internal/catalogue collections.go): words joined by + // colons, so an old namespace's placeholder names one as it reads, `shell:zsh:normal`. + collectionShape = regexp.MustCompile(`^[a-z0-9][a-z0-9-]*(:[a-z0-9][a-z0-9-]*)*$`) + // itemNameShape, attributeKeyShape and attributeValueShape are an item's name and attributes (mesh-controller + // internal/catalogue collections.go); none holds a colon, a comma or an equals sign. + itemNameShape = regexp.MustCompile(`^[a-z0-9][a-z0-9-]*$`) + attributeKeyShape = regexp.MustCompile(`^[a-z][a-z0-9-]*$`) + attributeValueShape = regexp.MustCompile(`^[a-z0-9][a-z0-9_.-]*$`) + // nodeShape is a machine's name as the vault writes it in a secret's name (mesh-vault intake.go namePart). + nodeShape = regexp.MustCompile(`^[a-z0-9][a-z0-9._-]{0,62}$`) + ownerWords = []string{"owned", "seed", "managed"} + takeWords = []string{"restart-on", "reload-on", "read-on-use"} +) + +// errNoPlan is an assignment the controller has not handed to the provisioner. +var errNoPlan = errors.New("no plan") + +// planOf reads an assignment's value: its module and its plan, errNoPlan when it has none, or why the plan cannot +// be acted on. +func planOf(node, key string, value []byte) (string, Plan, error) { + var a Assignment + if err := json.Unmarshal(value, &a); err != nil { + return "", Plan{}, fmt.Errorf("the assignment %s is not the controller's JSON: %w", key, err) + } + if a.Node != node || a.Node+"."+a.Module != key { + return "", Plan{}, fmt.Errorf("the assignment %s names node %q and module %q, not its key's", key, a.Node, + a.Module) + } + if len(a.Provisioning) == 0 || string(a.Provisioning) == "null" { + return a.Module, Plan{}, errNoPlan + } + var p Plan + dec := json.NewDecoder(bytes.NewReader(a.Provisioning)) + dec.DisallowUnknownFields() + if err := dec.Decode(&p); err != nil { + return a.Module, Plan{}, fmt.Errorf("the plan of %s cannot be read: %w", a.Module, err) + } + if wrong := p.problems(); len(wrong) > 0 { + return a.Module, Plan{}, fmt.Errorf("the plan of %s cannot be acted on: %s", a.Module, + strings.Join(wrong, "; ")) + } + return a.Module, p, nil +} + +func (p Plan) problems() []string { + var out []string + if p.Templates.Source == "" { + out = append(out, "it names no source for its templates") + } + if !digestShape.MatchString(p.Templates.Digest) { + out = append(out, fmt.Sprintf("its templates' digest %q is not sha256:<64 hex>", p.Templates.Digest)) + } + if len(p.Files) == 0 && len(p.Items) == 0 { + out = append(out, "it fills no file and contributes no item") + } + seen := map[string]bool{} + for _, f := range p.Files { + switch { + case !filepath.IsAbs(f.Path) || filepath.Clean(f.Path) != f.Path: + out = append(out, fmt.Sprintf("%s: %q is not a clean absolute path", f.ID, f.Path)) + case seen[f.Path]: + out = append(out, fmt.Sprintf("%s: %s is named twice", f.ID, f.Path)) + } + seen[f.Path] = true + if !plainTemplate(f.Template) { + out = append(out, fmt.Sprintf("%s: template %q is not a plain path inside the templates archive", f.ID, + f.Template)) + } + if !slices.Contains(ownerWords, f.OwnerAtRuntime) { + out = append(out, fmt.Sprintf("%s: owner-at-runtime %q is not owned, seed or managed", f.ID, + f.OwnerAtRuntime)) + } + if f.Takes != "" && !slices.Contains(takeWords, f.Takes) { + out = append(out, fmt.Sprintf("%s: takes %q is not restart-on, reload-on or read-on-use", f.ID, f.Takes)) + } + } + named := map[string]bool{} + for _, sec := range p.Secrets { + if !secretShape.MatchString(sec.Name) { + out = append(out, fmt.Sprintf("secret %q is not a secret's name", sec.Name)) + } + if named[sec.Name] { + out = append(out, fmt.Sprintf("secret %s is named twice", sec.Name)) + } + named[sec.Name] = true + if sec.Path == "" { + continue + } + switch { + case !filepath.IsAbs(sec.Path) || filepath.Clean(sec.Path) != sec.Path: + out = append(out, fmt.Sprintf("secret %s: %q is not a clean absolute path", sec.Name, sec.Path)) + case seen[sec.Path]: + out = append(out, fmt.Sprintf("secret %s: %s is named twice", sec.Name, sec.Path)) + } + seen[sec.Path] = true + } + bound := map[[2]string]bool{} + for _, pr := range p.Provisions { + which := pr.Name + if which == "" { + which = pr.Provision + " " + pr.Local + } + switch { + case pr.Name == "": + case !secretShape.MatchString(pr.Name): + out = append(out, fmt.Sprintf("provision credential %q is not a secret's name", pr.Name)) + case named[pr.Name]: + out = append(out, fmt.Sprintf("secret %s is named twice", pr.Name)) + } + if pr.Name != "" { + named[pr.Name] = true + } + if !secretShape.MatchString(pr.Provision) { + out = append(out, fmt.Sprintf("%s: provision %q is not a provision's name", which, pr.Provision)) + } + if !nodeShape.MatchString(pr.Provider) { + out = append(out, fmt.Sprintf("%s: provider %q is not a machine's name", which, pr.Provider)) + } + if pr.CredentialForm != "" && !slices.Contains(credentialForms, pr.CredentialForm) { + out = append(out, fmt.Sprintf("%s: credential-form %q is not %s", which, pr.CredentialForm, + strings.Join(credentialForms, ", "))) + } + if key := [2]string{pr.Provision, pr.Local}; bound[key] { + out = append(out, fmt.Sprintf("%s: its provider is named twice", which)) + } else { + bound[key] = true + } + if pr.Path == "" { + continue + } + if pr.Name == "" { + out = append(out, fmt.Sprintf("%s: %s is a file with no credential to fill it with", which, pr.Path)) + continue + } + switch { + case !filepath.IsAbs(pr.Path) || filepath.Clean(pr.Path) != pr.Path: + out = append(out, fmt.Sprintf("secret %s: %q is not a clean absolute path", pr.Name, pr.Path)) + case seen[pr.Path]: + out = append(out, fmt.Sprintf("secret %s: %s is named twice", pr.Name, pr.Path)) + } + seen[pr.Path] = true + } + contributed := map[string]bool{} + for _, it := range p.Items { + if !collectionShape.MatchString(it.Collection) { + out = append(out, fmt.Sprintf("item %q is not a collection's name", it.Collection)) + } + if it.Name != "" && !itemNameShape.MatchString(it.Name) { + out = append(out, fmt.Sprintf("item %s: %q is not a name", it.Collection, it.Name)) + } + if contributed[it.key()] { + out = append(out, fmt.Sprintf("item %s is named twice: a module contributes one item to a collection, "+ + "or several each with a name of its own", it.key())) + } + contributed[it.key()] = true + for _, k := range sortedKeys(it.Attributes) { + if v := it.Attributes[k]; !attributeFits(it.Collection, k, v) { + out = append(out, fmt.Sprintf("item %s: attribute %s=%q is not a name and a value", it.key(), k, v)) + } else if n, err := strconv.Atoi(v); k == OrderAttribute && (err != nil || n < 0 || n > 99) { + out = append(out, fmt.Sprintf("item %s: order=%q is not a whole number from 0 to 99", it.key(), v)) + } + } + if !plainTemplate(it.Template) { + out = append(out, fmt.Sprintf("item %s: template %q is not a plain path inside the templates archive", + it.Collection, it.Template)) + } + } + collected := map[string]bool{} + for _, c := range p.Collections { + if !collectionShape.MatchString(c.Name) { + out = append(out, fmt.Sprintf("collection %q is not a collection's name", c.Name)) + } + if collected[c.Name] { + out = append(out, fmt.Sprintf("collection %s is named twice", c.Name)) + } + collected[c.Name] = true + if strings.ContainsAny(c.Comment, "\n\r") { + out = append(out, fmt.Sprintf("collection %s: its comment holds a line break", c.Name)) + } + } + for k := range p.Placement { + kind, name, ok := strings.Cut(k, ":") + if !ok || name == "" || !slices.Contains(placementKinds, kind) { + out = append(out, fmt.Sprintf("placement %q is not : of a placement kind", k)) + } + } + slices.Sort(out) + return out +} + +// plainTemplate is whether a template's path is a plain path inside the templates archive. +func plainTemplate(t string) bool { + return t != "" && !strings.HasPrefix(t, "/") && path.Clean(t) == t && t != ".." && !strings.HasPrefix(t, "../") +} diff --git a/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/providers_test.go b/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/providers_test.go new file mode 100644 index 00000000..4841a24c --- /dev/null +++ b/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/providers_test.go @@ -0,0 +1,104 @@ +package main + +import ( + "reflect" + "strings" + "testing" +) + +// A plan names, for each provision its module requires, the provider serving it (design 57 V5a, issue 760): the +// client reads that strictly, before the controller writes it, and fills nothing from an entry with no credential; +// the vault acts on it. +func TestAPlanNamingItsProvidersIsRead(t *testing.T) { + value := `{"node":"g14","module":"app","provisioning":{` + + `"templates":{"source":"x","digest":"sha256:` + strings.Repeat("0", 64) + `"},` + + `"files":[{"id":"conf","path":"/etc/app.conf","template":"app.conf","owner-at-runtime":"owned"}],` + + `"provisions":[{"provision":"postgres-database","provider":"novox","provider-module":"postgres"},` + + `{"provision":"s3-bucket","local":"hot","provider":"ace","provider-module":"minio"},` + + `{"name":"forge-read","provision":"forge-read","provider":"novox","provider-module":"forge",` + + `"path":"/var/lib/app/forge.secret"}]}}` + _, p, err := planOf("g14", "g14.app", []byte(value)) + if err != nil { + t.Fatal(err) + } + want := []PlannedProvision{{Provision: "postgres-database", Provider: "novox", ProviderModule: "postgres"}, + {Provision: "s3-bucket", Local: "hot", Provider: "ace", ProviderModule: "minio"}, + {Name: "forge-read", Provision: "forge-read", Provider: "novox", ProviderModule: "forge", + Path: "/var/lib/app/forge.secret"}} + if len(p.Provisions) != len(want) { + t.Fatalf("the plan's provisions are read as %+v, want %+v", p.Provisions, want) + } + for i := range want { + if !reflect.DeepEqual(p.Provisions[i], want[i]) { + t.Fatalf("the plan's provisions are read as %+v, want %+v", p.Provisions, want) + } + } + // Only the credential is a file to fill: an entry naming a provider alone fills nothing. + targets, why := fillTargets(p, map[string]string{"app.conf": "x"}) + if why != "" || len(targets) != 2 || targets[1].Path != "/var/lib/app/forge.secret" { + t.Fatalf("the files filled are %+v (%s)", targets, why) + } + for why, provisions := range map[string][]PlannedProvision{ + "a file and no credential's name": {{Provision: "p", Provider: "a", ProviderModule: "m", Path: "/var/lib/p"}}, + "one provision and local twice": {{Provision: "p", Local: "l", Provider: "a", ProviderModule: "m"}, + {Provision: "p", Local: "l", Provider: "b", ProviderModule: "m"}}, + "no provider": {{Provision: "p", ProviderModule: "m"}}, + } { + bad := p + bad.Provisions = provisions + if len(bad.problems()) == 0 { + t.Errorf("a plan with %s was taken", why) + } + } +} + +// Each entry names the credential form of its provider's offer (issue 768), which the vault acts on: the client +// reads it strictly, before the controller writes it, and refuses a form it does not know. +func TestAPlanNamingEachProvidersCredentialFormIsRead(t *testing.T) { + value := `{"node":"g14","module":"app","provisioning":{` + + `"templates":{"source":"x","digest":"sha256:` + strings.Repeat("0", 64) + `"},` + + `"files":[{"id":"conf","path":"/etc/app.conf","template":"app.conf","owner-at-runtime":"owned"}],` + + `"provisions":[{"provision":"postgres-database","provider":"novox","provider-module":"postgres",` + + `"credential-form":"pair"},` + + `{"provision":"sonarr-api","provider":"ace","provider-module":"sonarr","credential-form":"shared"},` + + `{"name":"forge-read","provision":"forge-read","provider":"novox","provider-module":"forge",` + + `"credential-form":"issued","path":"/var/lib/app/forge.secret"},` + + `{"provision":"smtp-relay","provider":"ace","provider-module":"relay","credential-form":"given"}]}}` + _, p, err := planOf("g14", "g14.app", []byte(value)) + if err != nil { + t.Fatal(err) + } + var forms []string + for _, pr := range p.Provisions { + forms = append(forms, pr.CredentialForm) + } + if strings.Join(forms, ",") != "pair,shared,issued,given" { + t.Fatalf("the credential forms are read as %v", forms) + } + bad := strings.Replace(value, `"credential-form":"pair"`, `"credential-form":"one-for-all"`, 1) + if _, _, err := planOf("g14", "g14.app", []byte(bad)); err == nil || !strings.Contains(err.Error(), "one-for-all") { + t.Fatalf("a credential form the client does not know was taken: %v", err) + } +} + +// A consumer's plan entry may carry what its module contributes to the provider (issue 776): the client reads it +// strictly, keeps it on the entry, and still refuses any other field it does not know. +func TestAPlanEntryCarryingItsContributionIsRead(t *testing.T) { + value := `{"node":"g14","module":"app","provisioning":{` + + `"templates":{"source":"x","digest":"sha256:` + strings.Repeat("0", 64) + `"},` + + `"files":[{"id":"conf","path":"/etc/app.conf","template":"app.conf","owner-at-runtime":"owned"}],` + + `"provisions":[{"provision":"postgres-database","provider":"novox","provider-module":"postgres",` + + `"contributes":{"name":"nextcloud","extensions":["pg_trgm"]}}]}}` + _, p, err := planOf("g14", "g14.app", []byte(value)) + if err != nil { + t.Fatal(err) + } + want := map[string]any{"name": "nextcloud", "extensions": []any{"pg_trgm"}} + if len(p.Provisions) != 1 || !reflect.DeepEqual(p.Provisions[0].Contributes, want) { + t.Fatalf("the plan's provisions are read as %+v, want the contribution %v kept", p.Provisions, want) + } + bad := strings.Replace(value, `"contributes":`, `"contribution":`, 1) + if _, _, err := planOf("g14", "g14.app", []byte(bad)); err == nil { + t.Error("a plan entry with a field the client does not know was taken") + } +} diff --git a/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/pulled_in_test.go b/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/pulled_in_test.go new file mode 100644 index 00000000..34791bf9 --- /dev/null +++ b/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/pulled_in_test.go @@ -0,0 +1,78 @@ +package main + +import ( + "encoding/json" + "fmt" + "strings" + "testing" + "time" +) + +// A module only pulled in by another on this node has a value of its own in the assignments mirror (mesh-controller +// internal/inventory AssignmentValues, issue 765): its plan, what requires it, and no time it was assigned. The +// client fills it exactly like an assigned module: its files, its secrets from the vault and its items. + +// pullIn writes a module's value as the controller writes one only pulled in, by requiredBy. +func (b *aBus) pullIn(t *testing.T, module, requiredBy string, plan Plan) { + t.Helper() + p, err := json.Marshal(plan) + if err != nil { + t.Fatal(err) + } + b.put(t, AssignmentsBucket, node+"."+module, fmt.Sprintf(`{"node":%q,"module":%q,"required-by":%q,`+ + `"provisioning":%s}`, node, module, requiredBy, p)) +} + +func TestAPulledInModulesValueIsReadForItsPlan(t *testing.T) { + plan := contributor(t, "shell:zsh:normal", "alias nox=mesh-cli") + p, err := json.Marshal(plan) + if err != nil { + t.Fatal(err) + } + value := fmt.Sprintf(`{"node":"g14","module":"mesh-cli","required-by":"desk","provisioning":%s}`, p) + module, got, err := planOf("g14", "g14.mesh-cli", []byte(value)) + if err != nil || module != "mesh-cli" || len(got.Items) != 1 || got.Items[0].Collection != "shell:zsh:normal" { + t.Fatalf("a pulled-in module's value reads as %s %+v (%v)", module, got, err) + } + // Still strict in the plan. + spoilt := strings.Replace(value, `"items":`, `"surprise":1,"items":`, 1) + if _, _, err := planOf("g14", "g14.mesh-cli", []byte(spoilt)); err == nil { + t.Fatalf("a pulled-in module's plan with an unknown field is acted on: %s", spoilt) + } +} + +func TestNatsAPulledInModuleIsFilledWithItsSecretAndItsItemIsRecorded(t *testing.T) { + b := startBus(t) + heard := b.listen(t) + vault := startVault(t, b) + const token, password = "tok-765-never-on-the-bus", "pw-765-never-on-the-bus" + vault.keep("own/g14/app/app-token", token, false) + vault.keep("own/g14/app/db-password", password, false) + e := startEngine(t, map[string]string{appConf: "owned", appDB: "owned", zshrc: "owned"}) + b.config(t, "app", "font-size", "13") + b.pullIn(t, "app", "desk", app(t)) + b.assign(t, "zsh", zshCollector(t)) + b.pullIn(t, "mesh-cli", "desk", contributor(t, "shell:zsh:normal", "alias nox=mesh-cli\n")) + aClient(t, b, e, t.TempDir(), time.Hour) + + eventually(t, "the pulled-in app.conf is filled with its secret", func() bool { + return e.read(t, appConf) == "token = "+token+"\nfont = 13\n" + }) + eventually(t, "its secret's own file holds it alone", func() bool { return e.read(t, appDB) == password }) + eventually(t, "the pulled-in contributor's item is recorded", func() bool { + got := b.keys(t, CollectBucket, node+".") + return len(got) == 1 && got[0] == "g14.shell=3Azsh=3Anormal.mesh-cli" + }) + eventually(t, "zshrc holds the pulled-in contributor's item", func() bool { + return e.read(t, zshrc) == "# zsh\n# mesh-cli\nalias nox=mesh-cli\n# then\n# end\n" + }) + eventually(t, "prov.applied says app applied", func() bool { + r, _ := heard.last("app") + return r.Status == StatusApplied && len(r.Files) == 2 + }) + // What required it lets it go: its item leaves the collection. + b.del(t, AssignmentsBucket, node+".mesh-cli") + eventually(t, "zshrc loses the item of a module no longer pulled in", func() bool { + return e.read(t, zshrc) == "# zsh\n# then\n# end\n" + }) +} diff --git a/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/secrets.go b/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/secrets.go new file mode 100644 index 00000000..b2bba03c --- /dev/null +++ b/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/secrets.go @@ -0,0 +1,254 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "slices" + "strings" + "time" + + "git.novox.be/novox/mesh-sdk/go/seal" + "github.com/nats-io/nats.go" +) + +// A module's secrets, from the vault (novox/hq ADR 0349 clauses 7 and 11, design 57 V4, issue 647). +// +// **Asked for on request and reply, sealed to this client's key.** The client asks the vault's `get` in its own +// name, on the subject that names its account, so the vault knows from the bus's word which machine's client asks; +// it sends the public half of a key it made when it started and keeps in memory alone, and the vault answers the +// secret sealed to it (the sealed box of ADR 0206). Only the box crosses the bus. The value lives in this process +// while it fills the module's files, and reaches the machine only through the node-engine's local write socket. +// It is never written to a bucket, a report, an event or a log. +// +// **Asked once, and again when it changes.** A secret the client holds is asked again when the vault says it +// changed (`secret-changed`: which, by whom, its generation, never a value), so a rotation reaches the file +// without a new declaration; and every time the client starts or its node comes back, when it starts from nothing. +// A secret the vault does not hold yet, or a vault that does not answer, is waited for; a refusal fails the module +// with the vault's reason, and is asked again only after RefusedFor, so a refused client does not fill the vault's +// record of hand-outs every minute. + +// VaultSeat is the vault's seat, and GetVerb the verb a client asks. +const ( + VaultSeat = "mesh-vault" + GetVerb = "get" +) + +// SecretChangedSubject is where the vault says a secret changed (mesh-controller link.EventSubject of mesh-vault's +// `secret-changed`): never a value. +const SecretChangedSubject = "mesh.mod.mesh-vault.event.secret-changed" + +// RefusedFor is how long a refusal stands before the vault is asked again. +const RefusedFor = 10 * time.Minute + +// VaultName is a module's own secret on one machine as the vault names it (mesh-vault intake.go OwnSecret). +func VaultName(node, module, name string) string { return "own/" + node + "/" + module + "/" + name } + +// HolderName is the value a provider's software issued for one holder, a module on one machine, as the vault names +// it (mesh-vault perholder.go HolderSecret, issue 749). +func HolderName(provider, provision, node, module string) string { + return "provision/" + provider + "/" + provision + "/" + node + "/" + module +} + +// callerToken is a bus user's name as the last token of a call it publishes, each dot written `~` (mesh-controller +// internal/broker CallerToken). +func callerToken(user string) string { return strings.ReplaceAll(user, ".", "~") } + +// GetSubject is where this client asks the vault's `get` in its own name (mesh-controller internal/broker +// CalledSubject of `mesh.seat.mesh-vault.tool.get`). +func GetSubject(user string) string { + return "mesh.seat." + VaultSeat + ".call." + GetVerb + "." + callerToken(user) +} + +// errNotYet is a secret the vault cannot hand yet: it holds none of that name, or it does not answer. +var errNotYet = errors.New("not yet") + +// Secrets hands a module's secret, opened. +type Secrets interface { + // Get is the secret's value and generation, errNotYet (wrapped, with why) when it is to be waited for, or why + // it was refused. + Get(ctx context.Context, name string) (value string, generation int, err error) +} + +// VaultSecrets asks the vault over the bus. +type VaultSecrets struct { + Conn *nats.Conn + // User is the client's own bus account, `provisioner.`. + User string + // Key is the client's own key, made at start and never written down. + Key seal.KeyPair + // Within bounds one ask. + Within time.Duration +} + +// NewVaultSecrets is the vault reached as user, with a key of its own made now. +func NewVaultSecrets(conn *nats.Conn, user string) (*VaultSecrets, error) { + key, err := seal.GenerateKeyPair() + if err != nil { + return nil, fmt.Errorf("the client could not make its key: %w", err) + } + return &VaultSecrets{Conn: conn, User: user, Key: key, Within: 15 * time.Second}, nil +} + +// seatAnswer is a seat's answer on the bus (mesh-controller link.Answer). +type seatAnswer struct { + Result json.RawMessage `json:"result,omitempty"` + Error string `json:"error,omitempty"` +} + +// vaultAnswer is `get`'s result (mesh-vault vault.go Answer): never a value in the clear. +type vaultAnswer struct { + Name string `json:"name"` + Generation int `json:"generation"` + Sealed seal.Box `json:"sealed"` +} + +// Get asks the vault for name, sealed to the client's key, and opens it. +func (v *VaultSecrets) Get(ctx context.Context, name string) (string, int, error) { + body, err := json.Marshal(map[string]string{"name": name, "public_key": v.Key.PublicKey}) + if err != nil { + return "", 0, err + } + within := v.Within + if within <= 0 { + within = 15 * time.Second + } + asking, cancel := context.WithTimeout(ctx, within) + defer cancel() + msg, err := v.Conn.RequestWithContext(asking, GetSubject(v.User), body) + switch { + case errors.Is(err, nats.ErrNoResponders): + return "", 0, fmt.Errorf("%w: nothing answers the vault's get", errNotYet) + case errors.Is(err, context.DeadlineExceeded), errors.Is(err, nats.ErrTimeout): + return "", 0, fmt.Errorf("%w: the vault did not answer within %s", errNotYet, within) + case err != nil: + return "", 0, fmt.Errorf("asking the vault: %w", err) + } + var a seatAnswer + if err := json.Unmarshal(msg.Data, &a); err != nil { + return "", 0, errors.New("the vault answered something that is not a seat's answer") + } + if a.Error != "" { + if strings.Contains(a.Error, "holds no secret named") { + return "", 0, fmt.Errorf("%w: the vault holds no %s yet", errNotYet, name) + } + return "", 0, fmt.Errorf("the vault refused it: %s", a.Error) + } + var got vaultAnswer + if err := json.Unmarshal(a.Result, &got); err != nil || got.Name != name { + return "", 0, fmt.Errorf("the vault answered something that is not %s", name) + } + value, err := seal.Open(got.Sealed, v.Key.PrivateKey) + if err != nil { + return "", 0, fmt.Errorf("the vault's answer for %s does not open with the client's key: %w", name, err) + } + return value, got.Generation, nil +} + +// heldSecret is a secret as one run of the client holds it: its value, or why it was refused and when. +type heldSecret struct { + value string + generation int + refused string + refusedAt time.Time +} + +// changedSecret is what `secret-changed` says, as far as the client reads it. +type changedSecret struct { + Name string `json:"name"` + Generation int `json:"generation"` +} + +// secretOf is a module's secret, from what the run holds or from the vault: its value, or what it waits for, or +// why it cannot be filled. +func (c *Client) secretOf(ctx context.Context, s *state, module string, p Plan, n Need) (value, wait, fail string) { + name := "" + if slices.ContainsFunc(p.Secrets, func(sec PlannedSecret) bool { return sec.Name == n.Name }) { + name = VaultName(c.Node, module, n.Name) + } else if i := slices.IndexFunc(p.Provisions, func(pr PlannedProvision) bool { return pr.Name == n.Name }); i >= 0 { + // A credential of what the module requires, issued for this module on this machine (issue 757). + pr := p.Provisions[i] + name = HolderName(pr.Provider, pr.Provision, c.Node, module) + if pr.CredentialForm == "pair" { + // This consumer's own login from a provider that gives each consumer one (issue 779, ADR 0158 §11). + name = PairName(pr.Provider, pr.Provision, c.Node, module, pr.Local) + } + } else { + return "", "", fmt.Sprintf("%s: the plan names no secret %s, and the vault hands only what this machine's "+ + "declaration names", n, n.Name) + } + return c.vaultValue(ctx, s, n, name) +} + +// vaultValue is the vault's secret name for the need n, from what the run holds or from the vault: its value, or what +// it waits for, or why it cannot be filled. +func (c *Client) vaultValue(ctx context.Context, s *state, n Need, name string) (value, wait, fail string) { + if c.Secrets == nil { + return "", "", n.String() + ": this client has no way to ask the vault" + } + if h, ok := s.secrets[name]; ok { + if h.refused == "" { + return h.value, "", "" + } + if c.now().Sub(h.refusedAt) < RefusedFor { + return "", "", fmt.Sprintf("%s: %s", n, h.refused) + } + } + got, generation, err := c.Secrets.Get(ctx, name) + switch { + case errors.Is(err, errNotYet): + delete(s.secrets, name) + return "", fmt.Sprintf("%s (%s)", n, strings.TrimPrefix(err.Error(), errNotYet.Error()+": ")), "" + case err != nil: + s.secrets[name] = heldSecret{refused: err.Error(), refusedAt: c.now()} + return "", "", fmt.Sprintf("%s: %v", n, err) + } + s.secrets[name] = heldSecret{value: got, generation: generation} + return got, "", "" +} + +// secretChanged folds what the vault said changed into the state: a secret this run holds is asked again. It +// answers the module whose secret it is, or nothing. +func (c *Client) secretChanged(s *state, body []byte) []string { + var ch changedSecret + if err := json.Unmarshal(body, &ch); err != nil { + return nil + } + parts := strings.Split(ch.Name, "/") + var module string + switch { + case len(parts) == 4 && parts[0] == "own" && parts[1] == c.Node: + module = parts[2] + case len(parts) == 5 && parts[0] == "provision" && parts[3] == c.Node: + // A value issued for one of this machine's modules (issue 757). + module = parts[4] + case (len(parts) == 5 || len(parts) == 6) && parts[0] == "pair" && (parts[1] == c.Node || parts[3] == c.Node): + // A consumer's own credential (ADR 0373): from a provider on this machine, every provider here listing its + // consumers is filled again, which asks for it if it lists that consumer, held or waited for; and for a module + // on this machine, the module whose plan names it as its file's credential (issue 779). + if h, held := s.secrets[ch.Name]; held && h.refused == "" && h.generation >= ch.Generation { + return nil + } + delete(s.secrets, ch.Name) + var out []string + if parts[1] == c.Node { + out = sortedKeys(s.providing) + } + if parts[3] == c.Node && hasPlan(s, parts[4]) && !slices.Contains(out, parts[4]) { + out = append(out, parts[4]) + } + return out + default: + return nil + } + if !hasPlan(s, module) { + return nil + } + h, held := s.secrets[ch.Name] + if held && h.refused == "" && h.generation >= ch.Generation { + return nil + } + delete(s.secrets, ch.Name) + return []string{module} +} diff --git a/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/secrets_test.go b/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/secrets_test.go new file mode 100644 index 00000000..d2d2dda3 --- /dev/null +++ b/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/secrets_test.go @@ -0,0 +1,511 @@ +package main + +import ( + "context" + "encoding/json" + "strings" + "sync" + "testing" + "time" + + "git.novox.be/novox/mesh-sdk/go/seal" + "github.com/nats-io/nats.go" +) + +// A node's secrets come from the vault (novox/hq ADR 0349 clauses 7 and 11, design 57 V4, issue 647), against a +// real bus whose accounts are the client's as mesh-controller composes them: a moved module's file is filled with +// its secret from the vault's get, asked in the client's own name and answered sealed to its key; the value never +// crosses the bus in the clear; a secret rotated in the vault reaches its file without a new declaration; a secret +// the vault does not hold yet is waited for, and a refusal fails the module with the vault's reason. +// +// The vault is a stand-in answering `mesh-vault.get` as its seat does on the bus: on the subject naming the caller, +// sealing to the key the caller sends, handing only what it allows that caller (the vault's own tests prove its +// rule), and saying `secret-changed` with no value. + +// standInVault answers get on the bus, for provisioner.g14 alone and only the names it allows. +type standInVault struct { + t *testing.T + conn *nats.Conn + mu sync.Mutex + held map[string]string + gen map[string]int + allowed map[string]bool + asked map[string]int + refuse string +} + +func startVault(t *testing.T, b *aBus) *standInVault { + t.Helper() + v := &standInVault{t: t, conn: b.conn, held: map[string]string{}, gen: map[string]int{}, allowed: map[string]bool{}, + asked: map[string]int{}} + sub, err := b.conn.Subscribe("mesh.seat.mesh-vault.call.get.*", func(m *nats.Msg) { + caller := strings.ReplaceAll(m.Subject[strings.LastIndex(m.Subject, ".")+1:], "~", ".") + var args struct { + Name string `json:"name"` + PublicKey string `json:"public_key"` + } + _ = json.Unmarshal(m.Data, &args) + v.mu.Lock() + defer v.mu.Unlock() + v.asked[args.Name]++ + answer := map[string]any{} + value, held := v.held[args.Name] + switch { + case caller != "provisioner."+node || !v.allowed[args.Name]: + answer["error"] = caller + " may not ask for " + args.Name + ": not on the list" + case v.refuse != "": + answer["error"] = v.refuse + case !held: + answer["error"] = "the vault holds no secret named " + args.Name + default: + box, err := seal.Seal(value, args.PublicKey) + if err != nil { + answer["error"] = err.Error() + break + } + answer["result"] = map[string]any{"name": args.Name, "generation": v.gen[args.Name], + "changed-at": time.Now().UTC(), "sealed": box} + } + body, _ := json.Marshal(answer) + _ = m.Respond(body) + }) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = sub.Unsubscribe() }) + _ = b.conn.Flush() + return v +} + +// keep holds value as name and allows the client to ask for it; with say, it says secret-changed as the vault does. +func (v *standInVault) keep(name, value string, say bool) { + v.mu.Lock() + v.held[name] = value + v.gen[name]++ + v.allowed[name] = true + gen := v.gen[name] + v.mu.Unlock() + if say { + body, _ := json.Marshal(map[string]any{"name": name, "by": "the test", "at": time.Now().UTC(), "generation": gen}) + if err := v.conn.Publish(SecretChangedSubject, body); err != nil { + v.t.Fatal(err) + } + } +} + +func (v *standInVault) times(name string) int { + v.mu.Lock() + defer v.mu.Unlock() + return v.asked[name] +} + +// sniffer hears every message on the bus, to prove a value never crosses it in the clear. +type sniffer struct { + mu sync.Mutex + seen []string +} + +func sniff(t *testing.T, b *aBus) *sniffer { + t.Helper() + s := &sniffer{} + sub, err := b.conn.Subscribe(">", func(m *nats.Msg) { + s.mu.Lock() + s.seen = append(s.seen, m.Subject+" "+string(m.Data)) + s.mu.Unlock() + }) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = sub.Unsubscribe() }) + _ = b.conn.Flush() + return s +} + +func (s *sniffer) carrying(value string) []string { + s.mu.Lock() + defer s.mu.Unlock() + var out []string + for _, m := range s.seen { + if strings.Contains(m, value) { + out = append(out, m) + } + } + return out +} + +func (s *sniffer) heard() int { + s.mu.Lock() + defer s.mu.Unlock() + return len(s.seen) +} + +const appConf, appDB = "/etc/app/app.conf", "/var/lib/app/db-password" + +// app is a moved module with a template naming a secret and a config value, and a secret kept in a file of its own. +func app(t *testing.T) Plan { + tpl, _ := templates(t, map[string]string{"app.conf": "token = ${secret:app-token}\nfont = ${config:font-size}\n"}) + return Plan{Templates: tpl, Files: []PlannedFile{{ID: "conf", Path: appConf, Template: "app.conf", + OwnerAtRuntime: "owned", Takes: "restart-on"}}, + Secrets: []PlannedSecret{{Name: "app-token"}, {Name: "db-password", Path: appDB}}} +} + +func TestNatsAMovedModulesFileIsFilledWithItsSecretFromTheVault(t *testing.T) { + b := startBus(t) + heard := b.listen(t) + wire := sniff(t, b) + vault := startVault(t, b) + const token, password = "tok-3f9a1c-never-on-the-bus", "pw-77b2e0-never-on-the-bus" + vault.keep("own/g14/app/app-token", token, false) + vault.keep("own/g14/app/db-password", password, false) + e := startEngine(t, map[string]string{appConf: "owned", appDB: "owned"}) + b.config(t, "app", "font-size", "13") + b.assign(t, "app", app(t)) + aClient(t, b, e, t.TempDir(), time.Hour) + + eventually(t, "app.conf is filled with its secret", func() bool { + return e.read(t, appConf) == "token = "+token+"\nfont = 13\n" + }) + eventually(t, "the secret's own file holds it alone", func() bool { return e.read(t, appDB) == password }) + eventually(t, "prov.applied says applied", func() bool { + r, _ := heard.last("app") + return r.Status == StatusApplied && len(r.Files) == 2 + }) + eventually(t, "the needs name the secrets, never their values", func() bool { + got := b.keys(t, NeedsBucket, node+".app.") + return strings.Join(got, " ") == "g14.app.config.font-size g14.app.secret.app-token g14.app.secret.db-password" + }) + if wire.heard() == 0 { + t.Fatal("the sniffer heard nothing: it proves nothing") + } + for _, value := range []string{token, password} { + if said := wire.carrying(value); len(said) > 0 { + t.Errorf("the value crossed the bus in the clear: %q", said) + } + } + if vault.times("own/g14/app/app-token") != 1 || vault.times("own/g14/app/db-password") != 1 { + t.Errorf("the vault was asked %d and %d times, want once each", vault.times("own/g14/app/app-token"), + vault.times("own/g14/app/db-password")) + } +} + +func TestNatsASecretRotatedInTheVaultReachesItsFileWithoutANewDeclaration(t *testing.T) { + b := startBus(t) + heard := b.listen(t) + vault := startVault(t, b) + vault.keep("own/g14/app/app-token", "first", false) + vault.keep("own/g14/app/db-password", "pw", false) + e := startEngine(t, map[string]string{appConf: "owned", appDB: "owned"}) + b.config(t, "app", "font-size", "13") + b.assign(t, "app", app(t)) + aClient(t, b, e, t.TempDir(), time.Hour) + eventually(t, "app.conf is filled", func() bool { return strings.Contains(e.read(t, appConf), "token = first\n") }) + assigned, err := b.kv[AssignmentsBucket].Get(context.Background(), node+".app") + if err != nil { + t.Fatal(err) + } + + vault.keep("own/g14/app/app-token", "second", true) + eventually(t, "the rotated value reaches its file", func() bool { + return strings.Contains(e.read(t, appConf), "token = second\n") + }) + eventually(t, "prov.applied says it was updated", func() bool { + r, _ := heard.last("app") + return r.Status == StatusApplied && r.Files[0].Action == "updated" + }) + now, err := b.kv[AssignmentsBucket].Get(context.Background(), node+".app") + if err != nil || now.Revision() != assigned.Revision() { + t.Errorf("the module's assignment was written again: revision %d, was %d", now.Revision(), assigned.Revision()) + } + if moved, _ := e.counts(appConf); moved != 2 { + t.Errorf("app.conf moved %d times, want 2: the program restarts once for the rotation", moved) + } + if moved, _ := e.counts(appDB); moved != 1 { + t.Errorf("the other secret's file moved %d times: only what changed is written", moved) + } + // What another machine's secret, or a secret no plan here names, changes is no reason to ask. + vault.keep("own/ace/app/app-token", "elsewhere", true) + vault.keep("own/g14/other/token", "unplanned", true) + time.Sleep(300 * time.Millisecond) + if n := vault.times("own/g14/app/app-token"); n != 2 { + t.Errorf("the vault was asked for app-token %d times, want 2", n) + } + if vault.times("own/ace/app/app-token")+vault.times("own/g14/other/token") != 0 { + t.Error("a change of a secret this machine's plans do not name made the client ask") + } +} + +func TestNatsASecretTheVaultDoesNotHoldYetIsWaitedFor(t *testing.T) { + b := startBus(t) + heard := b.listen(t) + vault := startVault(t, b) + vault.keep("own/g14/app/db-password", "pw", false) + vault.allowed["own/g14/app/app-token"] = true // the plan names it; the vault does not hold it yet + e := startEngine(t, map[string]string{appConf: "owned", appDB: "owned"}) + b.config(t, "app", "font-size", "13") + b.assign(t, "app", app(t)) + aClient(t, b, e, t.TempDir(), time.Hour) + + eventually(t, "prov.applied says it waits for the secret", func() bool { + r, _ := heard.last("app") + return r.Status == StatusWaiting && len(r.WaitsFor) == 1 && + strings.HasPrefix(r.WaitsFor[0], "secret:app-token (the vault holds no") + }) + if _, asked := e.counts(appConf); asked != 0 { + t.Error("a file waiting for its secret was written") + } + if _, asked := e.counts(appDB); asked != 0 { + t.Error("a module waiting for one value had another file written: every value or none") + } + vault.keep("own/g14/app/app-token", "arrived", true) + eventually(t, "it is filled once the vault holds it", func() bool { + return strings.Contains(e.read(t, appConf), "token = arrived\n") && e.read(t, appDB) == "pw" + }) +} + +func TestNatsARefusalFailsTheModuleWithTheVaultsReasonAndIsNotAskedEveryPass(t *testing.T) { + b := startBus(t) + heard := b.listen(t) + vault := startVault(t, b) + vault.keep("own/g14/app/app-token", "tok", false) + vault.keep("own/g14/app/db-password", "pw", false) + vault.refuse = "provisioner.g14 may not ask for own/g14/app/app-token: app's plan on g14 names no secret app-token" + e := startEngine(t, map[string]string{appConf: "owned", appDB: "owned"}) + b.config(t, "app", "font-size", "13") + b.assign(t, "app", app(t)) + aClient(t, b, e, t.TempDir(), 100*time.Millisecond) + + eventually(t, "prov.applied says it failed, with the vault's reason", func() bool { + r, _ := heard.last("app") + return r.Status == StatusFailed && strings.Contains(r.Reason, "names no secret app-token") + }) + time.Sleep(600 * time.Millisecond) + if n := vault.times("own/g14/app/app-token"); n != 1 { + t.Errorf("a refused secret was asked %d times in six drift passes, want once", n) + } + if _, asked := e.counts(appConf); asked != 0 { + t.Error("a file whose secret was refused was written") + } +} + +func TestNatsAVaultThatDoesNotAnswerIsWaitedFor(t *testing.T) { + b := startBus(t) + heard := b.listen(t) + e := startEngine(t, map[string]string{appConf: "owned", appDB: "owned"}) + b.config(t, "app", "font-size", "13") + b.assign(t, "app", app(t)) + aClient(t, b, e, t.TempDir(), time.Hour) + eventually(t, "prov.applied says it waits for the vault", func() bool { + r, _ := heard.last("app") + return r.Status == StatusWaiting && len(r.WaitsFor) == 2 && strings.Contains(r.WaitsFor[0], "nothing answers") + }) +} + +// The plan's secrets are read strictly: a name that is not a secret's, one named twice, and a file path that is not +// clean and absolute, or a template's, each refuse the plan. +func TestAPlansSecretsAreReadStrictly(t *testing.T) { + base := func() Plan { + return Plan{Templates: Templates{Source: "x", Digest: "sha256:" + strings.Repeat("0", 64)}, + Files: []PlannedFile{{ID: "conf", Path: appConf, Template: "app.conf", OwnerAtRuntime: "owned"}}} + } + for why, secrets := range map[string][]PlannedSecret{ + "a name that is not a secret's": {{Name: "App Token"}}, + "a name given twice": {{Name: "t"}, {Name: "t"}}, + "a relative path": {{Name: "t", Path: "etc/t"}}, + "a template's path": {{Name: "t", Path: appConf}}, + } { + p := base() + p.Secrets = secrets + if len(p.problems()) == 0 { + t.Errorf("a plan with %s was taken", why) + } + } + p := base() + p.Secrets = []PlannedSecret{{Name: "app-token"}, {Name: "db-password", Path: appDB}} + if wrong := p.problems(); len(wrong) > 0 { + t.Errorf("a good plan was refused: %v", wrong) + } +} + +// The forge's read credential for claude-code on this machine (issue 757, issue 737): git's credential-store line +// the forge's software issued for this holder alone, kept by the vault as +// provision//gitea-read//claude-code. +const ( + forgeFile = "/var/lib/claude-code/state/gitea-read.secret" + settingsFile = "/etc/claude-code/managed-settings.json" + forgeHolder = "provision/novox/gitea-read/" + node + "/claude-code" +) + +// claudeCode is claude-code's plan with the credential of what it requires, gitea-read from novox, kept in a file of +// its own and named in a template. +func claudeCode(t *testing.T) Plan { + tpl, _ := templates(t, map[string]string{"managed-settings.json": `{"forge":"${secret:gitea-read}"}`}) + return Plan{Templates: tpl, Files: []PlannedFile{{ID: "settings", Path: settingsFile, + Template: "managed-settings.json", OwnerAtRuntime: "owned"}}, + Provisions: []PlannedProvision{{Name: "gitea-read", Provision: "gitea-read", Provider: "novox", + Path: forgeFile}}} +} + +func TestNatsAModulesRequiredCredentialIssuedForItIsFilledFromTheVaultAndFollowsARotation(t *testing.T) { + b := startBus(t) + heard := b.listen(t) + wire := sniff(t, b) + vault := startVault(t, b) + const first = "https://agent-g14:gta-first-never-on-the-bus@forge.example" + const second = "https://agent-g14:gta-second-never-on-the-bus@forge.example" + vault.keep(forgeHolder, first, false) + e := startEngine(t, map[string]string{forgeFile: "owned", settingsFile: "owned"}) + b.assign(t, "claude-code", claudeCode(t)) + aClient(t, b, e, t.TempDir(), time.Hour) + + eventually(t, "gitea-read's file holds the value issued for claude-code here, alone", func() bool { + return e.read(t, forgeFile) == first + }) + eventually(t, "the template naming it is filled with it", func() bool { + return e.read(t, settingsFile) == `{"forge":"`+first+`"}` + }) + eventually(t, "prov.applied says applied", func() bool { + r, _ := heard.last("claude-code") + return r.Status == StatusApplied && len(r.Files) == 2 + }) + vault.keep(forgeHolder, second, true) + eventually(t, "a value the forge issued anew reaches the file without a new declaration", func() bool { + return e.read(t, forgeFile) == second + }) + // Another machine's or another module's holder value changing is no reason to ask. + vault.keep("provision/novox/gitea-read/ace/claude-code", "elsewhere", true) + vault.keep("provision/novox/gitea-read/"+node+"/zsh", "unplanned", true) + time.Sleep(300 * time.Millisecond) + if n := vault.times(forgeHolder); n != 2 { + t.Errorf("the vault was asked for the holder value %d times, want 2", n) + } + if vault.times("provision/novox/gitea-read/ace/claude-code")+vault.times("provision/novox/gitea-read/"+node+"/zsh") != 0 { + t.Error("a change of a value no plan here names made the client ask") + } + if wire.heard() == 0 { + t.Fatal("the sniffer heard nothing: it proves nothing") + } + for _, value := range []string{first, second, "gta-first-never-on-the-bus", "gta-second-never-on-the-bus"} { + if said := wire.carrying(value); len(said) > 0 { + t.Errorf("the value crossed the bus in the clear: %q", said) + } + } +} + +// A required credential the vault refuses fails the module with the vault's reason, and nothing is written. +func TestNatsARequiredCredentialTheVaultRefusesFailsTheModule(t *testing.T) { + b := startBus(t) + heard := b.listen(t) + vault := startVault(t, b) + vault.keep(forgeHolder, "https://agent-g14:tok@forge.example", false) + vault.refuse = "provisioner.g14 may not ask for " + forgeHolder + ": claude-code's plan on g14 names no credential of gitea-read from novox" + e := startEngine(t, map[string]string{forgeFile: "owned", settingsFile: "owned"}) + b.assign(t, "claude-code", claudeCode(t)) + aClient(t, b, e, t.TempDir(), time.Hour) + eventually(t, "prov.applied says it failed, with the vault's reason", func() bool { + r, _ := heard.last("claude-code") + return r.Status == StatusFailed && strings.Contains(r.Reason, "names no credential of gitea-read") + }) + if _, asked := e.counts(forgeFile); asked != 0 { + t.Error("a file whose credential was refused was written") + } +} + +// The plan's provisions are read strictly. +func TestAPlansProvisionsAreReadStrictly(t *testing.T) { + base := func() Plan { + return Plan{Templates: Templates{Source: "x", Digest: "sha256:" + strings.Repeat("0", 64)}, + Files: []PlannedFile{{ID: "conf", Path: appConf, Template: "app.conf", OwnerAtRuntime: "owned"}}, + Secrets: []PlannedSecret{{Name: "own"}}} + } + good := PlannedProvision{Name: "gitea-read", Provision: "gitea-read", Provider: "novox", Path: forgeFile} + for why, change := range map[string]func(*PlannedProvision){ + "a name that is not a secret's": func(p *PlannedProvision) { p.Name = "Gitea Read" }, + "a name an own secret has": func(p *PlannedProvision) { p.Name = "own" }, + "no provision": func(p *PlannedProvision) { p.Provision = "" }, + "a provider that is no machine": func(p *PlannedProvision) { p.Provider = "../ace" }, + "no provider": func(p *PlannedProvision) { p.Provider = "" }, + "a relative path": func(p *PlannedProvision) { p.Path = "var/x" }, + "a template's path": func(p *PlannedProvision) { p.Path = appConf }, + } { + p := base() + pr := good + change(&pr) + p.Provisions = []PlannedProvision{pr} + if len(p.problems()) == 0 { + t.Errorf("a plan with %s was taken", why) + } + } + p := base() + p.Provisions = []PlannedProvision{good} + if wrong := p.problems(); len(wrong) > 0 { + t.Errorf("a good plan was refused: %v", wrong) + } + p.Provisions = append(p.Provisions, good) + if len(p.problems()) == 0 { + t.Error("a plan naming one credential twice was taken") + } +} + +const ( + dbFile = "/var/lib/nextcloud/state/database.secret" + dbConf = "/etc/nextcloud/config.php" + dbHolder = "pair/novox/postgres-database/" + node + "/nextcloud" +) + +func nextcloud(t *testing.T) Plan { + tpl, _ := templates(t, map[string]string{"config.php": `dbpass=${secret:database}`}) + return Plan{Templates: tpl, Files: []PlannedFile{{ID: "config", Path: dbConf, Template: "config.php", + OwnerAtRuntime: "owned"}}, + Provisions: []PlannedProvision{{Name: "database", Provision: "postgres-database", Provider: "novox", + ProviderModule: "postgres", CredentialForm: "pair", Path: dbFile}}} +} + +// A consumer's own login from a provider that gives each consumer one (issue 779) is the vault's pair row for this +// module on this machine: its secrets file holds it alone, and it is asked for again when the vault says it changed. +func TestNatsAModulesOwnLoginFromAPerLoginProviderIsFilledFromTheVaultAndFollowsARotation(t *testing.T) { + b := startBus(t) + heard := b.listen(t) + wire := sniff(t, b) + vault := startVault(t, b) + const first, second = "login-first-never-on-the-bus", "login-second-never-on-the-bus" + vault.keep(dbHolder, first, false) + e := startEngine(t, map[string]string{dbFile: "owned", dbConf: "owned"}) + b.assign(t, "nextcloud", nextcloud(t)) + aClient(t, b, e, t.TempDir(), time.Hour) + + eventually(t, "the secrets file holds the module's own login, alone", func() bool { + return e.read(t, dbFile) == first + }) + eventually(t, "the template naming it is filled with it", func() bool { + return e.read(t, dbConf) == "dbpass="+first + }) + eventually(t, "prov.applied says applied", func() bool { + r, _ := heard.last("nextcloud") + return r.Status == StatusApplied && len(r.Files) == 2 + }) + vault.keep(dbHolder, second, true) + eventually(t, "a login rotated in the vault reaches the file without a new declaration", func() bool { + return e.read(t, dbFile) == second + }) + // Another consumer's login changing is no reason to ask. + other := "pair/novox/postgres-database/ace/nextcloud" + vault.keep(other, "elsewhere", true) + time.Sleep(300 * time.Millisecond) + if n := vault.times(dbHolder); n != 2 { + t.Errorf("the vault was asked for the login %d times, want 2", n) + } + if vault.times(other) != 0 { + t.Error("a change of another consumer's login made the client ask") + } + for _, value := range []string{first, second} { + if said := wire.carrying(value); len(said) > 0 { + t.Errorf("the login crossed the bus in the clear: %q", said) + } + } +} + +func TestAConsumersLoginIsNamedWithItsLocalName(t *testing.T) { + if got := PairName("novox", "postgres-database", "g14", "umami", "stats"); got != "pair/novox/postgres-database/g14/umami/stats" { + t.Errorf("a login under a local name: %s", got) + } +} diff --git a/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/template.go b/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/template.go new file mode 100644 index 00000000..1e3f3033 --- /dev/null +++ b/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/template.go @@ -0,0 +1,194 @@ +package main + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "os" + "path" + "path/filepath" + "regexp" + "strings" + "time" +) + +// A module's templates, fetched by digest (novox/hq ADR 0349 clause 2, design 57 P5): the archive its build packed +// its templates directory into, fetched from where the plan says and used only when it hashes to the plan's +// digest — the bytes come from a network the mesh does not control, and only the digest makes them the ones the +// mesh built. Kept in the client's state by digest, so a node that cannot reach the artifact store still fills +// from what it fetched. + +// maxTemplates bounds a templates archive: configuration text, never a theme. +const maxTemplates = 32 << 20 + +// Fetcher fetches an archive's bytes from its source. +type Fetcher func(ctx context.Context, source string) ([]byte, error) + +// HTTPFetch is the artifact store, over HTTP. +func HTTPFetch(ctx context.Context, source string) ([]byte, error) { + if !strings.HasPrefix(source, "https://") && !strings.HasPrefix(source, "http://") { + return nil, fmt.Errorf("%s is not an address the templates can be fetched from", source) + } + ctx, cancel := context.WithTimeout(ctx, 2*time.Minute) + defer cancel() + req, err := http.NewRequestWithContext(ctx, http.MethodGet, source, nil) + if err != nil { + return nil, err + } + resp, err := http.DefaultClient.Do(req) + if err != nil { + return nil, fmt.Errorf("cannot fetch %s: %w", source, err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("fetching %s: %s", source, resp.Status) + } + body, err := io.ReadAll(io.LimitReader(resp.Body, maxTemplates+1)) + if err != nil { + return nil, err + } + if len(body) > maxTemplates { + return nil, fmt.Errorf("%s is over %d bytes, more than a module's templates", source, maxTemplates) + } + return body, nil +} + +// templateStore keeps fetched archives by digest under dir, and their texts in memory. +type templateStore struct { + dir string + fetch Fetcher + texts map[string]map[string]string +} + +// load is every template's text of the archive the plan names, by its path inside the archive. +func (s *templateStore) load(ctx context.Context, t Templates) (map[string]string, error) { + if texts, ok := s.texts[t.Digest]; ok { + return texts, nil + } + kept := filepath.Join(s.dir, strings.TrimPrefix(t.Digest, "sha256:")+".tar.gz") + body, err := os.ReadFile(kept) + if err != nil || digestOf(body) != t.Digest { + if body, err = s.fetch(ctx, t.Source); err != nil { + return nil, err + } + if got := digestOf(body); got != t.Digest { + return nil, fmt.Errorf("the templates at %s are %s, and the plan names %s: nothing was filled from them", + t.Source, got, t.Digest) + } + if err := writeKept(kept, body); err != nil { + // Said and gone on: the archive is in hand, and is fetched again next start. + logf("[provisioner-client] the templates %s could not be kept at %s: %v", t.Digest, kept, err) + } + } + texts, err := unpackTexts(body) + if err != nil { + return nil, fmt.Errorf("the templates %s cannot be read: %w", t.Digest, err) + } + if s.texts == nil { + s.texts = map[string]map[string]string{} + } + s.texts[t.Digest] = texts + return texts, nil +} + +func digestOf(b []byte) string { + sum := sha256.Sum256(b) + return "sha256:" + hex.EncodeToString(sum[:]) +} + +func writeKept(at string, body []byte) error { + if err := os.MkdirAll(filepath.Dir(at), 0o700); err != nil { + return err + } + tmp, err := os.CreateTemp(filepath.Dir(at), ".templates-*") + if err != nil { + return err + } + defer os.Remove(tmp.Name()) + if _, err := tmp.Write(body); err != nil { + tmp.Close() + return err + } + if err := tmp.Close(); err != nil { + return err + } + return os.Rename(tmp.Name(), at) +} + +// unpackTexts is every regular file of a gzipped tar, by its clean path inside it. +func unpackTexts(body []byte) (map[string]string, error) { + zipped, err := gzip.NewReader(bytes.NewReader(body)) + if err != nil { + return nil, err + } + defer zipped.Close() + r := tar.NewReader(zipped) + out := map[string]string{} + for { + h, err := r.Next() + if errors.Is(err, io.EOF) { + return out, nil + } + if err != nil { + return nil, err + } + if h.Typeflag != tar.TypeReg { + continue + } + b, err := io.ReadAll(io.LimitReader(r, maxTemplates)) + if err != nil { + return nil, err + } + out[strings.TrimPrefix(path.Clean(h.Name), "./")] = string(b) + } +} + +// placeholder is `${:}`, as ADR 0349 clause 2 writes one. +var placeholder = regexp.MustCompile(`\$\{([a-z][a-z0-9-]*):([^}]+)\}`) + +// Need is one value a template names. +type Need struct { + Kind string + Name string +} + +func (n Need) String() string { return n.Kind + ":" + n.Name } + +// needsOf is every placeholder a text names, in the order each first appears. +func needsOf(text string) []Need { + var out []Need + seen := map[Need]bool{} + for _, m := range placeholder.FindAllStringSubmatch(text, -1) { + n := Need{Kind: m[1], Name: m[2]} + if !seen[n] { + seen[n] = true + out = append(out, n) + } + } + return out +} + +// render fills every placeholder of text from values, which holds each. +func render(text string, values map[Need]string) string { + return placeholder.ReplaceAllStringFunc(text, func(ref string) string { + m := placeholder.FindStringSubmatch(ref) + return values[Need{Kind: m[1], Name: m[2]}] + }) +} + +// configText is a config value as a file takes it: a string as its text, anything else as its JSON. +func configText(raw json.RawMessage) string { + var s string + if json.Unmarshal(raw, &s) == nil { + return s + } + return string(raw) +} diff --git a/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/testdata/accounts.conf b/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/testdata/accounts.conf new file mode 100644 index 00000000..85e77ee0 --- /dev/null +++ b/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/testdata/accounts.conf @@ -0,0 +1,25 @@ +# The provisioner client's account as mesh-controller composes it (internal/broker ComposeAccounts, feature branch +# 57-config-and-provisioning with issue 647, which grants it the vault's secret-changed, issue 685, which lets its +# own machine's tool runner ask it, and issue 679, which lets it read the whole assignments mirror and its machine's +# keys of a provider's bucket of entries, here postgres_bound, and for gitea's route, route-adapter_bound and route-proxy_bound, issue 792; node-tools_bound for claude-code's console port, issue 761), the tool runner's account of the same machine +# carrying claude-code, and a `domain` user granted everything, standing for the controller, mesh-config and the +# vault, which own the buckets the client reads and answer its asks. Composed again if the client's grants change. + +accounts { + MESH { + jetstream: enabled + users = [ + { user: "domain", password: "x" } + { user: "provisioner.g14", password: "x", permissions: { + publish: { allow: ["$JS.API.CONSUMER.CREATE.KV_mesh-config_resolved.*.$KV.mesh-config_resolved.g14.>", "$JS.API.CONSUMER.CREATE.KV_mesh-controller_assignments.>", "$JS.API.CONSUMER.CREATE.KV_mesh-provisioner_applied.*.$KV.mesh-provisioner_applied.g14.>", "$JS.API.CONSUMER.CREATE.KV_mesh-provisioner_collect.*.$KV.mesh-provisioner_collect.g14.>", "$JS.API.CONSUMER.CREATE.KV_mesh-provisioner_needs.*.$KV.mesh-provisioner_needs.g14.>", "$JS.API.CONSUMER.CREATE.KV_mesh-provisioner_registry.>", "$JS.API.CONSUMER.CREATE.KV_postgres_bound.*.$KV.postgres_bound.g14.>", "$JS.API.CONSUMER.CREATE.KV_route-adapter_bound.*.$KV.route-adapter_bound.g14.>", "$JS.API.CONSUMER.CREATE.KV_route-proxy_bound.*.$KV.route-proxy_bound.g14.>", "$JS.API.CONSUMER.CREATE.KV_node-tools_bound.*.$KV.node-tools_bound.g14.>", "$JS.API.CONSUMER.DELETE.KV_mesh-config_resolved.>", "$JS.API.CONSUMER.DELETE.KV_mesh-controller_assignments.>", "$JS.API.CONSUMER.DELETE.KV_mesh-provisioner_applied.>", "$JS.API.CONSUMER.DELETE.KV_mesh-provisioner_collect.>", "$JS.API.CONSUMER.DELETE.KV_mesh-provisioner_needs.>", "$JS.API.CONSUMER.DELETE.KV_mesh-provisioner_registry.>", "$JS.API.CONSUMER.DELETE.KV_postgres_bound.>", "$JS.API.CONSUMER.DELETE.KV_route-adapter_bound.>", "$JS.API.CONSUMER.DELETE.KV_route-proxy_bound.>", "$JS.API.CONSUMER.DELETE.KV_node-tools_bound.>", "$JS.API.DIRECT.GET.KV_mesh-config_resolved.$KV.mesh-config_resolved.g14.>", "$JS.API.DIRECT.GET.KV_mesh-controller_assignments.>", "$JS.API.DIRECT.GET.KV_mesh-provisioner_applied.$KV.mesh-provisioner_applied.g14.>", "$JS.API.DIRECT.GET.KV_mesh-provisioner_collect.$KV.mesh-provisioner_collect.g14.>", "$JS.API.DIRECT.GET.KV_mesh-provisioner_needs.$KV.mesh-provisioner_needs.g14.>", "$JS.API.DIRECT.GET.KV_mesh-provisioner_registry.>", "$JS.API.DIRECT.GET.KV_postgres_bound.$KV.postgres_bound.g14.>", "$JS.API.DIRECT.GET.KV_route-adapter_bound.$KV.route-adapter_bound.g14.>", "$JS.API.DIRECT.GET.KV_route-proxy_bound.$KV.route-proxy_bound.g14.>", "$JS.API.DIRECT.GET.KV_node-tools_bound.$KV.node-tools_bound.g14.>", "$JS.API.STREAM.INFO.KV_mesh-config_resolved", "$JS.API.STREAM.INFO.KV_mesh-controller_assignments", "$JS.API.STREAM.INFO.KV_mesh-provisioner_applied", "$JS.API.STREAM.INFO.KV_mesh-provisioner_collect", "$JS.API.STREAM.INFO.KV_mesh-provisioner_needs", "$JS.API.STREAM.INFO.KV_mesh-provisioner_registry", "$JS.API.STREAM.INFO.KV_postgres_bound", "$JS.API.STREAM.INFO.KV_route-adapter_bound", "$JS.API.STREAM.INFO.KV_route-proxy_bound", "$JS.API.STREAM.INFO.KV_node-tools_bound", "$JS.FC.KV_mesh-config_resolved.>", "$JS.FC.KV_mesh-controller_assignments.>", "$JS.FC.KV_mesh-provisioner_applied.>", "$JS.FC.KV_mesh-provisioner_collect.>", "$JS.FC.KV_mesh-provisioner_needs.>", "$JS.FC.KV_mesh-provisioner_registry.>", "$JS.FC.KV_postgres_bound.>", "$JS.FC.KV_route-adapter_bound.>", "$JS.FC.KV_route-proxy_bound.>", "$JS.FC.KV_node-tools_bound.>", "$KV.mesh-provisioner_applied.g14.>", "$KV.mesh-provisioner_collect.g14.>", "$KV.mesh-provisioner_needs.g14.>", "mesh.seat.mesh-vault.call.get.provisioner~g14", "prov.applied.g14.>", "prov.changed.g14.>"] } + subscribe: { allow: ["_INBOX.provisioner.g14.>", "mesh.mod.mesh-vault.event.secret-changed", "prov.ask.g14"] } + allow_responses: { max: 1, ttl: "1m" } + } } + { user: "g14.node-tools", password: "x", permissions: { + publish: { allow: ["$JS.API.DIRECT.GET.ASSIGNMENTS.mesh.assignment.g14.*", "$SRV.INFO", "$SRV.INFO.>", "$SRV.PING", "$SRV.PING.>", "mesh.control.g14.tools-alive", "mesh.mod.*.call.*.*.g14~node-tools", "mesh.mod.*.call.*.g14~node-tools", "mesh.mod.*.tool.>", "mesh.seat.*.call.*.*.g14~node-tools", "mesh.seat.*.call.*.g14~node-tools", "mesh.seat.*.tool.>", "prov.ask.g14"], deny: ["mesh.seat.node-launcher.tool.secret", "mesh.seat.node-launcher.tool.secret.*", "mesh.mod.*.tool.node-launcher.secret", "mesh.mod.*.tool.node-launcher.secret.*", "mesh.seat.node-launcher.call.secret.>", "mesh.mod.*.call.node-launcher.secret.>"] } + subscribe: { allow: ["$SRV.INFO", "$SRV.INFO.claude-code", "$SRV.INFO.claude-code.>", "$SRV.INFO.node-tools", "$SRV.INFO.node-tools.>", "$SRV.PING", "$SRV.PING.claude-code", "$SRV.PING.claude-code.>", "$SRV.PING.node-tools", "$SRV.PING.node-tools.>", "$SRV.STATS", "$SRV.STATS.claude-code", "$SRV.STATS.claude-code.>", "$SRV.STATS.node-tools", "$SRV.STATS.node-tools.>", "_INBOX.g14.node-tools.>", "mesh.assignment.g14.*", "mesh.mod.claude-code.call.>", "mesh.mod.claude-code.tool.>", "prov.changed.g14.>"] } + allow_responses: { max: 1, ttl: "1m" } + } } + ] + } +} diff --git a/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/testdata/bars/battery.bottom-bar.toml b/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/testdata/bars/battery.bottom-bar.toml new file mode 100644 index 00000000..d5ec4c7c --- /dev/null +++ b/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/testdata/bars/battery.bottom-bar.toml @@ -0,0 +1,119 @@ +# The bottom bar (module i3status-rust, novox/hq ADR 0208). Owned by the mesh: replaced at every +# push. Adopted from the workstations of 2026-10-04 with what follows a machine's hardware or one +# person's devices left out: a block that depends on the hardware is contributed by the module that +# knows it (novox/hq ADR 0255), and a headset's address is not the bar's to know. Every block written +# here works the same on any machine. +icons_format = "{icon}" + +[theme] +theme = "plain" +[theme.overrides] +idle_bg = "#000000" +idle_fg = "#f1f1f1" +info_bg = "#000000" +info_fg = "#f1f1f1" +good_bg = "#000000" +good_fg = "#859900" +warning_bg = "#000000" +warning_fg = "#de5200" +critical_bg = "#000000" +critical_fg = "#ff0000" +separator = " | " +separator_fg = "#ffffff" +separator_bg = "#000000" + +[icons] +icons = "custom-icons" + +[[block]] +block = "tea_timer" +format = "$icon{ $minutes:$seconds|}" +done_cmd = "notify-send 'Timer finished'" + +# Pending updates: official and, where an AUR helper is installed, AUR. The script keeps the last good +# count through a network gap and shows an error only after ten failed checks in a row. Click: the +# list, in the launcher's menu. +[[block]] +block = "custom" +command = "~/.local/bin/i3status-updates 10" +json = true +interval = 1800 +[[block.click]] +button = "left" +cmd = "checkupdates | dmenu -l 20 -p Updates" + +[[block]] +block = "cpu" + +[[block]] +block = "disk_space" +path = "/" +info_type = "used" +alert_unit = "GB" +interval = 20 +warning = 850 +alert = 920 +format = "$icon / $used.eng(w:2) ($percentage.eng(w:2))" + +[[block]] +block = "disk_space" +path = "/home" +info_type = "used" +alert_unit = "GB" +interval = 20 +warning = 850 +alert = 920 +format = "$icon /home $used.eng(w:2) ($percentage.eng(w:2))" + +[[block]] +block = "memory" +# What programs hold, without the cache the kernel gives back when asked: the figure the memory-pressure +# module judges by. Counting the cache as used read near full on a machine with room to spare. +format = "$icon $mem_used.eng(w:2) ($mem_used_percents.eng(w:2))" +format_alt = "$icon_swap $swap_used.eng(w:2) ($swap_used_percents.eng(w:2))" + +# Blocks other modules contribute beside the machine's resources (novox/hq ADR 0255), rendered from +# what they show by this module's template, and only on a machine that has what they show. + +[[block]] +block = "docker" +interval = 2 +format = "$icon $running/$total" + +[[block]] +block = "sound" +[[block.click]] +button = "left" +cmd = "pavucontrol" + +# Blocks other modules contribute beside the sound (novox/hq ADR 0255): the power module's battery and +# power draw, where the machine has them. +# power +[[block]] +block = "battery" +format = "$icon $percentage{ $time|}" +charging_format = "$icon $percentage{ $time|}" +not_charging_format = "$icon $percentage" +full_format = "$icon $percentage" +empty_format = "$icon $percentage" +missing_format = "" + + +# The weather from the Norwegian Meteorological Institute, which needs no key, where the machine is. +[[block]] +block = "weather" +format = "$icon $weather_verbose ($location) $temp" +autolocate = true +autolocate_interval = 600 +[block.service] +name = "metno" + +[[block]] +block = "notify" +driver = "dunst" +format = " $icon {($notification_count.eng(w:1))|}" + +[[block]] +block = "time" +interval = 1 +format = "$timestamp.datetime(f:'%a %d/%m %H:%M:%S')" diff --git a/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/testdata/bars/battery.top-bar.toml b/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/testdata/bars/battery.top-bar.toml new file mode 100644 index 00000000..70714fd9 --- /dev/null +++ b/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/testdata/bars/battery.top-bar.toml @@ -0,0 +1,36 @@ +# The top bar (module i3status-rust, novox/hq ADR 0208): the focused window's title and the uptime. +# Owned by the mesh: replaced at every push. Adopted unchanged from the workstations of 2026-10-04. +icons_format = "{icon}" + +[theme] +theme = "plain" +[theme.overrides] +idle_bg = "#000000" +idle_fg = "#f1f1f1" +info_bg = "#000000" +info_fg = "#f1f1f1" +good_bg = "#000000" +good_fg = "#859900" +warning_bg = "#000000" +warning_fg = "#de5200" +critical_bg = "#000000" +critical_fg = "#ff0000" +separator = " | " +separator_fg = "#ffffff" +separator_bg = "#000000" + +[icons] +icons = "custom-icons" + +[[block]] +block = "focused_window" +[block.format] +full = " $title.str(max_w:50) |" +short = " $title.str(max_w:100) |" + +[[block]] +block = "uptime" +interval = 60 + +# Blocks other modules contribute to the top bar (novox/hq ADR 0255). None do yet; both places are kept +# so that one that does is never dropped. diff --git a/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/testdata/bars/both.bottom-bar.toml b/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/testdata/bars/both.bottom-bar.toml new file mode 100644 index 00000000..ad3a4e65 --- /dev/null +++ b/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/testdata/bars/both.bottom-bar.toml @@ -0,0 +1,126 @@ +# The bottom bar (module i3status-rust, novox/hq ADR 0208). Owned by the mesh: replaced at every +# push. Adopted from the workstations of 2026-10-04 with what follows a machine's hardware or one +# person's devices left out: a block that depends on the hardware is contributed by the module that +# knows it (novox/hq ADR 0255), and a headset's address is not the bar's to know. Every block written +# here works the same on any machine. +icons_format = "{icon}" + +[theme] +theme = "plain" +[theme.overrides] +idle_bg = "#000000" +idle_fg = "#f1f1f1" +info_bg = "#000000" +info_fg = "#f1f1f1" +good_bg = "#000000" +good_fg = "#859900" +warning_bg = "#000000" +warning_fg = "#de5200" +critical_bg = "#000000" +critical_fg = "#ff0000" +separator = " | " +separator_fg = "#ffffff" +separator_bg = "#000000" + +[icons] +icons = "custom-icons" + +[[block]] +block = "tea_timer" +format = "$icon{ $minutes:$seconds|}" +done_cmd = "notify-send 'Timer finished'" + +# Pending updates: official and, where an AUR helper is installed, AUR. The script keeps the last good +# count through a network gap and shows an error only after ten failed checks in a row. Click: the +# list, in the launcher's menu. +[[block]] +block = "custom" +command = "~/.local/bin/i3status-updates 10" +json = true +interval = 1800 +[[block.click]] +button = "left" +cmd = "checkupdates | dmenu -l 20 -p Updates" + +[[block]] +block = "cpu" + +[[block]] +block = "disk_space" +path = "/" +info_type = "used" +alert_unit = "GB" +interval = 20 +warning = 850 +alert = 920 +format = "$icon / $used.eng(w:2) ($percentage.eng(w:2))" + +[[block]] +block = "disk_space" +path = "/home" +info_type = "used" +alert_unit = "GB" +interval = 20 +warning = 850 +alert = 920 +format = "$icon /home $used.eng(w:2) ($percentage.eng(w:2))" + +[[block]] +block = "memory" +# What programs hold, without the cache the kernel gives back when asked: the figure the memory-pressure +# module judges by. Counting the cache as used read near full on a machine with room to spare. +format = "$icon $mem_used.eng(w:2) ($mem_used_percents.eng(w:2))" +format_alt = "$icon_swap $swap_used.eng(w:2) ($swap_used_percents.eng(w:2))" + +# Blocks other modules contribute beside the machine's resources (novox/hq ADR 0255), rendered from +# what they show by this module's template, and only on a machine that has what they show. + +[[block]] +block = "docker" +interval = 2 +format = "$icon $running/$total" + +[[block]] +block = "sound" +[[block.click]] +button = "left" +cmd = "pavucontrol" + +# Blocks other modules contribute beside the sound (novox/hq ADR 0255): the power module's battery and +# power draw, where the machine has them. +# power +[[block]] +block = "battery" +format = "$icon $percentage{ $time|}" +charging_format = "$icon $percentage{ $time|}" +not_charging_format = "$icon $percentage" +full_format = "$icon $percentage" +empty_format = "$icon $percentage" +missing_format = "" + +# power +[[block]] +block = "custom" +command = "~/.local/bin/i3status-value power.draw workstation" +interval = 5 +hide_when_empty = true + + +# The weather from the Norwegian Meteorological Institute, which needs no key, where the machine is. +[[block]] +block = "weather" +format = "$icon $weather_verbose ($location) $temp" +autolocate = true +autolocate_interval = 600 +[block.service] +name = "metno" + +[[block]] +block = "notify" +driver = "dunst" +format = " $icon {($notification_count.eng(w:1))|}" + +[[block]] +block = "time" +interval = 1 +format = "$timestamp.datetime(f:'%a %d/%m %H:%M:%S')" diff --git a/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/testdata/bars/both.top-bar.toml b/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/testdata/bars/both.top-bar.toml new file mode 100644 index 00000000..70714fd9 --- /dev/null +++ b/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/testdata/bars/both.top-bar.toml @@ -0,0 +1,36 @@ +# The top bar (module i3status-rust, novox/hq ADR 0208): the focused window's title and the uptime. +# Owned by the mesh: replaced at every push. Adopted unchanged from the workstations of 2026-10-04. +icons_format = "{icon}" + +[theme] +theme = "plain" +[theme.overrides] +idle_bg = "#000000" +idle_fg = "#f1f1f1" +info_bg = "#000000" +info_fg = "#f1f1f1" +good_bg = "#000000" +good_fg = "#859900" +warning_bg = "#000000" +warning_fg = "#de5200" +critical_bg = "#000000" +critical_fg = "#ff0000" +separator = " | " +separator_fg = "#ffffff" +separator_bg = "#000000" + +[icons] +icons = "custom-icons" + +[[block]] +block = "focused_window" +[block.format] +full = " $title.str(max_w:50) |" +short = " $title.str(max_w:100) |" + +[[block]] +block = "uptime" +interval = 60 + +# Blocks other modules contribute to the top bar (novox/hq ADR 0255). None do yet; both places are kept +# so that one that does is never dropped. diff --git a/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/testdata/bars/none.bottom-bar.toml b/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/testdata/bars/none.bottom-bar.toml new file mode 100644 index 00000000..59dc87a1 --- /dev/null +++ b/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/testdata/bars/none.bottom-bar.toml @@ -0,0 +1,109 @@ +# The bottom bar (module i3status-rust, novox/hq ADR 0208). Owned by the mesh: replaced at every +# push. Adopted from the workstations of 2026-10-04 with what follows a machine's hardware or one +# person's devices left out: a block that depends on the hardware is contributed by the module that +# knows it (novox/hq ADR 0255), and a headset's address is not the bar's to know. Every block written +# here works the same on any machine. +icons_format = "{icon}" + +[theme] +theme = "plain" +[theme.overrides] +idle_bg = "#000000" +idle_fg = "#f1f1f1" +info_bg = "#000000" +info_fg = "#f1f1f1" +good_bg = "#000000" +good_fg = "#859900" +warning_bg = "#000000" +warning_fg = "#de5200" +critical_bg = "#000000" +critical_fg = "#ff0000" +separator = " | " +separator_fg = "#ffffff" +separator_bg = "#000000" + +[icons] +icons = "custom-icons" + +[[block]] +block = "tea_timer" +format = "$icon{ $minutes:$seconds|}" +done_cmd = "notify-send 'Timer finished'" + +# Pending updates: official and, where an AUR helper is installed, AUR. The script keeps the last good +# count through a network gap and shows an error only after ten failed checks in a row. Click: the +# list, in the launcher's menu. +[[block]] +block = "custom" +command = "~/.local/bin/i3status-updates 10" +json = true +interval = 1800 +[[block.click]] +button = "left" +cmd = "checkupdates | dmenu -l 20 -p Updates" + +[[block]] +block = "cpu" + +[[block]] +block = "disk_space" +path = "/" +info_type = "used" +alert_unit = "GB" +interval = 20 +warning = 850 +alert = 920 +format = "$icon / $used.eng(w:2) ($percentage.eng(w:2))" + +[[block]] +block = "disk_space" +path = "/home" +info_type = "used" +alert_unit = "GB" +interval = 20 +warning = 850 +alert = 920 +format = "$icon /home $used.eng(w:2) ($percentage.eng(w:2))" + +[[block]] +block = "memory" +# What programs hold, without the cache the kernel gives back when asked: the figure the memory-pressure +# module judges by. Counting the cache as used read near full on a machine with room to spare. +format = "$icon $mem_used.eng(w:2) ($mem_used_percents.eng(w:2))" +format_alt = "$icon_swap $swap_used.eng(w:2) ($swap_used_percents.eng(w:2))" + +# Blocks other modules contribute beside the machine's resources (novox/hq ADR 0255), rendered from +# what they show by this module's template, and only on a machine that has what they show. + +[[block]] +block = "docker" +interval = 2 +format = "$icon $running/$total" + +[[block]] +block = "sound" +[[block.click]] +button = "left" +cmd = "pavucontrol" + +# Blocks other modules contribute beside the sound (novox/hq ADR 0255): the power module's battery and +# power draw, where the machine has them. + +# The weather from the Norwegian Meteorological Institute, which needs no key, where the machine is. +[[block]] +block = "weather" +format = "$icon $weather_verbose ($location) $temp" +autolocate = true +autolocate_interval = 600 +[block.service] +name = "metno" + +[[block]] +block = "notify" +driver = "dunst" +format = " $icon {($notification_count.eng(w:1))|}" + +[[block]] +block = "time" +interval = 1 +format = "$timestamp.datetime(f:'%a %d/%m %H:%M:%S')" diff --git a/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/testdata/bars/none.top-bar.toml b/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/testdata/bars/none.top-bar.toml new file mode 100644 index 00000000..70714fd9 --- /dev/null +++ b/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/testdata/bars/none.top-bar.toml @@ -0,0 +1,36 @@ +# The top bar (module i3status-rust, novox/hq ADR 0208): the focused window's title and the uptime. +# Owned by the mesh: replaced at every push. Adopted unchanged from the workstations of 2026-10-04. +icons_format = "{icon}" + +[theme] +theme = "plain" +[theme.overrides] +idle_bg = "#000000" +idle_fg = "#f1f1f1" +info_bg = "#000000" +info_fg = "#f1f1f1" +good_bg = "#000000" +good_fg = "#859900" +warning_bg = "#000000" +warning_fg = "#de5200" +critical_bg = "#000000" +critical_fg = "#ff0000" +separator = " | " +separator_fg = "#ffffff" +separator_bg = "#000000" + +[icons] +icons = "custom-icons" + +[[block]] +block = "focused_window" +[block.format] +full = " $title.str(max_w:50) |" +short = " $title.str(max_w:100) |" + +[[block]] +block = "uptime" +interval = 60 + +# Blocks other modules contribute to the top bar (novox/hq ADR 0255). None do yet; both places are kept +# so that one that does is never dropped. diff --git a/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/testdata/bars/power-meter.bottom-bar.toml b/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/testdata/bars/power-meter.bottom-bar.toml new file mode 100644 index 00000000..ae6c2ce2 --- /dev/null +++ b/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/testdata/bars/power-meter.bottom-bar.toml @@ -0,0 +1,116 @@ +# The bottom bar (module i3status-rust, novox/hq ADR 0208). Owned by the mesh: replaced at every +# push. Adopted from the workstations of 2026-10-04 with what follows a machine's hardware or one +# person's devices left out: a block that depends on the hardware is contributed by the module that +# knows it (novox/hq ADR 0255), and a headset's address is not the bar's to know. Every block written +# here works the same on any machine. +icons_format = "{icon}" + +[theme] +theme = "plain" +[theme.overrides] +idle_bg = "#000000" +idle_fg = "#f1f1f1" +info_bg = "#000000" +info_fg = "#f1f1f1" +good_bg = "#000000" +good_fg = "#859900" +warning_bg = "#000000" +warning_fg = "#de5200" +critical_bg = "#000000" +critical_fg = "#ff0000" +separator = " | " +separator_fg = "#ffffff" +separator_bg = "#000000" + +[icons] +icons = "custom-icons" + +[[block]] +block = "tea_timer" +format = "$icon{ $minutes:$seconds|}" +done_cmd = "notify-send 'Timer finished'" + +# Pending updates: official and, where an AUR helper is installed, AUR. The script keeps the last good +# count through a network gap and shows an error only after ten failed checks in a row. Click: the +# list, in the launcher's menu. +[[block]] +block = "custom" +command = "~/.local/bin/i3status-updates 10" +json = true +interval = 1800 +[[block.click]] +button = "left" +cmd = "checkupdates | dmenu -l 20 -p Updates" + +[[block]] +block = "cpu" + +[[block]] +block = "disk_space" +path = "/" +info_type = "used" +alert_unit = "GB" +interval = 20 +warning = 850 +alert = 920 +format = "$icon / $used.eng(w:2) ($percentage.eng(w:2))" + +[[block]] +block = "disk_space" +path = "/home" +info_type = "used" +alert_unit = "GB" +interval = 20 +warning = 850 +alert = 920 +format = "$icon /home $used.eng(w:2) ($percentage.eng(w:2))" + +[[block]] +block = "memory" +# What programs hold, without the cache the kernel gives back when asked: the figure the memory-pressure +# module judges by. Counting the cache as used read near full on a machine with room to spare. +format = "$icon $mem_used.eng(w:2) ($mem_used_percents.eng(w:2))" +format_alt = "$icon_swap $swap_used.eng(w:2) ($swap_used_percents.eng(w:2))" + +# Blocks other modules contribute beside the machine's resources (novox/hq ADR 0255), rendered from +# what they show by this module's template, and only on a machine that has what they show. + +[[block]] +block = "docker" +interval = 2 +format = "$icon $running/$total" + +[[block]] +block = "sound" +[[block.click]] +button = "left" +cmd = "pavucontrol" + +# Blocks other modules contribute beside the sound (novox/hq ADR 0255): the power module's battery and +# power draw, where the machine has them. +# power +[[block]] +block = "custom" +command = "~/.local/bin/i3status-value power.draw workstation" +interval = 5 +hide_when_empty = true + + +# The weather from the Norwegian Meteorological Institute, which needs no key, where the machine is. +[[block]] +block = "weather" +format = "$icon $weather_verbose ($location) $temp" +autolocate = true +autolocate_interval = 600 +[block.service] +name = "metno" + +[[block]] +block = "notify" +driver = "dunst" +format = " $icon {($notification_count.eng(w:1))|}" + +[[block]] +block = "time" +interval = 1 +format = "$timestamp.datetime(f:'%a %d/%m %H:%M:%S')" diff --git a/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/testdata/bars/power-meter.top-bar.toml b/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/testdata/bars/power-meter.top-bar.toml new file mode 100644 index 00000000..70714fd9 --- /dev/null +++ b/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/testdata/bars/power-meter.top-bar.toml @@ -0,0 +1,36 @@ +# The top bar (module i3status-rust, novox/hq ADR 0208): the focused window's title and the uptime. +# Owned by the mesh: replaced at every push. Adopted unchanged from the workstations of 2026-10-04. +icons_format = "{icon}" + +[theme] +theme = "plain" +[theme.overrides] +idle_bg = "#000000" +idle_fg = "#f1f1f1" +info_bg = "#000000" +info_fg = "#f1f1f1" +good_bg = "#000000" +good_fg = "#859900" +warning_bg = "#000000" +warning_fg = "#de5200" +critical_bg = "#000000" +critical_fg = "#ff0000" +separator = " | " +separator_fg = "#ffffff" +separator_bg = "#000000" + +[icons] +icons = "custom-icons" + +[[block]] +block = "focused_window" +[block.format] +full = " $title.str(max_w:50) |" +short = " $title.str(max_w:100) |" + +[[block]] +block = "uptime" +interval = 60 + +# Blocks other modules contribute to the top bar (novox/hq ADR 0255). None do yet; both places are kept +# so that one that does is never dropped. diff --git a/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/testdata/environment/all.posix b/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/testdata/environment/all.posix new file mode 100644 index 00000000..aa61849d --- /dev/null +++ b/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/testdata/environment/all.posix @@ -0,0 +1,26 @@ +# adwaita +export GTK2_RC_FILES='/usr/share/themes/Adwaita-dark/gtk-2.0/gtkrc' +export GTK_THEME='Adwaita:dark' +export QT_QPA_PLATFORMTHEME='qt6ct' +export QT_SELECT='6' +export QT_STYLE_OVERRIDE='Fusion' +export XCURSOR_SIZE='24' +export XCURSOR_THEME='Adwaita' +# clipmenu +export CM_HISTLENGTH='15' +export CM_MAX_CLIPS='500' +export CM_SELECTIONS='clipboard' +# i3 +export XDG_CURRENT_DESKTOP='i3' +export XDG_SESSION_DESKTOP='i3' +# xterm +export TERMINAL='xterm' +# zsh +export EDITOR='vim' +export VISUAL='vim' +export XDG_CONFIG_HOME='/home/op/.config' +# zsh +case ":${PATH}:" in *':/home/op/scripts/bin:'*) ;; *) PATH='/home/op/scripts/bin'"${PATH:+:${PATH}}" ;; esac +case ":${PATH}:" in *':/home/op/scripts:'*) ;; *) PATH='/home/op/scripts'"${PATH:+:${PATH}}" ;; esac +case ":${PATH}:" in *':/home/op/.local/bin:'*) ;; *) PATH='/home/op/.local/bin'"${PATH:+:${PATH}}" ;; esac +export PATH diff --git a/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/testdata/environment/all.systemd b/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/testdata/environment/all.systemd new file mode 100644 index 00000000..70e7709a --- /dev/null +++ b/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/testdata/environment/all.systemd @@ -0,0 +1,23 @@ +# adwaita +GTK2_RC_FILES=/usr/share/themes/Adwaita-dark/gtk-2.0/gtkrc +GTK_THEME=Adwaita:dark +QT_QPA_PLATFORMTHEME=qt6ct +QT_SELECT=6 +QT_STYLE_OVERRIDE=Fusion +XCURSOR_SIZE=24 +XCURSOR_THEME=Adwaita +# clipmenu +CM_HISTLENGTH=15 +CM_MAX_CLIPS=500 +CM_SELECTIONS=clipboard +# i3 +XDG_CURRENT_DESKTOP=i3 +XDG_SESSION_DESKTOP=i3 +# xterm +TERMINAL=xterm +# zsh +EDITOR=vim +VISUAL=vim +XDG_CONFIG_HOME=/home/op/.config +# zsh +PATH=/home/op/.local/bin:/home/op/scripts:/home/op/scripts/bin${PATH:+:$PATH} diff --git a/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/testdata/environment/zsh.posix b/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/testdata/environment/zsh.posix new file mode 100644 index 00000000..905c1574 --- /dev/null +++ b/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/testdata/environment/zsh.posix @@ -0,0 +1,9 @@ +# zsh +export EDITOR='vim' +export VISUAL='vim' +export XDG_CONFIG_HOME='/home/op/.config' +# zsh +case ":${PATH}:" in *':/home/op/scripts/bin:'*) ;; *) PATH='/home/op/scripts/bin'"${PATH:+:${PATH}}" ;; esac +case ":${PATH}:" in *':/home/op/scripts:'*) ;; *) PATH='/home/op/scripts'"${PATH:+:${PATH}}" ;; esac +case ":${PATH}:" in *':/home/op/.local/bin:'*) ;; *) PATH='/home/op/.local/bin'"${PATH:+:${PATH}}" ;; esac +export PATH diff --git a/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/testdata/environment/zsh.systemd b/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/testdata/environment/zsh.systemd new file mode 100644 index 00000000..2a28ce0a --- /dev/null +++ b/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/testdata/environment/zsh.systemd @@ -0,0 +1,6 @@ +# zsh +EDITOR=vim +VISUAL=vim +XDG_CONFIG_HOME=/home/op/.config +# zsh +PATH=/home/op/.local/bin:/home/op/scripts:/home/op/scripts/bin${PATH:+:$PATH} diff --git a/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/testdata/i3/all.config b/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/testdata/i3/all.config new file mode 100644 index 00000000..6475ac58 --- /dev/null +++ b/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/testdata/i3/all.config @@ -0,0 +1,310 @@ +# i3 config file (v4), written by the mesh (module i3, novox/hq ADR 0208). Replaced at every push; +# change the module instead. i3's user guide is the reference. +# +# Other modules add to this configuration as contributions to node-display-session (novox/hq ADR +# 0212): the mesh places their lines near the end, each module's under a line naming it, where every +# variable set here ($mod, $ws1 … $ws10) is in scope. A file of yours in ~/.config/i3/config.d/ is read +# after them, by the include at the very end, and is yours. +# +# The reload watcher of this module reloads i3 when this file or a drop-in changes, after checking the +# result with i3 -C; it never reloads into a configuration with errors. + +# Font for window titles, and the bars below: the mesh's monospace face (research 026/04). +font pango:JetBrainsMono Nerd Font 11 + +# XDG autostart entries (~/.config/autostart, /etc/xdg/autostart), started once at login. +exec --no-startup-id dex --autostart --environment i3 + +######################################### +###### Keys #### +######################################### +# To find key symbols: xmodmap -pke / xmodmap -pm +set $mod Mod4 +set $alt Mod1 +set $shift Shift +set $ctrl Control + +# use these keys for focus, movement, and resize directions when reaching for +# the arrows is not convenient +set $left h +set $down j +set $up k +set $right l + +# use Mouse+$mod to drag floating windows to their wanted position +floating_modifier $mod + +# move tiling windows via drag & drop by left-clicking into the title bar, +# or left-clicking anywhere into the window while holding the floating modifier. +tiling_drag modifier titlebar + +# start a terminal: whichever the terminal module names in $TERMINAL +bindsym $mod+Return exec i3-sensible-terminal + +# kill focused window +bindsym $mod+$shift+q kill + +# change focus +bindsym $mod+$left focus left +bindsym $mod+$down focus down +bindsym $mod+$up focus up +bindsym $mod+$right focus right + +bindsym $mod+Left focus left +bindsym $mod+Down focus down +bindsym $mod+Up focus up +bindsym $mod+Right focus right + +# move focused window +bindsym $mod+$shift+$left move left +bindsym $mod+$shift+$down move down +bindsym $mod+$shift+$up move up +bindsym $mod+$shift+$right move right + +bindsym $mod+$shift+Left move left +bindsym $mod+$shift+Down move down +bindsym $mod+$shift+Up move up +bindsym $mod+$shift+Right move right + +# split in horizontal orientation +bindsym $mod+c split h +# split in vertical orientation +bindsym $mod+v split v + +# enter fullscreen mode for the focused container +bindsym $mod+f fullscreen toggle + +# change container layout (stacked, tabbed, toggle split) +bindsym $mod+s layout stacking +bindsym $mod+w layout tabbed +bindsym $mod+e layout toggle split + +# toggle tiling / floating +bindsym $mod+$shift+space floating toggle + +# change focus between tiling / floating windows +bindsym $mod+space focus mode_toggle + +# focus the parent container +bindsym $mod+a focus parent + +# alt-tab functionality +bindsym $mod+Tab workspace back_and_forth + +######################################### +###### Workspace mgmt #### +######################################### +set $ws1 "1" +set $ws2 "2" +set $ws3 "3" +set $ws4 "4" +set $ws5 "5" +set $ws6 "6" +set $ws7 "7" +set $ws8 "8" +set $ws9 "9" +set $ws10 "10" + +bindsym $mod+1 workspace number $ws1 +bindsym $mod+2 workspace number $ws2 +bindsym $mod+3 workspace number $ws3 +bindsym $mod+4 workspace number $ws4 +bindsym $mod+5 workspace number $ws5 +bindsym $mod+6 workspace number $ws6 +bindsym $mod+7 workspace number $ws7 +bindsym $mod+8 workspace number $ws8 +bindsym $mod+9 workspace number $ws9 +bindsym $mod+0 workspace number $ws10 + +bindsym $mod+$shift+1 move container to workspace number $ws1 +bindsym $mod+$shift+2 move container to workspace number $ws2 +bindsym $mod+$shift+3 move container to workspace number $ws3 +bindsym $mod+$shift+4 move container to workspace number $ws4 +bindsym $mod+$shift+5 move container to workspace number $ws5 +bindsym $mod+$shift+6 move container to workspace number $ws6 +bindsym $mod+$shift+7 move container to workspace number $ws7 +bindsym $mod+$shift+8 move container to workspace number $ws8 +bindsym $mod+$shift+9 move container to workspace number $ws9 +bindsym $mod+$shift+0 move container to workspace number $ws10 + +######################################### +###### Window mgmt #### +######################################### +set $resize_px 10 px +bindsym $mod+$ctrl+$left resize shrink width $resize_px +bindsym $mod+$ctrl+$down resize grow height $resize_px +bindsym $mod+$ctrl+$up resize shrink height $resize_px +bindsym $mod+$ctrl+$right resize grow width $resize_px + +######################################### +###### Session mgmt #### +######################################### +bindsym $mod+$shift+e exec "i3-nagbar -t warning -m 'You pressed the exit shortcut. Do you really want to exit i3? This will end your X session.' -B 'Yes, exit i3' 'i3-msg exit'" + +# Lock the screen through logind, so the one locker the lock screen's module runs handles it (and +# suspend and lid close too). +bindsym $mod+Delete exec --no-startup-id loginctl lock-session + +# reload the configuration file +bindsym $mod+$shift+c reload + +# restart i3 inplace (preserves your layout/session, can be used to upgrade i3) +bindsym $mod+$shift+r restart + +######################################### +###### Borders #### +######################################### +default_border pixel 1 +smart_borders on + +######################################### +###### Gaps #### +######################################### +gaps inner 0 +gaps outer 0 + +# Only the accent-bearing slots are themed; background and text keep i3's own defaults. Borders are +# `pixel`, so no title bar shows: the colour says which window has focus. +# border background text indicator child_border +client.focused #de5200 #de5200 #1E2127 #de5200 #de5200 +client.urgent #900000 #900000 #ffffff #900000 #900000 + +######################################### +###### Until their modules carry them ## +######################################### +# Each line below belongs to something other than i3, named on its line. When that module is written +# it contributes the line to node-display-session, and the line goes from here in the same change. +# The launcher, the clipboard, the wallpaper, the bars, a machine model's keys and the peripherals' +# tray already contribute theirs (rofi, clipmenu, feh, i3status-rust, asus-zephyrus-g14, +# polychromatic). + +# the operator's scripts: volume, games volume, sessions, screenshot +bindsym XF86AudioRaiseVolume exec --no-startup-id volume-notify up +bindsym XF86AudioLowerVolume exec --no-startup-id volume-notify down +bindsym XF86AudioMute exec --no-startup-id volume-notify mute +bindsym XF86AudioMicMute exec --no-startup-id mic-notify +bindsym $ctrl+XF86AudioRaiseVolume exec --no-startup-id set-games-volume 5 +bindsym $ctrl+XF86AudioLowerVolume exec --no-startup-id set-games-volume -5 +bindsym $mod+$shift+Return exec --no-startup-id ~/scripts/i3-sessions/launcher.sh +bindsym --release $ctrl+$shift+x exec --no-startup-id $XDG_CONFIG_HOME/i3/scripts/screenshot.sh + +######################################### +###### Other modules' lines #### +######################################### +# Placed by the mesh from every other module's contribution (novox/hq ADR 0212): the launcher, the +# clipboard, the wallpaper, the bars, a machine model's keys, the peripherals' tray. Each module's +# under a line naming it. +# asus-zephyrus-g14 +# The laptop's own lines in i3 (module asus-zephyrus-g14, novox/hq ADR 0208, ADR 0210). Owned by the +# mesh: replaced at every push. Once the controller places contributions to node-display-session +# (ADR 0210), these become the module's contribution instead of a file in i3's directory. +# +# The keys the firmware turns into ordinary key presses. The vendor keys that reach no X client are +# triggerhappy's (/etc/triggerhappy/triggers.d/asus-g14.conf): M4 and Fn+F4/F5 for media, Fn+F7/F8 for +# the panel, Fn+F10 for the touchpad. The keyboard backlight (Fn+F2/F3) is the firmware's and asusd's. + +# Fn+F6, the screenshot key: the firmware sends Super+Shift+S. Released before it runs, because the +# screenshot grabs the pointer to select a region, which fails while the key is still held. +bindsym --release $mod+Shift+s exec --no-startup-id $XDG_CONFIG_HOME/i3/scripts/screenshot.sh + +# Fn+F9, the display key: the firmware sends Super+P. Odd workspaces to the panel, even ones to the +# external output. +bindsym $mod+p exec --no-startup-id @ASUS_LIB@/bin/zephyrus-display order + +# The keyboard backlight's level, shown when it changes. +exec --no-startup-id @ASUS_LIB@/bin/zephyrus-kbd-notify +# The laptop's own lines in i3 (module asus-zephyrus-g14, novox/hq ADR 0208, ADR 0210). Owned by the +# mesh: replaced at every push. Once the controller places contributions to node-display-session +# (ADR 0210), these become the module's contribution instead of a file in i3's directory. +# +# The model's panel and touchpad. + +# The internal panel is the primary output, where the bars' tray goes. Which monitors are on and where +# is the display server's (autorandr, run at every session start). +exec --no-startup-id @ASUS_LIB@/bin/zephyrus-display primary + +# The touchpad's settings again, by hand. X applies them itself whenever the device appears. +bindsym $mod+Shift+x exec --no-startup-id @ASUS_LIB@/bin/zephyrus-touchpad reset +# clipmenu +# The clipboard's history key (module clipmenu, novox/hq ADR 0208). Owned by the mesh: replaced at +# every push. clipmenu shows the history through `dmenu`, the node's dmenu-compatible command, which +# the holder of node-launcher answers (rofi on the workstations); the chosen entry is put back on the +# clipboard. +bindsym $mod+period exec --no-startup-id clipmenu -p Clipboard +# dunst +# The notifications' key (module dunst, novox/hq ADR 0208). Owned by the mesh: replaced at every push. +# Super+N opens the menu (rofi) of the actions and links of every notification on screen, so one is +# chosen from the keyboard as a tap chooses it on the notification. +bindsym $mod+n exec --no-startup-id dunstctl context +# feh +# The wallpaper's key (module feh, novox/hq ADR 0208). Owned by the mesh: replaced at every push. +# It puts the declared wallpaper back, after a monitor change or a wallpaper set for the session. +bindsym $mod+Shift+b exec --no-startup-id ~/.fehbg +# i3status-rust +# The bars (module i3status-rust, novox/hq ADR 0208). Owned by the mesh: replaced at every push. +# i3 reads this file through its configuration's `include ~/.config/i3/config.d/*.conf`. The bottom +# bar shows the machine; the top bar the focused window and the tray, on the primary output. The +# face is the monospace one every desktop module names. +bar { + font pango:JetBrainsMono Nerd Font 11 + position bottom + status_command i3status-rs ~/.config/i3status-rust/bottom-bar.toml + tray_output none + colors { + separator #ffffff + background #000000 + statusline #ffffff + # The text on an accent-coloured button is dark: light text on the accent was barely + # legible. + focused_workspace #de5200 #de5200 #000000 + active_workspace #de5200 #de5200 #000000 + inactive_workspace #000000 #000000 #ffffff + urgent_workspace #2f343a #900000 #ffffff + } +} + +bar { + font pango:JetBrainsMono Nerd Font 11 + position top + status_command i3status-rs ~/.config/i3status-rust/top-bar.toml + workspace_buttons no + tray_output primary + colors { + separator #ffffff + background #000000 + statusline #ffffff + } +} +# polychromatic +# The Razer peripherals' tray (module polychromatic, novox/hq ADR 0208, ADR 0212). Owned by the mesh: +# replaced at every push. This line is the tray's one start, at the session's start (exec, not +# exec_always, so a reload starts nothing). The package's own login helper starts it too while the +# application's setting "Start the tray applet when I log on" is ticked; polychromatic_check names +# that as a second start. +exec --no-startup-id polychromatic-tray-applet +# rofi +# The launcher's key bindings (module rofi, novox/hq ADR 0208). Owned by the mesh: replaced at every +# push. i3 reads this file through its configuration's `include ~/.config/i3/config.d/*.conf`, after +# the variables it sets, so $mod is i3's. +bindsym $mod+d exec --no-startup-id ~/.local/bin/rofi-launch drun +bindsym $mod+t exec --no-startup-id ~/.local/bin/rofi-launch run +bindsym $mod+Shift+t exec --no-startup-id ~/.local/bin/rofi-launch sudo +bindsym $mod+Shift+w exec --no-startup-id ~/.local/bin/rofi-launch window +bindsym $mod+Escape exec --no-startup-id ~/.local/bin/rofi-powermenu +# slack +# Slack (module slack, novox/hq ADR 0208, ADR 0212). Owned by the mesh: replaced at every push. +# Its one start: at login, to the tray (-s), with the arguments of the package's own desktop entry. +# No XDG autostart entry starts it as well; slack_check names one if it appears. +exec --no-startup-id /usr/bin/slack --gtk-version=3 -s +# The chat window. Slack owns four X windows, and the only one i3 manages has the class "slack" in +# lower case; the three of class "Slack" (the packaged entry's StartupWMClass) are its unmanaged +# helpers and its tray icon, so a rule on class="Slack" matches nothing. window_role keeps a call or +# screen-share window out of the rules. $ws3 is i3's, in scope here. +for_window [class="(?i)^slack$" window_role="browser-window"] move to workspace $ws3 +for_window [class="(?i)^slack$" window_role="browser-window"] floating disable +for_window [class="(?i)^slack$" window_role="browser-window"] border pixel 2 + +######################################### +###### Your own files #### +######################################### +include ~/.config/i3/config.d/*.conf diff --git a/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/testdata/i3/i3-only.config b/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/testdata/i3/i3-only.config new file mode 100644 index 00000000..12751ffa --- /dev/null +++ b/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/testdata/i3/i3-only.config @@ -0,0 +1,201 @@ +# i3 config file (v4), written by the mesh (module i3, novox/hq ADR 0208). Replaced at every push; +# change the module instead. i3's user guide is the reference. +# +# Other modules add to this configuration as contributions to node-display-session (novox/hq ADR +# 0212): the mesh places their lines near the end, each module's under a line naming it, where every +# variable set here ($mod, $ws1 … $ws10) is in scope. A file of yours in ~/.config/i3/config.d/ is read +# after them, by the include at the very end, and is yours. +# +# The reload watcher of this module reloads i3 when this file or a drop-in changes, after checking the +# result with i3 -C; it never reloads into a configuration with errors. + +# Font for window titles, and the bars below: the mesh's monospace face (research 026/04). +font pango:JetBrainsMono Nerd Font 11 + +# XDG autostart entries (~/.config/autostart, /etc/xdg/autostart), started once at login. +exec --no-startup-id dex --autostart --environment i3 + +######################################### +###### Keys #### +######################################### +# To find key symbols: xmodmap -pke / xmodmap -pm +set $mod Mod4 +set $alt Mod1 +set $shift Shift +set $ctrl Control + +# use these keys for focus, movement, and resize directions when reaching for +# the arrows is not convenient +set $left h +set $down j +set $up k +set $right l + +# use Mouse+$mod to drag floating windows to their wanted position +floating_modifier $mod + +# move tiling windows via drag & drop by left-clicking into the title bar, +# or left-clicking anywhere into the window while holding the floating modifier. +tiling_drag modifier titlebar + +# start a terminal: whichever the terminal module names in $TERMINAL +bindsym $mod+Return exec i3-sensible-terminal + +# kill focused window +bindsym $mod+$shift+q kill + +# change focus +bindsym $mod+$left focus left +bindsym $mod+$down focus down +bindsym $mod+$up focus up +bindsym $mod+$right focus right + +bindsym $mod+Left focus left +bindsym $mod+Down focus down +bindsym $mod+Up focus up +bindsym $mod+Right focus right + +# move focused window +bindsym $mod+$shift+$left move left +bindsym $mod+$shift+$down move down +bindsym $mod+$shift+$up move up +bindsym $mod+$shift+$right move right + +bindsym $mod+$shift+Left move left +bindsym $mod+$shift+Down move down +bindsym $mod+$shift+Up move up +bindsym $mod+$shift+Right move right + +# split in horizontal orientation +bindsym $mod+c split h +# split in vertical orientation +bindsym $mod+v split v + +# enter fullscreen mode for the focused container +bindsym $mod+f fullscreen toggle + +# change container layout (stacked, tabbed, toggle split) +bindsym $mod+s layout stacking +bindsym $mod+w layout tabbed +bindsym $mod+e layout toggle split + +# toggle tiling / floating +bindsym $mod+$shift+space floating toggle + +# change focus between tiling / floating windows +bindsym $mod+space focus mode_toggle + +# focus the parent container +bindsym $mod+a focus parent + +# alt-tab functionality +bindsym $mod+Tab workspace back_and_forth + +######################################### +###### Workspace mgmt #### +######################################### +set $ws1 "1" +set $ws2 "2" +set $ws3 "3" +set $ws4 "4" +set $ws5 "5" +set $ws6 "6" +set $ws7 "7" +set $ws8 "8" +set $ws9 "9" +set $ws10 "10" + +bindsym $mod+1 workspace number $ws1 +bindsym $mod+2 workspace number $ws2 +bindsym $mod+3 workspace number $ws3 +bindsym $mod+4 workspace number $ws4 +bindsym $mod+5 workspace number $ws5 +bindsym $mod+6 workspace number $ws6 +bindsym $mod+7 workspace number $ws7 +bindsym $mod+8 workspace number $ws8 +bindsym $mod+9 workspace number $ws9 +bindsym $mod+0 workspace number $ws10 + +bindsym $mod+$shift+1 move container to workspace number $ws1 +bindsym $mod+$shift+2 move container to workspace number $ws2 +bindsym $mod+$shift+3 move container to workspace number $ws3 +bindsym $mod+$shift+4 move container to workspace number $ws4 +bindsym $mod+$shift+5 move container to workspace number $ws5 +bindsym $mod+$shift+6 move container to workspace number $ws6 +bindsym $mod+$shift+7 move container to workspace number $ws7 +bindsym $mod+$shift+8 move container to workspace number $ws8 +bindsym $mod+$shift+9 move container to workspace number $ws9 +bindsym $mod+$shift+0 move container to workspace number $ws10 + +######################################### +###### Window mgmt #### +######################################### +set $resize_px 10 px +bindsym $mod+$ctrl+$left resize shrink width $resize_px +bindsym $mod+$ctrl+$down resize grow height $resize_px +bindsym $mod+$ctrl+$up resize shrink height $resize_px +bindsym $mod+$ctrl+$right resize grow width $resize_px + +######################################### +###### Session mgmt #### +######################################### +bindsym $mod+$shift+e exec "i3-nagbar -t warning -m 'You pressed the exit shortcut. Do you really want to exit i3? This will end your X session.' -B 'Yes, exit i3' 'i3-msg exit'" + +# Lock the screen through logind, so the one locker the lock screen's module runs handles it (and +# suspend and lid close too). +bindsym $mod+Delete exec --no-startup-id loginctl lock-session + +# reload the configuration file +bindsym $mod+$shift+c reload + +# restart i3 inplace (preserves your layout/session, can be used to upgrade i3) +bindsym $mod+$shift+r restart + +######################################### +###### Borders #### +######################################### +default_border pixel 1 +smart_borders on + +######################################### +###### Gaps #### +######################################### +gaps inner 0 +gaps outer 0 + +# Only the accent-bearing slots are themed; background and text keep i3's own defaults. Borders are +# `pixel`, so no title bar shows: the colour says which window has focus. +# border background text indicator child_border +client.focused #de5200 #de5200 #1E2127 #de5200 #de5200 +client.urgent #900000 #900000 #ffffff #900000 #900000 + +######################################### +###### Until their modules carry them ## +######################################### +# Each line below belongs to something other than i3, named on its line. When that module is written +# it contributes the line to node-display-session, and the line goes from here in the same change. +# The launcher, the clipboard, the wallpaper, the bars, a machine model's keys and the peripherals' +# tray already contribute theirs (rofi, clipmenu, feh, i3status-rust, asus-zephyrus-g14, +# polychromatic). + +# the operator's scripts: volume, games volume, sessions, screenshot +bindsym XF86AudioRaiseVolume exec --no-startup-id volume-notify up +bindsym XF86AudioLowerVolume exec --no-startup-id volume-notify down +bindsym XF86AudioMute exec --no-startup-id volume-notify mute +bindsym XF86AudioMicMute exec --no-startup-id mic-notify +bindsym $ctrl+XF86AudioRaiseVolume exec --no-startup-id set-games-volume 5 +bindsym $ctrl+XF86AudioLowerVolume exec --no-startup-id set-games-volume -5 +bindsym $mod+$shift+Return exec --no-startup-id ~/scripts/i3-sessions/launcher.sh +bindsym --release $ctrl+$shift+x exec --no-startup-id $XDG_CONFIG_HOME/i3/scripts/screenshot.sh + +######################################### +###### Other modules' lines #### +######################################### +# Placed by the mesh from every other module's contribution (novox/hq ADR 0212): the launcher, the +# clipboard, the wallpaper, the bars, a machine model's keys, the peripherals' tray. Each module's +# under a line naming it. + +######################################### +###### Your own files #### +######################################### +include ~/.config/i3/config.d/*.conf diff --git a/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/testdata/xorg/all.xinitrc b/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/testdata/xorg/all.xinitrc new file mode 100644 index 00000000..2f28bc21 --- /dev/null +++ b/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/testdata/xorg/all.xinitrc @@ -0,0 +1,86 @@ +# The mesh's block (module xorg, novox/hq ADR 0208 §5): the graphical session's start, replaced at +# every push. It ends by starting the session, so no line below it in this file runs any more. A line +# of your own that must run at every session start goes in ~/.xinitrc.local, which this block sources +# just before the session starts (the module's README lists where each of today's lines went). + +# 1. The account's environment (ADR 0203): the file every shell sources, written by node-environment. +if [ -r "/home/op/.config/mesh/environment.sh" ]; then + . "/home/op/.config/mesh/environment.sh" +fi + +# The session bus is the user manager's. pam_systemd normally hands it over; when it did not, take +# the canonical socket rather than starting a second bus, which runs every D-Bus-activated service +# twice (two notifiers, two portals) and hides one copy from the other. +if [ -z "$DBUS_SESSION_BUS_ADDRESS" ] && [ -S "/run/user/$(id -u)/bus" ]; then + DBUS_SESSION_BUS_ADDRESS="unix:path=/run/user/$(id -u)/bus" + export DBUS_SESSION_BUS_ADDRESS +fi + +# 2. The session's own words into the user manager and D-Bus activation, so the portal, the notifier +# and every user service see the display and the desktop's identity. An explicit list and never +# --all: a session's environment may hold things no user service should be able to read back. +mesh_words= +for mesh_word in DISPLAY XAUTHORITY XDG_SESSION_TYPE XDG_CURRENT_DESKTOP XDG_SESSION_DESKTOP \ + XDG_CONFIG_HOME XDG_DATA_DIRS GTK_THEME GTK2_RC_FILES QT_QPA_PLATFORMTHEME QT_STYLE_OVERRIDE \ + QT_SELECT XCURSOR_THEME XCURSOR_SIZE TERMINAL; do + if printenv "$mesh_word" >/dev/null 2>&1; then + mesh_words="$mesh_words $mesh_word" + fi +done +dbus-update-activation-environment --systemd $mesh_words +unset mesh_word mesh_words + +# 3. X resources: the mesh's, then yours in ~/.Xresources, which win. The mesh's file is merged +# without the C preprocessor, so no compiler is needed for it. +xrdb -nocpp -merge "/home/op/.config/xorg/xresources" +if [ -r "$HOME/.Xresources" ]; then + xrdb -merge "$HOME/.Xresources" +fi + +# 4. The monitors: the saved layout profile that matches the monitors connected, if one does. +if command -v autorandr >/dev/null 2>&1; then + autorandr --change >/dev/null 2>&1 || true +fi + +# 5. Other modules' session lines (ADR 0208 §4), then yours. +# gnome-keyring +# The ssh agent (module gnome-keyring, novox/hq ADR 0208): gcr's, which the account's service manager +# starts on first use from its socket. Named here, for the session and every terminal it starts, until +# the account's environment can say a path under the runtime directory (see the module's README). +SSH_AUTH_SOCK="${XDG_RUNTIME_DIR:-/run/user/$(id -u)}/gcr/ssh" +export SSH_AUTH_SOCK +# adwaita +# The appearance (module adwaita): GSettings is where the portal reads dark or light, and the portal is +# the only way it reaches Electron, Chromium, Firefox and flatpaks. Set at every session start to the +# module's default; adwaita_appearance switches it for a session. +gsettings set org.gnome.desktop.interface color-scheme 'prefer-dark' || true +gsettings set org.gnome.desktop.interface gtk-theme 'Adwaita' || true +gsettings set org.gnome.desktop.interface icon-theme 'Adwaita' || true +gsettings set org.gnome.desktop.interface cursor-theme 'Adwaita' || true +gsettings set org.gnome.desktop.interface cursor-size 24 || true +gsettings set org.gnome.desktop.interface font-name 'Inter 11' || true +gsettings set org.gnome.desktop.interface monospace-font-name 'JetBrainsMono Nerd Font 11' || true +# feh +# The wallpaper (module feh, novox/hq ADR 0208): the declared one, set once per session. +"$HOME/.fehbg" +# i3status-rust +# The bar watchdog (module i3status-rust, novox/hq ADR 0208): i3 never restarts a bar that dies; this +# brings it back. Once per session, ending with the session's own process ($$). +"$HOME/.local/bin/i3bar-watchdog" "$$" & +# screen-lock +# The lock screen (module screen-lock, novox/hq ADR 0208): the session locks after 30 minutes idle, +# the displays go to standby and suspend then and off after an hour, and xss-lock runs the locker on +# idle, before suspend and on logind's Lock. xss-lock needs this login session, so it starts here and +# not as a unit. It is started again if it exits, for as long as this session lasts ($$ is the +# session's own process, which becomes the window manager). +xset s 1800 1800 +xset dpms 1800 1800 3600 +(while kill -0 $$ 2>/dev/null; do xss-lock --transfer-sleep-lock -- "$HOME/.local/bin/screen-lock"; sleep 2; done) & +if [ -r "$HOME/.xinitrc.local" ]; then + . "$HOME/.xinitrc.local" +fi + +# 6. The session itself: the holder of node-display-session starts it from the last slot. +# i3 +# The session: i3, with 25 MiB of debug log kept in memory for i3-dump-log. +exec i3 --shmlog-size=26214400 diff --git a/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/testdata/xorg/all.xresources b/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/testdata/xorg/all.xresources new file mode 100644 index 00000000..b2b7b655 --- /dev/null +++ b/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/testdata/xorg/all.xresources @@ -0,0 +1,48 @@ +! The mesh's X resources (module xorg, novox/hq ADR 0208 §4), replaced at every push. The session's +! start merges them with xrdb -nocpp before ~/.Xresources, so your own resources there win. Without +! the preprocessor there is no #include or #define here, and a line starting with # is skipped, +! which is what the line naming each contributing module is. +! +! Font rendering, as both workstations had it. 96 DPI becomes a setting with issue 168; xorg_dpi +! changes it for the running session. +Xft.dpi: 96 +Xft.antialias: 1 +Xft.hinting: 1 +Xft.hintstyle: hintslight +Xft.rgba: rgb +# adwaita +! adwaita: the cursor, for X programs that take it from the resources. +Xcursor.theme: Adwaita +Xcursor.size: 24 +# xterm +! xterm: the face, the palette and the clipboard keys. Scoped to the XTerm class, so they +! outrank any `*name` resource of yours and touch no other X program. +XTerm*termName: xterm-256color +XTerm*saveLines: 10000 +XTerm*renderFont: true +XTerm*faceName: JetBrainsMono Nerd Font +XTerm*faceSize: 12 +XTerm*boldMode: false +XTerm*alwaysBoldMode: false +XTerm*allowBoldFonts: true +XTerm*background: #000000 +XTerm*foreground: #a8a8a8 +XTerm*color0: #000000 +XTerm*color8: #666666 +XTerm*color1: #9e1828 +XTerm*color9: #bc5766 +XTerm*color2: #008800 +XTerm*color10: #61a171 +XTerm*color3: #d2bb4b +XTerm*color11: #e7db52 +XTerm*color4: #414171 +XTerm*color12: #5085af +XTerm*color5: #963c59 +XTerm*color13: #a97a99 +XTerm*color6: #418179 +XTerm*color14: #6ba4a4 +XTerm*color7: #bebebe +XTerm*color15: #ffffff +XTerm*VT100.Translations: #override \n\ + Ctrl Shift ~Alt C: copy-selection(CLIPBOARD) \n\ + Ctrl Shift ~Alt V: insert-selection(CLIPBOARD) diff --git a/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/testdata/xorg/i3-xterm.xinitrc b/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/testdata/xorg/i3-xterm.xinitrc new file mode 100644 index 00000000..30e7b6ab --- /dev/null +++ b/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/testdata/xorg/i3-xterm.xinitrc @@ -0,0 +1,53 @@ +# The mesh's block (module xorg, novox/hq ADR 0208 §5): the graphical session's start, replaced at +# every push. It ends by starting the session, so no line below it in this file runs any more. A line +# of your own that must run at every session start goes in ~/.xinitrc.local, which this block sources +# just before the session starts (the module's README lists where each of today's lines went). + +# 1. The account's environment (ADR 0203): the file every shell sources, written by node-environment. +if [ -r "/home/op/.config/mesh/environment.sh" ]; then + . "/home/op/.config/mesh/environment.sh" +fi + +# The session bus is the user manager's. pam_systemd normally hands it over; when it did not, take +# the canonical socket rather than starting a second bus, which runs every D-Bus-activated service +# twice (two notifiers, two portals) and hides one copy from the other. +if [ -z "$DBUS_SESSION_BUS_ADDRESS" ] && [ -S "/run/user/$(id -u)/bus" ]; then + DBUS_SESSION_BUS_ADDRESS="unix:path=/run/user/$(id -u)/bus" + export DBUS_SESSION_BUS_ADDRESS +fi + +# 2. The session's own words into the user manager and D-Bus activation, so the portal, the notifier +# and every user service see the display and the desktop's identity. An explicit list and never +# --all: a session's environment may hold things no user service should be able to read back. +mesh_words= +for mesh_word in DISPLAY XAUTHORITY XDG_SESSION_TYPE XDG_CURRENT_DESKTOP XDG_SESSION_DESKTOP \ + XDG_CONFIG_HOME XDG_DATA_DIRS GTK_THEME GTK2_RC_FILES QT_QPA_PLATFORMTHEME QT_STYLE_OVERRIDE \ + QT_SELECT XCURSOR_THEME XCURSOR_SIZE TERMINAL; do + if printenv "$mesh_word" >/dev/null 2>&1; then + mesh_words="$mesh_words $mesh_word" + fi +done +dbus-update-activation-environment --systemd $mesh_words +unset mesh_word mesh_words + +# 3. X resources: the mesh's, then yours in ~/.Xresources, which win. The mesh's file is merged +# without the C preprocessor, so no compiler is needed for it. +xrdb -nocpp -merge "/home/op/.config/xorg/xresources" +if [ -r "$HOME/.Xresources" ]; then + xrdb -merge "$HOME/.Xresources" +fi + +# 4. The monitors: the saved layout profile that matches the monitors connected, if one does. +if command -v autorandr >/dev/null 2>&1; then + autorandr --change >/dev/null 2>&1 || true +fi + +# 5. Other modules' session lines (ADR 0208 §4), then yours. +if [ -r "$HOME/.xinitrc.local" ]; then + . "$HOME/.xinitrc.local" +fi + +# 6. The session itself: the holder of node-display-session starts it from the last slot. +# i3 +# The session: i3, with 25 MiB of debug log kept in memory for i3-dump-log. +exec i3 --shmlog-size=26214400 diff --git a/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/testdata/xorg/i3-xterm.xresources b/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/testdata/xorg/i3-xterm.xresources new file mode 100644 index 00000000..d0b1b65a --- /dev/null +++ b/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/testdata/xorg/i3-xterm.xresources @@ -0,0 +1,44 @@ +! The mesh's X resources (module xorg, novox/hq ADR 0208 §4), replaced at every push. The session's +! start merges them with xrdb -nocpp before ~/.Xresources, so your own resources there win. Without +! the preprocessor there is no #include or #define here, and a line starting with # is skipped, +! which is what the line naming each contributing module is. +! +! Font rendering, as both workstations had it. 96 DPI becomes a setting with issue 168; xorg_dpi +! changes it for the running session. +Xft.dpi: 96 +Xft.antialias: 1 +Xft.hinting: 1 +Xft.hintstyle: hintslight +Xft.rgba: rgb +# xterm +! xterm: the face, the palette and the clipboard keys. Scoped to the XTerm class, so they +! outrank any `*name` resource of yours and touch no other X program. +XTerm*termName: xterm-256color +XTerm*saveLines: 10000 +XTerm*renderFont: true +XTerm*faceName: JetBrainsMono Nerd Font +XTerm*faceSize: 12 +XTerm*boldMode: false +XTerm*alwaysBoldMode: false +XTerm*allowBoldFonts: true +XTerm*background: #000000 +XTerm*foreground: #a8a8a8 +XTerm*color0: #000000 +XTerm*color8: #666666 +XTerm*color1: #9e1828 +XTerm*color9: #bc5766 +XTerm*color2: #008800 +XTerm*color10: #61a171 +XTerm*color3: #d2bb4b +XTerm*color11: #e7db52 +XTerm*color4: #414171 +XTerm*color12: #5085af +XTerm*color5: #963c59 +XTerm*color13: #a97a99 +XTerm*color6: #418179 +XTerm*color14: #6ba4a4 +XTerm*color7: #bebebe +XTerm*color15: #ffffff +XTerm*VT100.Translations: #override \n\ + Ctrl Shift ~Alt C: copy-selection(CLIPBOARD) \n\ + Ctrl Shift ~Alt V: insert-selection(CLIPBOARD) diff --git a/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/xorg_golden_test.go b/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/xorg_golden_test.go new file mode 100644 index 00000000..5efbdab6 --- /dev/null +++ b/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/xorg_golden_test.go @@ -0,0 +1,98 @@ +package main + +import ( + "encoding/json" + "os" + "path/filepath" + "testing" + "time" +) + +// The graphical session's two files, xinitrc and xresources, filled by this client from the items of the collections +// shell:xinitrc: and shell:xresources: their seven contributors declare, are byte for byte what the +// controller composed from the old shell entries before the move (design 57 L3, issue 807). The module files are the +// catalogue's own; testdata/xorg holds the two files as the controller composed them, for a machine with all seven +// contributors and for one with only i3 and xterm. The not handed half is pinned in mesh-controller by the same files. + +func TestNatsTheSessionFilesFilledFromTheirContributorsItemsAreWhatTheOldFormComposed(t *testing.T) { + modules := filepath.Join("..", "..", "..") + read := func(path string) string { + b, err := os.ReadFile(filepath.Join(modules, path)) + if err != nil { + t.Fatal(err) + } + return string(b) + } + all := []string{"adwaita", "i3status-rust", "i3", "feh", "gnome-keyring", "screen-lock", "xterm"} + for name, contributors := range map[string][]string{"all": all, "i3-xterm": {"i3", "xterm"}} { + t.Run(name, func(t *testing.T) { + facts := map[string]string{"machine:account-home": "/home/op"} + tpl, _ := templates(t, map[string]string{"xinitrc": read("xorg/files/xinitrc"), + "xresources": read("xorg/files/xresources")}) + collector := Plan{Templates: tpl, Placement: facts, Files: []PlannedFile{ + {ID: "xinitrc", Path: "/session/xinitrc", Template: "xinitrc", OwnerAtRuntime: "owned"}, + {ID: "xresources", Path: "/session/xresources", Template: "xresources", OwnerAtRuntime: "owned"}}} + for _, f := range []string{"xinitrc", "xresources"} { + for _, slot := range []string{"first", "normal", "last"} { + collector.Collections = append(collector.Collections, + PlannedCollection{Name: "shell:" + f + ":" + slot, Comment: "#"}) + } + } + var manifest struct { + Moved bool `json:"moved"` + } + if err := json.Unmarshal([]byte(read("xorg/module.json")), &manifest); err != nil || !manifest.Moved { + t.Fatalf("xorg is not moved: %v", err) + } + b := startBus(t) + e := startEngine(t, map[string]string{"/session/xinitrc": "owned", "/session/xresources": "owned"}) + b.assign(t, "xorg", collector) + for _, m := range contributors { + var c struct { + Moved bool `json:"moved"` + Shell []any `json:"shell"` + Items []struct { + Collection string `json:"collection"` + Name string `json:"name"` + Attributes map[string]string `json:"attributes"` + Template string `json:"template"` + } `json:"items"` + } + if err := json.Unmarshal([]byte(read(m+"/module.json")), &c); err != nil { + t.Fatal(err) + } + if !c.Moved || len(c.Shell) != 0 { + t.Fatalf("%s is not moved or still holds shell entries", m) + } + texts := map[string]string{} + var planned []PlannedItem + for _, it := range c.Items { + if it.Collection != "shell:xinitrc:first" && it.Collection != "shell:xinitrc:normal" && + it.Collection != "shell:xinitrc:last" && it.Collection != "shell:xresources:normal" { + continue + } + id := filepath.Base(it.Template) + texts[id] = read(m + "/" + it.Template) + planned = append(planned, PlannedItem{Collection: it.Collection, Name: it.Name, Template: id, + Attributes: it.Attributes}) + } + if len(planned) == 0 { + t.Fatalf("%s declares no item of the session files", m) + } + tp, _ := templates(t, texts) + b.assign(t, m, Plan{Templates: tp, Items: planned, Placement: facts}) + } + aClient(t, b, e, t.TempDir(), time.Hour) + for _, f := range []string{"xinitrc", "xresources"} { + want, err := os.ReadFile(filepath.Join("testdata", "xorg", name+"."+f)) + if err != nil { + t.Fatal(err) + } + eventually(t, "the "+f+" is what it was", func() bool { return e.read(t, "/session/"+f) == string(want) }) + if t.Failed() { + t.Logf("got:\n%s", e.read(t, "/session/"+f)) + } + } + }) + } +} diff --git a/modules/mesh-provisioner-client/go.mod b/modules/mesh-provisioner-client/go.mod new file mode 100644 index 00000000..39b43e4a --- /dev/null +++ b/modules/mesh-provisioner-client/go.mod @@ -0,0 +1,22 @@ +module mesh-provisioner-client + +go 1.25.0 + +require ( + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261013235212-c8b07242a009 + github.com/nats-io/nats-server/v2 v2.11.17 + github.com/nats-io/nats.go v1.51.0 +) + +require ( + github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op // indirect + github.com/google/go-tpm v0.9.8 // indirect + github.com/klauspost/compress v1.18.5 // indirect + github.com/minio/highwayhash v1.0.4 // indirect + github.com/nats-io/jwt/v2 v2.8.1 // indirect + github.com/nats-io/nkeys v0.4.15 // indirect + github.com/nats-io/nuid v1.0.1 // indirect + golang.org/x/crypto v0.50.0 // indirect + golang.org/x/sys v0.43.0 // indirect + golang.org/x/time v0.15.0 // indirect +) diff --git a/modules/mesh-provisioner-client/go.sum b/modules/mesh-provisioner-client/go.sum new file mode 100644 index 00000000..da3d1f5d --- /dev/null +++ b/modules/mesh-provisioner-client/go.sum @@ -0,0 +1,27 @@ +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261013235212-c8b07242a009 h1:I9rm8HhYRJ0+mzQQz+VAH0kgv2YJWkvctEc67Yd0PX8= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261013235212-c8b07242a009/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY= +github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= +github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= +github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= +github.com/google/go-tpm v0.9.8/go.mod h1:h9jEsEECg7gtLis0upRBQU+GhYVH6jMjrFxI8u6bVUY= +github.com/klauspost/compress v1.18.5 h1:/h1gH5Ce+VWNLSWqPzOVn6XBO+vJbCNGvjoaGBFW2IE= +github.com/klauspost/compress v1.18.5/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= +github.com/minio/highwayhash v1.0.4 h1:asJizugGgchQod2ja9NJlGOWq4s7KsAWr5XUc9Clgl4= +github.com/minio/highwayhash v1.0.4/go.mod h1:GGYsuwP/fPD6Y9hMiXuapVvlIUEhFhMTh0rxU3ik1LQ= +github.com/nats-io/jwt/v2 v2.8.1 h1:V0xpGuD/N8Mi+fQNDynXohVvp7ZztevW5io8CUWlPmU= +github.com/nats-io/jwt/v2 v2.8.1/go.mod h1:nWnOEEiVMiKHQpnAy4eXlizVEtSfzacZ1Q43LIRavZg= +github.com/nats-io/nats-server/v2 v2.11.17 h1:GKEghcFK6A+aFx11Yf1LjgLC3txAwvyhnYzhBIQZA8I= +github.com/nats-io/nats-server/v2 v2.11.17/go.mod h1:B1sFVz4StNosQ903ak4N1G01Fl/9f8e06mXpFIE2K24= +github.com/nats-io/nats.go v1.51.0 h1:ByW84XTz6W03GSSsygsZcA+xgKK8vPGaa/FCAAEHnAI= +github.com/nats-io/nats.go v1.51.0/go.mod h1:26HypzazeOkyO3/mqd1zZd53STJN0EjCYF9Uy2ZOBno= +github.com/nats-io/nkeys v0.4.15 h1:JACV5jRVO9V856KOapQ7x+EY8Jo3qw1vJt/9Jpwzkk4= +github.com/nats-io/nkeys v0.4.15/go.mod h1:CpMchTXC9fxA5zrMo4KpySxNjiDVvr8ANOSZdiNfUrs= +github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= +github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= +golang.org/x/crypto v0.50.0 h1:zO47/JPrL6vsNkINmLoo/PH1gcxpls50DNogFvB5ZGI= +golang.org/x/crypto v0.50.0/go.mod h1:3muZ7vA7PBCE6xgPX7nkzzjiUq87kRItoJQM1Yo8S+Q= +golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.43.0 h1:Rlag2XtaFTxp19wS8MXlJwTvoh8ArU6ezoyFsMyCTNI= +golang.org/x/sys v0.43.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= +golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= diff --git a/modules/mesh-provisioner-client/module.json b/modules/mesh-provisioner-client/module.json new file mode 100644 index 00000000..af84e540 --- /dev/null +++ b/modules/mesh-provisioner-client/module.json @@ -0,0 +1,59 @@ +{ + "module": "mesh-provisioner-client", + "version": "1", + "claims": [ + { + "name": "node-provisioner", + "scope": "node" + } + ], + "own-secrets": { + "provisioner-broker": "${dir:state}/broker" + }, + "secrets-owner": "mesh-provisioner-client", + "resources": [ + { + "id": "account", + "type": "user", + "name": "mesh-provisioner-client", + "shell": "/usr/bin/nologin", + "home": "/var/lib/mesh-provisioner-client" + }, + { + "id": "state", + "type": "directory", + "mode": "0700", + "owner": "mesh-provisioner-client", + "place": "." + }, + { + "id": "client", + "type": "process", + "name": "mesh-provisioner-client", + "artifact": "client", + "run": [ + "./mesh-provisioner-client" + ], + "env": { + "MESH_PROVISIONER_CREDENTIAL": "${dir:state}/broker", + "MESH_PROVISIONER_STATE": "${dir:state}" + }, + "user": "mesh-provisioner-client", + "health": { + "kind": "unit" + } + } + ], + "build": { + "artifacts": [ + { + "name": "client", + "kind": "bundle", + "language": "go", + "system": "arch", + "from": "cmd/mesh-provisioner-client", + "binary": "mesh-provisioner-client" + } + ] + } +} diff --git a/modules/mesh-provisioner/README.md b/modules/mesh-provisioner/README.md new file mode 100644 index 00000000..88c246b3 --- /dev/null +++ b/modules/mesh-provisioner/README.md @@ -0,0 +1,33 @@ +# mesh-provisioner + +Holds the `mesh-provisioner` seat once for the mesh: the provisioner's server (novox/hq ADR 0349 clause 3, design 57 +P4). Go, one binary (`cmd/mesh-provisioner`), served by a runtime of its own on its own bus account (`runs-as`), so +that its account alone writes the registry. The provisioner client on each node (`node-provisioner`, design 57 P5) +is the module `mesh-provisioner-client`. + +## What it keeps + +- `mesh-provisioner_registry`: which domain answers which kind of placeholder, keyed by the kind (`config`, + `secret`, `bound` — the provider's own bucket, `_bound`, keyed + `..[.]` (mesh-sdk `go/provider`) — `consumers` — a provider's consumers' + credentials, asked of the vault by the provider machine's client (novox/hq ADR 0373) — `collected` — and `shell`, `environment` and `contribution`, read as collections of the same + name (design 57 L1) — and the placement namespaces `dir`, `access`, `port`, `seat`, `machine`). Each + answer names the owning module and where a client finds the values: a domain's bucket and its key shape + (placement's is the controller's assignments mirror, whose value carries the provisioner's plan of a module handed + to it), or a seat's verb asked on request and reply (a secret is never in a bucket). Compiled in + `registry.go`, written at start and every hour, only where it differs; a kind no domain answers is deleted. +- `mesh-provisioner_needs`: what each module on each node needs, `...`. +- `mesh-provisioner_applied`: what is applied where, `.`. +- `mesh-provisioner_collect`: the items of every collection, `..`, the collection + with a character a key's token cannot hold written `=` (`shell=3Azsh=3Anormal` is `shell:zsh:normal`), and + read back by its name. + +The last three are written by each node's provisioner client for its own node's keys alone: the controller grants +the client `.>` of each and nothing wider (mesh-controller `internal/broker/provisioner.go`), and a provider +reads the needs without writing them. This module writes only the registry. + +## The verb + +`mesh-provisioner.provisioning` with `what` `registry` (the default), `needs`, `applied` or `collect`, narrowed by +`node` and `module` (for `collect`, the contributor) and, for `collect`, `collection`. Reads only. A key out of its +bucket's shape is said in the answer, never dropped quietly. diff --git a/modules/mesh-provisioner/cmd/mesh-provisioner/main.go b/modules/mesh-provisioner/cmd/mesh-provisioner/main.go new file mode 100644 index 00000000..3955516d --- /dev/null +++ b/modules/mesh-provisioner/cmd/mesh-provisioner/main.go @@ -0,0 +1,95 @@ +// mesh-provisioner: the provisioner's server, holder of the mesh-provisioner seat once for the mesh (novox/hq ADR +// 0349 clause 3, design 57 P4). A Go bundle served by a runtime of its own, on its own bus account (`runs-as`, ADR +// 0259 §8), keeping four buckets: mesh-provisioner_registry (which domain answers which kind of placeholder, its +// alone), and mesh-provisioner_needs, _applied and _collect, which each machine's provisioner client writes for its +// own machine (the controller grants the client those keys alone, issue 670). stdout is the MCP channel; what this +// module says, it says on stderr. +package main + +import ( + "fmt" + "os" + "time" + + stdio "git.novox.be/novox/mesh-sdk/go" +) + +// Seat is the role this module holds, once for the mesh, and Verb its one verb. +const ( + Seat = "mesh-provisioner" + Verb = "provisioning" +) + +func logf(format string, a ...any) { fmt.Fprintf(os.Stderr, format+"\n", a...) } + +// runtimeStore is the bus, through the runtime that launched this bundle. +type runtimeStore struct{} + +func (runtimeStore) Get(state, key string) (*Entry, error) { + e, err := stdio.State(state).Get(key) + if err != nil || e == nil { + return nil, err + } + return &Entry{Key: e.Key, Value: e.Value, Revision: e.Revision}, nil +} + +func (runtimeStore) Put(state, key string, value any) error { + _, err := stdio.State(state).Put(key, value) + return err +} + +func (runtimeStore) Delete(state, key string) error { return stdio.State(state).Delete(key) } + +func (runtimeStore) Keys(state string) ([]string, error) { return stdio.State(state).Keys() } + +func main() { + go publishing(runtimeStore{}) + if err := stdio.Serve("", Tools(runtimeStore{})); err != nil { + logf("%v", err) + os.Exit(1) + } +} + +// publishing writes the registry once the runtime answers, and again every hour, so a registry emptied by hand or +// a bucket made again is written back without a restart. The runtime answers only once Serve runs, so the first +// pass is tried again until it lands. +func publishing(s Store) { + for wait := 2 * time.Second; ; { + time.Sleep(wait) + written, deleted, err := Publish(s) + switch { + case err != nil: + logf("[mesh-provisioner] the registry is not written yet (%v); trying again in %s", err, wait) + wait = min(wait*2, time.Minute) + continue + case len(written) > 0 || len(deleted) > 0: + logf("[mesh-provisioner] the registry: wrote %v, deleted %v", written, deleted) + } + wait = time.Hour + } +} + +// Tools is the seat's verb as the runtime serves it. +func Tools(s Store) []stdio.Tool { + return []stdio.Tool{{ + Name: Seat + "." + Verb, + Description: "Read what the provisioner keeps (novox/hq ADR 0349 clause 3): the registry (which domain " + + "answers which kind of placeholder, the default), the needs each module on each node recorded, what is " + + "applied where, or the items of every collection, narrowed by node and module, and collect by " + + "collection. Reads only: each node's provisioner client writes its own node's needs, applied and items.", + Input: map[string]any{"type": "object", "properties": map[string]any{ + "what": map[string]any{"type": "string", "enum": []string{Registry, Needs, Applied, Collect}, + "description": "registry when not given"}, + "node": map[string]any{"type": "string", "description": "a node"}, + "module": map[string]any{"type": "string", "description": "a module: for collect, a contributor"}, + "collection": map[string]any{"type": "string", "description": "for collect: a collection"}, + }}, + Run: func(args map[string]any) (any, error) { + q, err := queryOf(args) + if err != nil { + return nil, err + } + return Do(s, q) + }, + }} +} diff --git a/modules/mesh-provisioner/cmd/mesh-provisioner/provisioner_test.go b/modules/mesh-provisioner/cmd/mesh-provisioner/provisioner_test.go new file mode 100644 index 00000000..4661b226 --- /dev/null +++ b/modules/mesh-provisioner/cmd/mesh-provisioner/provisioner_test.go @@ -0,0 +1,306 @@ +package main + +import ( + "encoding/json" + "os" + "slices" + "sort" + "strings" + "sync" + "testing" +) + +// memStore is the bus as mesh-provisioner reaches it, in memory: the states it keeps. +type memStore struct { + mu sync.Mutex + states map[string]map[string][]byte + puts int +} + +func newStore() *memStore { return &memStore{states: map[string]map[string][]byte{}} } + +func (m *memStore) Get(state, key string) (*Entry, error) { + m.mu.Lock() + defer m.mu.Unlock() + v, ok := m.states[state][key] + if !ok { + return nil, nil + } + return &Entry{Key: key, Value: append([]byte(nil), v...)}, nil +} + +func (m *memStore) Put(state, key string, value any) error { + raw, err := json.Marshal(value) + if err != nil { + return err + } + m.mu.Lock() + defer m.mu.Unlock() + if m.states[state] == nil { + m.states[state] = map[string][]byte{} + } + m.states[state][key] = raw + m.puts++ + return nil +} + +func (m *memStore) Delete(state, key string) error { + m.mu.Lock() + defer m.mu.Unlock() + delete(m.states[state], key) + return nil +} + +func (m *memStore) Keys(state string) ([]string, error) { + m.mu.Lock() + defer m.mu.Unlock() + var keys []string + for k := range m.states[state] { + keys = append(keys, k) + } + sort.Strings(keys) + return keys, nil +} + +// write is a client's write, as the bus lets it through for its own machine's keys. +func (m *memStore) write(t *testing.T, state, key, raw string) { + t.Helper() + if err := m.Put(state, key, json.RawMessage(raw)); err != nil { + t.Fatal(err) + } +} + +// The registry answers every kind a moved module's template may name (ADR 0349 clause 2): config, secret, bound, +// collected and the placement namespaces, each with its domain and where a client finds it. +func TestTheRegistryAnswersEveryKindATemplateMayName(t *testing.T) { + var kinds []string + for _, a := range TheRegistry { + kinds = append(kinds, a.Kind) + if a.Domain == "" || a.Why == "" || !strings.HasPrefix(a.Decision, "novox/hq ADR ") { + t.Errorf("%s: an answer names its domain, why and its decision: %+v", a.Kind, a) + } + switch a.From { + case FromBucket: + if a.Bucket == "" || !strings.HasPrefix(a.Key, ".") { + t.Errorf("%s: a value in a bucket names the bucket and a key with the node first: %+v", a.Kind, a) + } + if a.Ask != "" { + t.Errorf("%s: from a bucket, asked of nobody: %+v", a.Kind, a) + } + case FromAsk: + if a.Ask == "" || a.Bucket != "" { + t.Errorf("%s: a value asked for names its verb and no bucket: %+v", a.Kind, a) + } + default: + t.Errorf("%s: from %q", a.Kind, a.From) + } + } + want := []string{"config", "secret", "bound", "consumers", "collected", "shell", "environment", "contribution", + "dir", "access", "port", "seat", "machine", "backup"} + if !slices.Equal(kinds, want) { + t.Errorf("the registry's kinds are %v, want %v", kinds, want) + } + // The buckets a client is granted to read its node's keys of (mesh-controller internal/broker/provisioner.go): + // the config domain's resolved values, this module's own collections, and placement in each module's assignment, + // where the controller carries the provisioner's plan (issue 671): a client never reads the node's declaration. + for kind, bucket := range map[string]string{"config": "mesh-config_resolved", "collected": "mesh-provisioner_" + Collect, + "shell": "mesh-provisioner_" + Collect, "environment": "mesh-provisioner_" + Collect, + "contribution": "mesh-provisioner_" + Collect, + "dir": "mesh-controller_assignments", "access": "mesh-controller_assignments", + "port": "mesh-controller_assignments", "seat": "mesh-controller_assignments", + "machine": "mesh-controller_assignments", "backup": "mesh-controller_assignments"} { + i := slices.IndexFunc(TheRegistry, func(a Answer) bool { return a.Kind == kind }) + if TheRegistry[i].Bucket != bucket { + t.Errorf("%s is found in %s, want %s", kind, TheRegistry[i].Bucket, bucket) + } + } + // A secret is never in a bucket (ADR 0201 §13): asked of the vault. + i := slices.IndexFunc(TheRegistry, func(a Answer) bool { return a.Kind == "secret" }) + if a := TheRegistry[i]; a.From != FromAsk || a.Ask != "mesh-vault.get" { + t.Errorf("a secret is answered by %+v, want asked of mesh-vault.get", a) + } + // A consumer's values are in its provider's own bucket, one entry per consumer, keyed as the providers' helper + // writes it (mesh-sdk go/provider, issue 678): the consumer's machine first. + i = slices.IndexFunc(TheRegistry, func(a Answer) bool { return a.Kind == "bound" }) + if a := TheRegistry[i]; a.From != FromBucket || a.Bucket != "_bound" || + a.Key != "..[.]" { + t.Errorf("a bound value is answered by %+v, want the provider's _bound, "+ + "..[.]", a) + } + // A provider's consumers' credentials are the vault's, asked by the provider machine's client (ADR 0373). + i = slices.IndexFunc(TheRegistry, func(a Answer) bool { return a.Kind == "consumers" }) + if a := TheRegistry[i]; a.Domain != "mesh-vault" || a.From != FromAsk || a.Ask != "mesh-vault.get" || + a.Decision != "novox/hq ADR 0373" { + t.Errorf("a provider's consumers are answered by %+v, want the vault, asked of mesh-vault.get (ADR 0373)", a) + } +} + +// The registry is written whole at start, a pass that finds it equal writes nothing, a changed answer is written +// back, and a kind no domain answers any more is deleted. +func TestTheRegistryIsWrittenBackAndAQuietPassWritesNothing(t *testing.T) { + s := newStore() + written, deleted, err := Publish(s) + if err != nil { + t.Fatal(err) + } + if len(written) != len(TheRegistry) || len(deleted) != 0 { + t.Fatalf("the first pass wrote %v and deleted %v", written, deleted) + } + for _, a := range TheRegistry { + e, _ := s.Get(Registry, a.Kind) + var got Answer + if e == nil || json.Unmarshal(e.Value, &got) != nil || got != a { + t.Errorf("the registry's %s holds %v, want %+v", a.Kind, e, a) + } + } + puts := s.puts + if written, deleted, err := Publish(s); err != nil || len(written)+len(deleted) != 0 || s.puts != puts { + t.Fatalf("a quiet pass wrote %v, deleted %v (%v)", written, deleted, err) + } + s.write(t, Registry, "config", `{"kind":"config","domain":"mesh-controller"}`) + s.write(t, Registry, "setting", `{"kind":"setting"}`) + written, deleted, err = Publish(s) + if err != nil || !slices.Equal(written, []string{"config"}) || !slices.Equal(deleted, []string{"setting"}) { + t.Fatalf("a changed and an unknown kind: wrote %v, deleted %v (%v)", written, deleted, err) + } +} + +func clientWrites(t *testing.T) *memStore { + s := newStore() + if _, _, err := Publish(s); err != nil { + t.Fatal(err) + } + s.write(t, Needs, "g14.dunst.config.font-size", `{"at":"2026-10-14T20:00:00Z"}`) + s.write(t, Needs, "g14.dunst.secret.token", `{}`) + s.write(t, Needs, "shanks.dunst.config.font-size", `{}`) + s.write(t, Needs, "shanks.gitea.bound.database.user", `{}`) + s.write(t, Applied, "g14.dunst", `{"build":"b1","outcome":"applied"}`) + s.write(t, Applied, "shanks.dunst", `{"build":"b1","outcome":"failed","reason":"waits for config font-size"}`) + s.write(t, Collect, "g14.zsh-snippets.git", `{}`) + s.write(t, Collect, "g14.bar-blocks.disk-load", `{}`) + s.write(t, Collect, "g14.shell=3Azsh=3Anormal.mesh-cli", `{}`) + return s +} + +func keysOf(r Read) []string { + var out []string + for _, row := range r.Rows { + out = append(out, row.Key) + } + return out +} + +// Each bucket is read by its shape, its parts named, and narrowed by node, module and collection. +func TestEachBucketIsReadByItsShapeAndNarrowed(t *testing.T) { + s := clientWrites(t) + for _, c := range []struct { + q Query + want []string + }{ + {Query{}, []string{"access", "backup", "bound", "collected", "config", "consumers", "contribution", "dir", "environment", + "machine", "port", "seat", "secret", "shell"}}, + {Query{What: Needs}, []string{"g14.dunst.config.font-size", "g14.dunst.secret.token", + "shanks.dunst.config.font-size", "shanks.gitea.bound.database.user"}}, + {Query{What: Needs, Node: "g14"}, []string{"g14.dunst.config.font-size", "g14.dunst.secret.token"}}, + {Query{What: Needs, Module: "dunst", Node: "shanks"}, []string{"shanks.dunst.config.font-size"}}, + {Query{What: Applied, Module: "dunst"}, []string{"g14.dunst", "shanks.dunst"}}, + {Query{What: Applied, Node: "ace"}, nil}, + {Query{What: Collect, Collection: "zsh-snippets"}, []string{"g14.zsh-snippets.git"}}, + {Query{What: Collect, Module: "disk-load"}, []string{"g14.bar-blocks.disk-load"}}, + // An old namespace's collection, by its name, not its key's token (design 57 L1). + {Query{What: Collect, Collection: "shell:zsh:normal"}, []string{"g14.shell=3Azsh=3Anormal.mesh-cli"}}, + } { + r, err := Do(s, c.q) + if err != nil { + t.Fatalf("%+v: %v", c.q, err) + } + if got := keysOf(r); !slices.Equal(got, c.want) || r.Count != len(c.want) { + t.Errorf("%+v: read %v (%d), want %v", c.q, got, r.Count, c.want) + } + } + r, _ := Do(s, Query{What: Needs, Node: "shanks", Module: "gitea"}) + if row := r.Rows[0]; row.Node != "shanks" || row.Module != "gitea" || row.Kind != "bound" || row.Name != "database.user" { + t.Errorf("a need's parts: %+v", row) + } + r, _ = Do(s, Query{What: Collect, Collection: "bar-blocks"}) + if row := r.Rows[0]; row.Node != "g14" || row.Collection != "bar-blocks" || row.Module != "disk-load" { + t.Errorf("an item's parts: %+v", row) + } + r, _ = Do(s, Query{What: Applied, Node: "shanks"}) + if string(r.Rows[0].Value) != `{"build":"b1","outcome":"failed","reason":"waits for config font-size"}` { + t.Errorf("what is applied is read as the client wrote it: %s", r.Rows[0].Value) + } +} + +// A key out of its bucket's shape is said, never dropped quietly. +func TestAKeyOutOfItsBucketsShapeIsSaid(t *testing.T) { + s := clientWrites(t) + s.write(t, Applied, "g14.dunst.extra", `{}`) + r, err := Do(s, Query{What: Applied}) + if err != nil { + t.Fatal(err) + } + if r.Count != 2 || !strings.Contains(r.Said, "g14.dunst.extra") || !strings.Contains(r.Said, ".") { + t.Errorf("an unshaped key: %d rows, said %q", r.Count, r.Said) + } +} + +// The verb's arguments are read strictly, and a narrowing that means nothing is refused rather than ignored. +func TestTheVerbsArgumentsAreReadStrictly(t *testing.T) { + s := clientWrites(t) + run := Tools(s)[0].Run + if Tools(s)[0].Name != "mesh-provisioner.provisioning" { + t.Fatalf("the verb is served as %s", Tools(s)[0].Name) + } + for _, args := range []map[string]any{ + {"what": "needs", "caller": "g14.dunst"}, + {"node": 3}, + {"what": "secrets"}, + {"node": "g14"}, + {"what": "needs", "collection": "zsh-snippets"}, + } { + if _, err := run(args); err == nil { + t.Errorf("%v was taken", args) + } + } + got, err := run(map[string]any{"what": "applied", "node": " g14 "}) + if err != nil { + t.Fatal(err) + } + if r := got.(Read); r.Count != 1 || r.Said != "1 in applied for node g14" { + t.Errorf("a read: %+v", r) + } +} + +// The manifest declares the seat, its verb, the four buckets this code keeps, and an account of its own, so that +// its account alone (with each client for its machine's keys) writes them. +func TestTheManifestDeclaresWhatTheCodeKeeps(t *testing.T) { + raw, err := os.ReadFile("../../module.json") + if err != nil { + t.Fatal(err) + } + var m struct { + Module string `json:"module"` + RunsAs string `json:"runs-as"` + Claims []struct { + Name string `json:"name"` + Scope string `json:"scope"` + Serves []string `json:"serves"` + } `json:"claims"` + State []string `json:"state"` + OwnSecrets map[string]string `json:"own-secrets"` + } + if err := json.Unmarshal(raw, &m); err != nil { + t.Fatal(err) + } + if m.Module != "mesh-provisioner" || m.RunsAs != "mesh-provisioner" || m.OwnSecrets["broker"] == "" { + t.Errorf("an account of its own: module %q, runs-as %q, own secrets %v", m.Module, m.RunsAs, m.OwnSecrets) + } + if len(m.Claims) != 1 || m.Claims[0].Name != Seat || m.Claims[0].Scope != "mesh" || + !slices.Equal(m.Claims[0].Serves, []string{Verb}) { + t.Errorf("the claim: %+v", m.Claims) + } + if want := []string{Registry, Needs, Applied, Collect}; !slices.Equal(m.State, want) { + t.Errorf("the state is %v, want %v", m.State, want) + } +} diff --git a/modules/mesh-provisioner/cmd/mesh-provisioner/provisioning.go b/modules/mesh-provisioner/cmd/mesh-provisioner/provisioning.go new file mode 100644 index 00000000..c5b0deba --- /dev/null +++ b/modules/mesh-provisioner/cmd/mesh-provisioner/provisioning.go @@ -0,0 +1,216 @@ +package main + +import ( + "encoding/json" + "fmt" + "slices" + "sort" + "strconv" + "strings" +) + +// The verb `provisioning`: a read of what this module keeps, narrowed by node and module. It writes nothing: +// the registry is this module's, and the needs, the applied and the items are each machine's client's, for its +// own machine (ADR 0349 clause 3). + +// Entry is one key's current value. +type Entry struct { + Key string + Value json.RawMessage + Revision uint64 +} + +// Store is the bus as this module reaches it: the state it keeps. +type Store interface { + Get(state, key string) (*Entry, error) + Put(state, key string, value any) error + Delete(state, key string) error + Keys(state string) ([]string, error) +} + +// Query is one call of the verb. +type Query struct { + What string + Node string + Module string + Collection string +} + +// Row is one key of a bucket, its parts named, and its value as written. +type Row struct { + Key string `json:"key"` + Node string `json:"node,omitempty"` + Module string `json:"module,omitempty"` + Kind string `json:"kind,omitempty"` + Name string `json:"name,omitempty"` + Collection string `json:"collection,omitempty"` + Value json.RawMessage `json:"value"` +} + +// Read is a read's answer: the bucket read, its rows, how many, and what narrowed them. +type Read struct { + What string `json:"what"` + Rows []Row `json:"rows"` + Count int `json:"count"` + Said string `json:"said"` +} + +// shapes are the keys of each bucket, the node first, as the clients write them. +var shapes = map[string]string{ + Needs: "...", + Applied: ".", + Collect: "..", +} + +// rowOf names a key's parts by its bucket's shape, or says the key is not in that shape. +func rowOf(what, key string) (Row, bool) { + r := Row{Key: key} + switch what { + case Registry: + r.Kind = key + return r, key != "" + case Needs: + parts := strings.SplitN(key, ".", 4) + if len(parts) != 4 || slices.Contains(parts, "") { + return r, false + } + r.Node, r.Module, r.Kind, r.Name = parts[0], parts[1], parts[2], parts[3] + case Applied: + parts := strings.Split(key, ".") + if len(parts) != 2 || slices.Contains(parts, "") { + return r, false + } + r.Node, r.Module = parts[0], parts[1] + case Collect: + parts := strings.SplitN(key, ".", 3) + if len(parts) != 3 || slices.Contains(parts, "") { + return r, false + } + // The contributor is the module whose item it is; the collection as its key's token writes it, a character a + // token cannot hold written `=` (`shell=3Azsh=3Anormal` is `shell:zsh:normal`). + collection, ok := fromKeyToken(parts[1]) + if !ok { + return r, false + } + r.Node, r.Collection, r.Module = parts[0], collection, parts[2] + default: + return r, false + } + return r, true +} + +// Do answers a read. +func Do(s Store, q Query) (Read, error) { + what := q.What + if what == "" { + what = Registry + } + if _, ok := shapes[what]; !ok && what != Registry { + return Read{}, fmt.Errorf("provisioning reads registry, needs, applied or collect, not %q", what) + } + if what == Registry && (q.Node != "" || q.Module != "" || q.Collection != "") { + return Read{}, fmt.Errorf("the registry is the same for every node and module; it is not narrowed") + } + if q.Collection != "" && what != Collect { + return Read{}, fmt.Errorf("collection narrows collect alone, not %s", what) + } + keys, err := s.Keys(what) + if err != nil { + return Read{}, fmt.Errorf("listing %s: %w", what, err) + } + out := Read{What: what, Rows: []Row{}} + var unshaped []string + for _, k := range keys { + r, ok := rowOf(what, k) + if !ok { + unshaped = append(unshaped, k) + continue + } + if (q.Node != "" && r.Node != q.Node) || (q.Module != "" && r.Module != q.Module) || + (q.Collection != "" && r.Collection != q.Collection) { + continue + } + e, err := s.Get(what, k) + if err != nil { + return Read{}, fmt.Errorf("reading %s's %s: %w", what, k, err) + } + if e == nil { + continue // deleted between the listing and the read + } + r.Value = e.Value + out.Rows = append(out.Rows, r) + } + sort.Slice(out.Rows, func(i, j int) bool { return out.Rows[i].Key < out.Rows[j].Key }) + out.Count = len(out.Rows) + out.Said = said(what, q, out.Count) + if len(unshaped) > 0 { + // Said, never dropped quietly: a key not in its bucket's shape is a client writing what it should not. + out.Said += fmt.Sprintf("; %d key(s) are not in the shape %s and are not listed: %s", len(unshaped), + shapes[what], strings.Join(unshaped, ", ")) + } + return out, nil +} + +func said(what string, q Query, n int) string { + var narrowed []string + for _, p := range [][2]string{{"node", q.Node}, {"module", q.Module}, {"collection", q.Collection}} { + if p[1] != "" { + narrowed = append(narrowed, p[0]+" "+p[1]) + } + } + s := fmt.Sprintf("%d in %s", n, what) + if len(narrowed) > 0 { + s += " for " + strings.Join(narrowed, ", ") + } + return s +} + +// queryOf reads a call's arguments strictly: an argument it does not know is refused, not ignored. +func queryOf(args map[string]any) (Query, error) { + var q Query + fields := map[string]*string{"what": &q.What, "node": &q.Node, "module": &q.Module, "collection": &q.Collection} + var unknown []string + for k, v := range args { + into, ok := fields[k] + if !ok { + unknown = append(unknown, k) + continue + } + if v == nil { + continue + } + s, ok := v.(string) + if !ok { + return Query{}, fmt.Errorf("%s is a string", k) + } + *into = strings.TrimSpace(s) + } + if len(unknown) > 0 { + sort.Strings(unknown) + return Query{}, fmt.Errorf("provisioning takes no %s; it takes what, node, module and collection", + strings.Join(unknown, ", ")) + } + return q, nil +} + +// fromKeyToken reads a key's token as the client writes it (mesh-provisioner-client collect.go): `=` is the +// character of that code. +func fromKeyToken(s string) (string, bool) { + var b strings.Builder + for i := 0; i < len(s); i++ { + if s[i] != '=' { + b.WriteByte(s[i]) + continue + } + if i+2 >= len(s) { + return "", false + } + v, err := strconv.ParseUint(s[i+1:i+3], 16, 8) + if err != nil { + return "", false + } + b.WriteByte(byte(v)) + i += 2 + } + return b.String(), true +} diff --git a/modules/mesh-provisioner/cmd/mesh-provisioner/registry.go b/modules/mesh-provisioner/cmd/mesh-provisioner/registry.go new file mode 100644 index 00000000..7d7d6f3e --- /dev/null +++ b/modules/mesh-provisioner/cmd/mesh-provisioner/registry.go @@ -0,0 +1,156 @@ +package main + +import ( + "bytes" + "encoding/json" + "fmt" + "slices" +) + +// The registry: which domain answers which kind of placeholder (novox/hq ADR 0349 clauses 1 and 3). A kind is the +// namespace of a placeholder, `${:…}`; its answer says which module owns the values and where a node's +// provisioner client finds them for its node. Compiled here and written to the bucket mesh-provisioner_registry, +// which this module alone writes and every client reads: a domain that comes to answer a kind is a change to this +// table, through the design that moves it (design 57). + +// The buckets this module keeps (its manifest's `state`), by local name. +const ( + Registry = "registry" + Needs = "needs" + Applied = "applied" + Collect = "collect" +) + +// Where a client finds a kind's values. +const ( + // FromBucket is a domain's bucket, watched for the node's keys: a watch delivers the current values first. + FromBucket = "bucket" + // FromAsk is a seat's verb asked on request and reply: a value never kept in a bucket (ADR 0201 §13). + FromAsk = "ask" +) + +// Answer is one kind's entry in the registry. +type Answer struct { + // Kind is the placeholder's namespace. + Kind string `json:"kind"` + // Domain is the module that owns the kind's values; for bound values, the module providing the provision + // the placeholder names, which is not one module but whichever the mesh resolved. + Domain string `json:"domain"` + // From is where a client finds the values: FromBucket or FromAsk. + From string `json:"from"` + // Bucket and Key, for a value in a bucket: the bucket, and the shape of a key in it, the node first. + Bucket string `json:"bucket,omitempty"` + Key string `json:"key,omitempty"` + // Ask, for a value asked for: the seat's verb, `.`. + Ask string `json:"ask,omitempty"` + // Why says the answer in a sentence, and Decision the record that decided it. + Why string `json:"why"` + Decision string `json:"decision"` +} + +// placement is the controller's: where a module runs, its directories, accesses, ports, seats and the machine's +// facts (ADR 0349 clause 6). On the bus, in the provisioner's plan of each module the controller handed to it, which +// its assignment carries (mesh-controller internal/inventory Provisioning, issue 671), or for a module only pulled in +// by another, its value of its own under the same key (issue 765): a client never reads the node's declaration, +// which the node-engine keeps for root alone. +func placement(kind, what string) Answer { + return Answer{Kind: kind, Domain: "mesh-controller", From: FromBucket, Bucket: "mesh-controller_assignments", + Key: ".", + Why: what + ": placement, the controller's, in the provisioner's plan its assignment carries (provisioning." + + "placement)", Decision: "novox/hq ADR 0349 clause 6"} +} + +// oldCollection is a namespace read as a collection of its own name (novox/hq ADR 0349 clause 13, design 57 L1). +func oldCollection(kind, what string) Answer { + return Answer{Kind: kind, Domain: "mesh-provisioner", From: FromBucket, Bucket: "mesh-provisioner_collect", + Key: "..", + Why: what + ": read as the collection `" + kind + ":`, its items recorded by the client of the node " + + "its contributors are placed on", Decision: "novox/hq ADR 0349 clause 13"} +} + +// TheRegistry is every kind a moved module's template may name, and its answer, in the order of ADR 0349 clause 2. +var TheRegistry = []Answer{ + {Kind: "config", Domain: "mesh-config", From: FromBucket, Bucket: "mesh-config_resolved", Key: "..", + Why: "a config value, resolved for each module on each node by mesh-config", Decision: "novox/hq ADR 0348"}, + {Kind: "secret", Domain: "mesh-vault", From: FromAsk, Ask: "mesh-vault.get", + Why: "a shared secret, handed by the vault sealed to the client's key, on request and reply only", + Decision: "novox/hq ADR 0349 clause 11"}, + // The provider's bucket and its keys, as the providers' helper writes them (mesh-sdk go/provider, issue 678): the + // bucket of the module the consumer's plan names as its provider, and the consumer's machine first. + {Kind: "bound", Domain: "provider", From: FromBucket, Bucket: "_bound", + Key: "..[.]", + Why: "a consumer's values, written by the module providing the provision to its own bucket, one entry per " + + "consumer, the bucket of the provider module the consumer's plan names", Decision: "novox/hq ADR 0349 clause 12"}, + // A provider's consumers' credentials, each consumer's pair credential, which the provider machine's client asks + // the vault for and fills into a file the provider alone reads (issue 679). + {Kind: "consumers", Domain: "mesh-vault", From: FromAsk, Ask: "mesh-vault.get", + Why: "each bound consumer's credential, asked of the vault by the provider machine's client and filled into a " + + "file only the provider reads, one entry per consumer", Decision: "novox/hq ADR 0373"}, + {Kind: "collected", Domain: "mesh-provisioner", From: FromBucket, Bucket: "mesh-provisioner_collect", + Key: "..", + Why: "the items of a collection, recorded by the client of the node its contributors are placed on", + Decision: "novox/hq ADR 0349 clause 13"}, + // The three namespaces the controller fills for a module not handed to the provisioner, read in a handed + // collector's template as the collection of the same name (design 57 L1): `${shell:zsh:normal}` is the + // collection `shell:zsh:normal`. + oldCollection("shell", "a shell's code in one slot, or a session file's or a power moment's"), + oldCollection("environment", "the account's environment in one format"), + oldCollection("contribution", "what a seat's holder receives of one kind"), + placement("dir", "a module's directory"), + placement("access", "a directory a module reaches of another's"), + placement("port", "a module's port"), + placement("seat", "where a seat's holder is reached"), + placement("machine", "a fact of the machine"), + placement("backup", "what the machine's modules back up: `lines` and `data`, as the controller derives them from each module's declared data"), +} + +// Publish brings the registry bucket to TheRegistry: a kind whose answer differs is written, a kind no longer in +// the table is deleted, and a pass that finds them equal writes nothing. It answers the kinds it wrote and +// deleted. +func Publish(s Store) (written, deleted []string, err error) { + known := map[string]bool{} + for _, a := range TheRegistry { + known[a.Kind] = true + want, err := json.Marshal(a) + if err != nil { + return written, deleted, err + } + have, err := s.Get(Registry, a.Kind) + if err != nil { + return written, deleted, fmt.Errorf("reading the registry's %s: %w", a.Kind, err) + } + if have != nil && sameJSON(have.Value, want) { + continue + } + if err := s.Put(Registry, a.Kind, a); err != nil { + return written, deleted, fmt.Errorf("writing the registry's %s: %w", a.Kind, err) + } + written = append(written, a.Kind) + } + keys, err := s.Keys(Registry) + if err != nil { + return written, deleted, fmt.Errorf("listing the registry: %w", err) + } + for _, k := range keys { + if known[k] { + continue + } + if err := s.Delete(Registry, k); err != nil { + return written, deleted, fmt.Errorf("deleting the registry's %s, a kind no domain answers: %w", k, err) + } + deleted = append(deleted, k) + } + slices.Sort(deleted) + return written, deleted, nil +} + +// sameJSON says two JSON texts hold the same value, whatever their spacing or the order of their keys. +func sameJSON(a, b []byte) bool { + var x, y any + if json.Unmarshal(a, &x) != nil || json.Unmarshal(b, &y) != nil { + return bytes.Equal(a, b) + } + ja, _ := json.Marshal(x) + jb, _ := json.Marshal(y) + return bytes.Equal(ja, jb) +} diff --git a/modules/mesh-provisioner/go.mod b/modules/mesh-provisioner/go.mod new file mode 100644 index 00000000..e89c5f04 --- /dev/null +++ b/modules/mesh-provisioner/go.mod @@ -0,0 +1,5 @@ +module mesh-provisioner + +go 1.25.0 + +require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261013235212-c8b07242a009 diff --git a/modules/mesh-provisioner/go.sum b/modules/mesh-provisioner/go.sum new file mode 100644 index 00000000..7e6c029b --- /dev/null +++ b/modules/mesh-provisioner/go.sum @@ -0,0 +1,2 @@ +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261013235212-c8b07242a009 h1:I9rm8HhYRJ0+mzQQz+VAH0kgv2YJWkvctEc67Yd0PX8= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261013235212-c8b07242a009/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY= diff --git a/modules/mesh-provisioner/module.json b/modules/mesh-provisioner/module.json new file mode 100644 index 00000000..11133aef --- /dev/null +++ b/modules/mesh-provisioner/module.json @@ -0,0 +1,55 @@ +{ + "module": "mesh-provisioner", + "version": "1", + "runs-as": "mesh-provisioner", + "claims": [ + { + "name": "mesh-provisioner", + "scope": "mesh", + "serves": [ + "provisioning" + ] + } + ], + "state": [ + "registry", + "needs", + "applied", + "collect" + ], + "own-secrets": { + "broker": "${dir:state}/broker" + }, + "secrets-owner": "mesh-provisioner", + "resources": [ + { + "id": "account", + "type": "user", + "name": "mesh-provisioner", + "shell": "/usr/bin/nologin", + "home": "/var/lib/mesh-provisioner" + }, + { + "id": "state", + "type": "directory", + "mode": "0700", + "owner": "mesh-provisioner", + "place": "." + } + ], + "build": { + "artifacts": [ + { + "name": "tools", + "kind": "bundle", + "language": "go", + "system": "arch", + "from": "cmd/mesh-provisioner", + "binary": "mesh-provisioner", + "loads": [ + "mesh-provisioner" + ] + } + ] + } +} diff --git a/modules/mesh-vault/README.md b/modules/mesh-vault/README.md new file mode 100644 index 00000000..d51bdd7a --- /dev/null +++ b/modules/mesh-vault/README.md @@ -0,0 +1,279 @@ +# mesh-vault + +Holds the `mesh-vault` seat: the mesh's shared secrets, each kept sealed, handed out only on request and +reply, sealed to whoever asked (novox/hq ADR 0085, ADR 0113, ADR 0349 §7 and §11). Go, one binary +(`cmd/mesh-vault`), launched by its node's runtime and speaking MCP to it through the Go SDK. + +## Where the secrets are kept, and why there + +In the vault's own store: one file per secret in `${dir:store}` on the vault's own disk, mode 0600 in a +0700 directory. Not a database, as most Go modules keep their data, because: + +- the vault is installed before any other module built on the runtime base (ADR 0113 §14), and a database + login is itself a shared secret the vault makes — the vault cannot need one to start; +- every value in the store is ciphertext anyway, so a database would add nothing a file does not; +- no secret is in a bucket, sealed or not (ADR 0201 §13), so the module declares no `state`. + +Each record holds the value twice, never in the clear: + +- **at rest**, sealed to the vault's own key (the sealed box of ADR 0206, `mesh-sdk/go/seal`). The key is + an X25519 pair the vault makes once in `${dir:state}/vault-key.json`; its private half never leaves it. + The key is made only for an empty store: with the key file missing and secrets in the store, the vault + does not start and says so, rather than make a key that opens none of them. +- **for the operator**, sealed to the operator key (ADR 0113 §22), so nothing exists only inside the + vault. A secret is refused, loudly, while the vault has no operator key. + +Every file is written whole, synced, renamed into place and its directory synced. + +### What the encryption at rest protects against, and what not + +The vault's private key lives in plaintext in `${dir:state}`, on the same disk as the store, readable by the +vault's account alone. So: + +- **It protects** against a copy of the store directory alone — a `store` backup, a file sent somewhere, + a listing of `${dir:store}` by a tool that reads files — holding anything usable, and against a value + being found by searching the disk for it. +- **It does not protect** against anyone who can read the vault's state directory too: root on the + vault's node, the vault's own account, or a backup that holds both `state` and `store` (both are + class `valuable`, so the node's backup holds both). Against those, at-rest encryption adds nothing; the + protection is the node's own security. Keeping the key elsewhere (another directory class, a key the + operator unlocks) is for V3's review, before real secrets move in. + +## Where the operator key comes from + +From the export the mesh writes to the vault's root directory (`${dir:root}/export.json`, the manifest's +`keeps`): its `operator-key` is the public half `operator key set` recorded in the controller. The vault +checks it every minute; a new key makes the vault seal every operator copy again to it — it can, because +it opens its own copy. Each record names the key its copy is sealed to, so a failure partway is finished on +the next minute's pass, and nothing is left sealed to an earlier key for good. + +The vault takes an operator key only as SPKI PEM (`SetOperatorKey`). A PEM private key is refused, and so +is a raw 32-byte key, because a raw private key looks the same. The one raw key it reads is the export's, +which the controller writes from what it recorded; the vault wraps it as SPKI before taking it. It keeps +the key raw in base64, the form of the operator's NaCl box. When the secrets move to the vault (design 57's +V3), the operator key moves with them. + +### How the operator opens a copy + +`mesh-vault.secret_vault_export` answers the vault's copies in the shape `secret export` writes. Saved to a +file, one is opened with the operator's private key by + + nox secret recover vault mesh-vault --key --from-export --out + +(`--out` is needed: a name holds `/`). `TestTheOperatorRecoversAVaultSecretFromItsExport` reads the file +as that command does and opens it as it does. `secret_export` still answers the controller's export, as +before; until V3 the controller's secrets are there. + +## The boxes and the key formats — settled + +Two boxes on the mesh today, both over X25519: + +| | format | keys | who uses it | +|---|---|---|---| +| NaCl anonymous box | XSalsa20-Poly1305, `box.SealAnonymous` | raw 32 bytes, base64 | the controller sealing into declarations; the node-engine's `sealing.key` (mesh-host `internal/identity/sealing.go`); the operator key and `secret recover --from-export` | +| the sealed box (ADR 0206) | X25519, HKDF-SHA256, AES-256-GCM, `{v, eph, iv, tag, ct}` | PEM: SPKI public, PKCS #8 private | the licence manager and `claude-code`; `mesh-sdk/go/seal` | + +What the vault uses: + +- **Its answers to `get` are the ADR 0206 box, for every asker, a node included.** The asker sends its + public key as SPKI PEM. A node's provisioner client (V4) asks with a key of its own, made when it starts + and kept in its memory alone: it runs under an account of its own and cannot read the node-engine's sealing + key, which is root's. A node's sealing key would open an answer too, being X25519 (wrapped as SPKI and PKCS #8, + `TestANodesSealingKeyOpensTheAnswer`), for the node-engine's own asks. A raw 32-byte key is refused: a raw + private key looks the same, and the vault takes no private key. +- **Its operator copies are the NaCl anonymous box**, to the operator key as it is made today, so the + operator's recovery opens them with the key and the command it already has. +- **The at-rest copy is the ADR 0206 box**, to the vault's own key. + +The NaCl box stays on the node-engine only for what the controller seals into declarations, until the last +module has moved (ADR 0349 §17). For the core's own credentials that travel in a declaration sealed by the +vault (ADR 0349 §16), the node-engine is to open the ADR 0206 box with the same sealing key; that is not built +yet (issue 647 leaves it to its own issue). + +## The seat's verb + +`mesh-vault.get` — `name`, `public_key` (SPKI PEM). It answers `{name, generation, changed-at, sealed}`, +`sealed` being the secret sealed to that key, and only to an asker allowed for that secret, the asker being +the bus user the runtime proved — never an argument. + +Every answer and every refusal is recorded in the vault's own store (`handed.jsonl`: who, which secret, +which generation, when, and why a refusal), synced before the answer is given — as ADR 0277 §25 records +asks; a secret whose hand-out cannot be recorded is not handed out. A refusal is recorded only for a caller +of a bus user's shape and a well-formed secret name, so an arbitrary argument fills nothing. The record is +bounded: rolled at 1 MiB into `handed.jsonl.1` … `.3`, the oldest dropped, each roll said loudly in the +journal and counted in `vault_status`. `secret_handed` reads it. + +`vault_status` says what needs a person: operator copies still sealed to an earlier key, an operator key in +the export the vault refused, and how often the record was rolled. + +**A failing re-seal is a condition** (ADR 0366, issue 738). The vault re-seals its operator copies to the +operator key once a minute. `vault_health` answers unhealthy once that has failed on three passes in a row, +saying what failed and why, and healthy again after one pass succeeds. The manifest declares it as the +health of the vault's own runtime (`runtime-health`), which the controller puts on `mesh-vault.own-runtime`; +the node-engine asks it every minute and the controller raises `module.mesh-vault..unhealthy` from +it, cleared on the first healthy answer. An operator key in the export the vault refuses fails the pass too. +How often the record was rolled still reaches a person through the journal and `vault_status` only. + +**Who may ask: a node's provisioner client, for what its plan names** (ADR 0349 §11, design 57 V4, issue 647; +`askers.go`). The caller is `provisioner.`, the client's own account (never the tool runner's, which an agent +on that machine shares), and the secret is `own///` for that same machine, where the +controller's assignment `.` (the assignments mirror, which the manifest `reads`) carries the +provisioner's plan and the plan's `secrets` name it; or a value a provider's software issued for one holder, +`provision////`, where that plan's `provisions` name the provision from that +provider with a credential's `name` (issue 757; an entry naming only the provider serving a provision, issue 760, +hands nothing). The vault cannot read a node's declaration, so the plan the +controller carries beside it is what the declaration names, as the vault reads it. A provision's one credential, +`provision//`, is handed to the machines the vault made it for (below) while the holder there +is handed to the provisioner: the provider's module, or a consumer whose plan still names it from that provider. A +module not handed to the provisioner, another machine's secret or holder value, and every other caller are refused, +and each refusal is recorded with why. A module's own account (issue 569) is not answered yet. + +**The controller, for the core's own credentials, sealed only to their machine** (ADR 0349 §16, issue 763; +`core.go`). The bus and mesh-provisioner need their files before any client runs, so their credentials travel in +the declaration sealed by the vault to their machine, which the controller carries and cannot open. The caller +`controller` is answered `own///` only for a module on the core's fixed list (`nats`, +`mesh-provisioner`, `mesh-provisioner-client`, the controller's list too), and only when the key it names is that +machine's sealing key as its own node-engine stated it in its own name, in `move-in`'s first step (`sealing_key`). +A machine whose engine stated none, any other key, any other module and any other name are refused and recorded. + +## Taking values in: `accept`, `secret-ask`, `issued`, `rotate` + +Design 57's V2 (issue 675; ADR 0349 clauses 8 to 10, ADR 0277 §22–§27 and ADR 0305 with the vault in the +controller's place). `intake.go`. + +**From the controller alone; the vault asks the desk and the provider itself.** The controller alone asks these +verbs, as the runtime proves its caller. Each value comes in sealed to an X25519 key the vault made for that one +call (raw, base64 — the NaCl anonymous box the desk's prompt and a provider's tool already seal to), and opens with +the private half, which lives in this process only and is forgotten once used or after two minutes. A value sealed +to any other key is refused, and a call is answered once. + +**The vault's own account** (issue 723). The vault runs as `mesh-vault` in a runtime of its own (`runs-as`, ADR +0259 §8), on a bus account of its own (`.mesh-vault`, from its own secret `broker`). The bus lets that +account call the desks' prompt (`node-launcher.secret` on any machine) and each issuing provider's tool in its own +name, and nothing else outside its seat (mesh-controller's `internal/broker` grants, proven on a real server). So +the prompt and the tool know the vault asks, and the controller carries neither the key nor the sealed value. The +controller still asks them itself for a module filled the old way, until R1 (issue 682) removes that path; they +answer both until then. Everything the vault keeps is its account's: its directories, the export it keeps, the +grant files the controller still writes for it. + +- `accept` — two steps: node, module, name (a module's own secret) or provider, provision (a provision's one + credential), and `by`; then `call` and `sealed`. The controller's terminal seals the value it read to the call's + key before it leaves the terminal. Kept with origin `accepted`; an accepted provision credential is never remade + (`rotate` refuses it); a value a provider's software issued is not taken by `accept`. +- `secret-ask` — one step: node, module, name, desk and `asked_by` (who asked the controller, as the bus named its + caller). The ask is recorded in `asks.jsonl` before anything else, and refused while another ask for the same + secret is open (two minutes at most) or when three were made in the last hour. The vault then asks the desk's + prompt itself, naming who asked as the record has it, with `timeout_seconds` 20. The answer is how it ended + (`given`, dismissed, timed-out, empty, refused, failed) and who asked; how each ask ended is recorded. + `secret_asks` reads the record. The hand-act log stays the controller's: it records the act once the vault has + taken the value. +- `issued` — one step: provision, provider (machine), module and tool. The vault asks that tool on the provider's + machine itself; the value is kept as the provision's one credential with origin `issued`. The same value again + writes nothing, says nothing and keeps the generation (ADR 0305 §5). +- `issued` for each holder apart (issue 749; ADR 0349 clause 11, ADR 0365 clause 4: the forge's read token per agent + account). The controller names the provision's holders from that provider (`holders`, each a node and a consuming + module), and the vault asks the tool with them. The tool answers either one value (`sealed`, kept as above) or one + per holder (`holders`: node, module and `sealed` each, all sealed to the call's key). Each is kept as + `provision////`, origin `issued`, and the same value again for + a holder writes nothing for it. The answer is taken whole or not at all: a holder not asked about, one named twice, a + value not sealed to the call or an empty one keeps nothing. A holder asked about and not answered is said + (`not_answered`) and keeps nothing. **`get` hands a holder's value to that holder alone**, its machine's provisioner + client (`provisioner.`) or the module under an account of its own (`.`), whatever the rule for + who may ask allows; every other caller is refused and the refusal recorded. + `TestOnARealBusTwoHoldersOnTwoMachinesEachGetOnlyTheirOwnIssuedValueAndNoneCrossesInTheClear` proves it on a NATS + server with a stand-in forge. +- `rotate` — a module's own secret or a provision's credential, made anew. Refused for an `issued` value (the + software rotates it, ADR 0305 §9) and an accepted provision credential (clause 8). From the controller alone, + which knows whether the module's secrets are filled from the vault yet, and with a `why`, said in the vault's + log. Who asked is the `by` the controller names, beside the controller; both are kept (`changed-by`, + `changed-via`). The controller's `rotate ` forwards here for a provider filled the new way (issue + 767): a shared value is made anew for every holder, never for one consumer alone; for a provider whose consumers + keep a login each, `consumer` and `consumer_module` (`--consumer`, `--module`) name the one consumer whose own + login is remade, and no other is touched. + +Names in the store: `own///`, `provision//` and, for a value issued for +one holder, `provision////`. V3 copies the +controller's rows in under these names. + +**Which modules reach the vault this way.** The controller sends a module's verbs here only when its secrets are +filled the new way (its machine's provisioner client asks the vault for them, V4); every other module keeps the +controller's path, so no value lands where its machine never looks (ADR 0349 clause 17; the release lead's decision +on issue 675). + +**What the issuing provider's test is here.** `TestAnIssuingProviderAnswersOnlyTheVaultsCallSealedToTheVault` and +`TestTheVaultAsksTheIssuingToolItselfAndOnlyASealedValueCrosses` use a stand-in provider that answers the vault's +account (and, until R1, the controller) and no other caller, and only sealed to the call's key. plex's own tool +answers the controller alone today; it must also answer `.mesh-vault` when it moves with the media catalogue, +after release 57. + +## Events + +- `secret-changed` — `name`, `by`, `at`, `generation`: which secret, who changed it, when. Never a value + or a sealed copy. +- `provisioned`, `rotated` — the custody ledger's, by fingerprint, as before. +- `provisioner.retirement` (ADR 0230 §35) — `waiting`, `settled`, `approved`, `rejected`, `retired`, + `reenabled`, `deleted`, naming holders, never a value. + +## The tools + +`secret_export`, `secret_holders`, `secret_verify` answer as the TypeScript vault's did, and its custody +ledger (`${dir:ledger}`, the same JSON files) is kept by a pass over the contributions file every minute. +`secret_handed` and `secret_vault_export` read the vault's own store. None returns a value. + +**The ledger follows ADR 0230.** A contributions file that does not say who is asked for (unreadable, not +JSON, no `given` list or a null one, or for another requirement) is not "nobody": the pass records nothing, +retires nobody and starts its count again. A holder no longer asked for is retired — a mark with when and +why in its own ledger file, its history kept — only after the same set has gone unasked in five passes and +ten minutes; more than three at once, or more than half of those held, waits for a person +(`provisioner_retire_approve`, `provisioner_retire_reject`); asked for again it is re-enabled; it is deleted +only by `provisioner_delete`, a person's act with a why. `provisioner_retirement` shows it all. The vault is +mark-only: it holds no access to switch off. + +The words are the controller's: the `provisioner.retirement` event is mesh-controller's `link.Retirement` +(`provider-node`, `consumers` as objects) and the `provisioner_retirement` answer its `link.RetirementState` +(`held`, `bound`, `retires_by: mark-only`), so `retire approve`, `retire reject` and `cleanup list` read the +vault and a set waiting for a person raises the controller's urgent condition. The acts take what the SDK's +harness takes: approve and reject repeat the set in `consumers` and refuse any other set; delete repeats the +name in `confirm`; each records the `by` and `via` the controller sends. `testdata/retirement-*.json` are +those types marshalled by the controller (feat/57 at dc36536), and the tests hold the vault's words to them. + +What the TypeScript vault did and this one does not: it consumed its own events only to print an audit line, +which the pass now prints itself. Its retirement ran on the SDK's harness, which this one follows: mark-only, +and a holder's ledger file removed only through a person's delete, as here. The difference is where the mark +lives: the harness kept it in its process, so a restart forgot it (ADR 0230 §13); here it is in the ledger +file. + +**The credential a provider shares with its consumers** (ADR 0349 clause 8, ADR 0158 §4–§6, design 57 V5, issue +677; `pair.go`). One value per provider assignment and provision, `provision//`. Once a minute +the vault reads every plan in the assignments mirror; each plan's `provisions` names the provider serving each +provision its module requires (issue 760). A consumer binds when an entry appears and unbinds when it goes or its +assignment goes; on either, the vault makes one new value for every holder together and says `secret-changed`, and +each holder's client asks again. What the move brought in (V3) is adopted when every pair row of a provider and +provision shares one generation stamp and one value (ADR 0158's shared form), with its origin, and nothing is +remade. Rows without a stamp (ADR 0048, a login per consumer) are never made one value (ADR 0158 §11): `rotate` of +the provision remakes each consumer's own apart, and each consumer's machine is handed only its own row, +`pair////[/]` (the provider's machine all of them). A provider the +vault holds no row of gets its first value in the form each plan entry names (`credential-form`, from the provider's +offer, issue 768): `shared`, one value for every holder; `pair`, a login of its own for each consumer bound; `issued` +or `given` are never made. No form named, plans that disagree, or a form it does not know: nothing is made. An accepted or issued value is never remade, only +its holders recorded. A provision whose plans name a credential per holder (issue 757) has no shared value. **No +value is made that a holder would never get:** while the provider's assignment, or a consumer the move brought in, +is still filled the old way (no plan), every make, remake and rotation is held back and the reason said in the +log. Who holds each value is kept in the store's `shared/` directory, never a value. + +## What comes later + +Taking values in is built (above); the controller's secrets move in with V3; nodes' provisioner clients ask with V4, +for each module handed to the provisioner; providers answer their consumers with B1–B3. + +## Tests + + go test ./... + +`vault_test.go` is ADR 0349's vault test list as far as this package goes: no bucket holds a secret; no +private key is accepted; every secret has an operator-sealed copy; `get` answers only an allowed asker, +sealed to its key; `secret-changed` carries no value. `intake_test.go` is ADR 0349's `secret-ask` test as far as +the vault answers it (recorded before the prompt, one open ask per secret and three an hour, the asker named as +the bus named it, the typed value opened only here), the issuing provider's test with a stand-in provider (ADR 0305 +§13's checks that belong to the vault: nothing rewritten for a value held, `rotate` refused, no other way in), and +accept and rotate. `followups_test.go` holds V1's review follow-ups. diff --git a/modules/mesh-vault/client.ts b/modules/mesh-vault/client.ts deleted file mode 100644 index c322db20..00000000 --- a/modules/mesh-vault/client.ts +++ /dev/null @@ -1,172 +0,0 @@ -// mesh-vault's ledger — vault's own code, living in the module (novox/hq ADR 0039). The provisioner and -// the tools both import it, and nothing outside vault does. -// -// **The vault holds no value.** A `secret` is an ordinary pair credential: the controller mints it, -// seals it to the consumer's node and to this one, and the host unseals this node's copy into the -// file the contribution names (ADR 0048). That file is already on this machine, readable by nothing -// but the vault's runtime, and it is the only copy the vault ever sees. Writing a second copy — -// plain, or sealed to a key the vault keeps — would put back exactly the single place that can open -// everything, which is what sealing to the machine was built to remove (ADR 0085's open question is -// how to recover WITHOUT that; the answer is not "keep one anyway"). -// -// So what the vault keeps is what makes a secret *owned* rather than merely delivered: who holds -// one, since when, its fingerprint, and every time it changed. Enough to say "this holder's value is -// the one the mesh last delivered" and "it has been rotated twice, last on Tuesday" — and never -// enough to say what it is. The fingerprint is the only thing a tool may take or return, which is -// the rule the source mesh's secret tools were built on: the secret is never an argument. - -import { createHash } from "node:crypto"; -import { mkdirSync, readdirSync, readFileSync, renameSync, unlinkSync, writeFileSync } from "node:fs"; -import { join } from "node:path"; - -/** One holder of a secret this vault provides — everything the vault knows, and no value. */ -export interface Held { - /** The login the mesh derived for the consumer — `-`, so it names the holder. */ - readonly as: string; - /** The consumer's node. */ - readonly consumer: string; - /** sha256 of the value the mesh last delivered, `sha256:`. Compared, never inverted. */ - readonly fingerprint: string; - /** Length of the value, so a holder can tell a truncated file from a wrong one. */ - readonly length: number; - /** When this holder was first granted a secret. */ - readonly since: string; - /** When the value last changed — equal to `since` until the first rotation. */ - readonly changed: string; - /** How many times the value has changed since `since`. */ - readonly rotations: number; - /** Every earlier fingerprint, oldest first: the audit trail a rotation leaves. */ - readonly history: readonly { readonly fingerprint: string; readonly until: string }[]; -} - -/** What recording a delivery found: a new holder, a changed value, or nothing new. */ -export type Outcome = "granted" | "rotated" | "unchanged"; - -/** sha256 of a value, as `sha256:`. The one thing about a secret that may be spoken. */ -export function fingerprint(value: string): string { - return "sha256:" + createHash("sha256").update(value, "utf8").digest("hex"); -} - -export class Ledger { - private readonly dir: string; - - constructor(dir: string) { - this.dir = dir; - mkdirSync(dir, { recursive: true, mode: 0o700 }); - } - - /** Build from the module's resolved environment: $MESH_VAULT_LEDGER is where holders are kept. */ - static fromEnv(env: NodeJS.ProcessEnv = process.env): Ledger { - const dir = env.MESH_VAULT_LEDGER; - if (!dir) { - throw new Error("MESH_VAULT_LEDGER is not set — the vault has nowhere to keep its ledger"); - } - return new Ledger(dir); - } - - /** - * Record that the mesh delivered `value` for `as`. Idempotent: the same value again changes - * nothing, a different value is a rotation and is remembered as one. The value is fingerprinted - * here and goes no further. - */ - record(as: string, consumer: string, value: string, now = new Date()): { held: Held; outcome: Outcome } { - const fp = fingerprint(value); - const at = now.toISOString(); - const before = this.get(as); - if (!before) { - const held: Held = { - as, consumer, fingerprint: fp, length: value.length, - since: at, changed: at, rotations: 0, history: [], - }; - this.write(held); - return { held, outcome: "granted" }; - } - if (before.fingerprint === fp && before.length === value.length) { - return { held: before, outcome: "unchanged" }; - } - const held: Held = { - ...before, consumer, fingerprint: fp, length: value.length, changed: at, - rotations: before.rotations + 1, - history: [...before.history, { fingerprint: before.fingerprint, until: at }], - }; - this.write(held); - return { held, outcome: "rotated" }; - } - - /** Forget a holder the mesh withdrew. Returns whether there was one to forget. */ - withdraw(as: string): boolean { - try { - unlinkSync(this.pathOf(as)); - return true; - } catch { - return false; - } - } - - get(as: string): Held | undefined { - try { - return JSON.parse(readFileSync(this.pathOf(as), "utf8")) as Held; - } catch { - return undefined; - } - } - - /** Every holder, by login. */ - list(): Held[] { - let names: string[]; - try { - names = readdirSync(this.dir); - } catch { - return []; - } - return names - .filter((n) => n.endsWith(".json")) - .map((n) => this.get(n.slice(0, -".json".length))) - .filter((h): h is Held => h !== undefined) - .sort((a, b) => a.as.localeCompare(b.as)); - } - - private pathOf(as: string): string { - if (!/^[a-z0-9][a-z0-9_.-]*$/.test(as)) { - throw new Error(`a login is a name, not a path: ${JSON.stringify(as)}`); - } - return join(this.dir, `${as}.json`); - } - - /** Written whole and renamed into place, so a reader never sees half a record. */ - private write(held: Held): void { - const final = this.pathOf(held.as); - const tmp = `${final}.${process.pid}.tmp`; - writeFileSync(tmp, JSON.stringify(held, null, 2) + "\n", { mode: 0o600 }); - renameSync(tmp, final); - } -} - -/** One entry of the mesh's contributions file, as the vault reads it for its tools. */ -export interface Contribution { - readonly from?: string; - readonly node?: string; - readonly as: string; - readonly secret: string; -} - -/** The consumers the mesh currently asks this vault to serve — the `receives` file, read plainly. */ -export function contributions(receives: string): Contribution[] { - let doc: { given?: Contribution[] }; - try { - doc = JSON.parse(readFileSync(receives, "utf8")) as { given?: Contribution[] }; - } catch { - return []; - } - return (doc.given ?? []).filter((g) => g.as && g.secret); -} - -/** Fingerprint of the value the host currently holds for one contribution, or why it could not. */ -export function deliveredFingerprint(c: Contribution): { fingerprint: string; length: number } | { error: string } { - try { - const value = readFileSync(c.secret, "utf8").replace(/\n$/, ""); - return { fingerprint: fingerprint(value), length: value.length }; - } catch (err) { - return { error: `the delivered secret is not readable: ${err}` }; - } -} diff --git a/modules/mesh-vault/cmd/mesh-vault/askers.go b/modules/mesh-vault/cmd/mesh-vault/askers.go new file mode 100644 index 00000000..2e385c3a --- /dev/null +++ b/modules/mesh-vault/cmd/mesh-vault/askers.go @@ -0,0 +1,175 @@ +package main + +import ( + "encoding/json" + "fmt" + "strings" +) + +// Who may ask `get` for which secret (novox/hq ADR 0349 §11, design 57 V4, issue 647). +// +// The rule is: a node's provisioner client, running as a service under a bus account of its own +// (`provisioner.`, design 57 P3), for a secret that node's declaration names; or a module, for a secret its +// manifest declares, only once its code runs under a bus account of its own (issue 569). The second does not exist +// yet, so the vault answers the first alone (Declared), and refuses every other caller. +// +// **What a node's declaration names, as the vault can read it.** The vault cannot read a node's declaration, which +// the node-engine keeps for root. The controller carries the provisioner's plan of each module it handed to the +// provisioner in that module's assignment, `.` of the assignments mirror (mesh-controller internal/ +// inventory Provisioning), the same value the node's client fills from; its `secrets` are the module's own secrets +// the declaration names for its files there. So a client is handed `own///` only when the +// asker is that node's client, the module is on that node and handed to the provisioner, and its plan names the +// secret. A module only pulled in by another has a value there of its own, which names what requires it (issue 765), +// and is read the same way. Everything else is refused, and the refusal recorded with why (vault.go). +// +// **A value a provider's software issued for one holder** (issue 757): the vault keeps it as +// `provision////` (perholder.go, issue 749). A node's client is handed it when +// the asker is that node's client (perholder.go's rule, before this one) and the module's plan there names that +// provision from that provider among its `provisions`: the credentials of what it requires that its machine's +// client fills from the vault. +// +// **The provision's one credential** (issue 677): `provision//` is handed to a node's client +// when the vault made, adopted or recorded it for a holder on that node (pair.go) and that holder is still there, +// handed to the provisioner: the provider's module on its machine, or a consumer whose plan there names the +// provision from that provider (without a name, which is a credential per holder). +// +// **A consumer's own login** (issue 677's note 12, ADR 0048, ADR 0158 §11): `pair//// +// [/]` is handed to that consumer's machine while its plan there names the provision from that +// provider under that local name, and to the provider's machine while the provider is handed there; never to +// another consumer. + +// Askers decides whether caller (the bus user the runtime proved) may have the secret name, and if not, +// why not. +type Askers interface { + May(caller, name string) (bool, string) +} + +// NoAsker allows nobody: what a vault with no rule given answers. +type NoAsker struct{} + +// May refuses every caller. +func (NoAsker) May(string, string) (bool, string) { + return false, "the vault hands a secret to no asker without a rule of who may ask (novox/hq ADR 0349 §11)" +} + +// AllowList is a stand-in rule: each caller, and the names it may have. +type AllowList map[string][]string + +// May allows a caller only the names listed for it. +func (l AllowList) May(caller, name string) (bool, string) { + for _, n := range l[caller] { + if n == name { + return true, "" + } + } + return false, "not on the list of who may ask for it" +} + +// ProvisionerPrefix begins a node's provisioner client's bus user, `provisioner.` (mesh-controller +// internal/broker, KindProvisioner). +const ProvisionerPrefix = "provisioner." + +// AssignmentsRead is the controller's assignments mirror as this module reads it (its manifest's `reads`). +const AssignmentsRead = "mesh-controller.assignments" + +// AssignmentReader reads one value of the assignments mirror, `.`: nil when there is none. +type AssignmentReader func(key string) ([]byte, error) + +// Declared is the rule the running vault keeps: a node's provisioner client, for a secret of a module on its node +// that the module's plan there names. +type Declared struct { + Read AssignmentReader + // Store is the vault's store, whose record of who holds a provision's one credential (pair.go) the rule reads. + Store *Store +} + +// plannedSecrets is as much of an assignment's value as the rule reads: the plan's secrets, by name. +type plannedSecrets struct { + Node string `json:"node"` + Module string `json:"module"` + Provisioning *struct { + Secrets []struct { + Name string `json:"name"` + } `json:"secrets"` + // Provisions are what the module requires, each with its provider (issue 760); one with a name is a + // credential the machine's client fills from the vault (issue 757). + Provisions []struct { + Name string `json:"name"` + Provision string `json:"provision"` + Provider string `json:"provider"` + } `json:"provisions"` + } `json:"provisioning"` +} + +// May allows caller the secret name when caller is `provisioner.` and name is `own///` +// that the plan of module on node names, or `provision////` whose provision from +// that provider the plan names (issue 757). +func (d Declared) May(caller, name string) (bool, string) { + node, ok := strings.CutPrefix(caller, ProvisionerPrefix) + if !ok || !namePart.MatchString(node) { + return false, "the vault hands a secret only to a node's provisioner client under its own account " + + "(provisioner.); a module's code is answered once it runs under an account of its own (issue 569)" + } + parts := strings.Split(name, "/") + if len(parts) == 3 && parts[0] == "provision" { + if why := sharedRefusal(d.Store, d.Read, node, parts[1], parts[2]); why != "" { + return false, why + } + return true, "" + } + if (len(parts) == 5 || len(parts) == 6) && parts[0] == "pair" { + if why := pairRowRefusal(d.Read, node, parts); why != "" { + return false, why + } + return true, "" + } + var holderNode, module, secret, provider string + switch { + case len(parts) == 4 && parts[0] == "own": + holderNode, module, secret = parts[1], parts[2], parts[3] + case len(parts) == 5 && parts[0] == "provision": + provider, secret, holderNode, module = parts[1], parts[2], parts[3], parts[4] + default: + return false, fmt.Sprintf("a node's provisioner client is handed a module's own secret, own/%s//, "+ + "the value a provider issued for one of its modules, provision///%s/, or "+ + "a provision's one credential, provision//; %s is none of them", node, node, name) + } + if holderNode != node { + return false, fmt.Sprintf("%s is a secret of machine %s, and %s asks for machine %s alone", name, holderNode, + caller, node) + } + if d.Read == nil { + return false, "the vault cannot read the assignments, so it cannot tell what " + node + "'s declaration names" + } + raw, err := d.Read(node + "." + module) + if err != nil { + return false, fmt.Sprintf("the assignment %s.%s could not be read, so what it names is not known: %v", node, + module, err) + } + if raw == nil { + return false, fmt.Sprintf("%s is not on %s: neither assigned there nor required by a module there", module, + node) + } + var a plannedSecrets + if err := json.Unmarshal(raw, &a); err != nil || a.Node != node || a.Module != module { + return false, fmt.Sprintf("the assignment %s.%s is not the controller's", node, module) + } + if a.Provisioning == nil { + return false, fmt.Sprintf("%s on %s is not handed to the provisioner, so its secrets reach it in its "+ + "declaration (novox/hq ADR 0349 §17)", module, node) + } + if provider != "" { + for _, p := range a.Provisioning.Provisions { + if p.Name != "" && p.Provision == secret && p.Provider == provider { + return true, "" + } + } + return false, fmt.Sprintf("%s's plan on %s names no credential of %s from %s", module, node, secret, provider) + } + for _, s := range a.Provisioning.Secrets { + if s.Name == secret { + return true, "" + } + } + return false, fmt.Sprintf("%s's plan on %s names no secret %s", module, node, secret) +} diff --git a/modules/mesh-vault/cmd/mesh-vault/core.go b/modules/mesh-vault/cmd/mesh-vault/core.go new file mode 100644 index 00000000..985fe26d --- /dev/null +++ b/modules/mesh-vault/cmd/mesh-vault/core.go @@ -0,0 +1,142 @@ +package main + +// The core's own credentials (novox/hq ADR 0349 §16, issue 763). The bus, the node-engine, the controller and +// mesh-provisioner need their files before any provisioner client runs, so their credentials travel in the +// declaration as values the vault sealed to the node they run on, which the controller carries and cannot open. +// +// **The controller is answered for a core module's own secret alone, and only sealed to that node's key.** The +// controller asks `get` for own/// with the key to seal to; the vault seals only when that +// key is the sealing key the node's own engine stated in its own name (`node.`, at the move: move.go). The +// controller names the key, so that a key it knows to be stale is refused loudly rather than answered with a box +// nobody opens; but it can never choose one, so a value it asks for is never sealed to a key it holds. +// +// **Where the node's key comes from.** Only from the node-engine of that machine, never from the controller's +// record of it, which is the asker's own word. A machine whose engine stated none is answered nothing. + +import ( + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "slices" + "strings" + "time" +) + +// CoreModules are the core's modules whose own secrets the controller carries sealed by the vault: the same fixed +// list as the controller's (ADR 0349 §16). The node-engine and the controller have no own secrets of their own here. +var CoreModules = []string{"mesh-provisioner", "mesh-provisioner-client", "nats"} + +// nodeKeyFile is a machine's sealing key as its engine stated it. +type nodeKeyFile struct { + Node string `json:"node"` + Key string `json:"key"` + By string `json:"by"` + Since time.Time `json:"since"` +} + +func (s *Store) nodeKeyPath(node string) string { + return filepath.Join(s.dir, "nodes", node+".json") +} + +// NodeKey is the sealing key node's engine stated (a raw X25519 public key in base64), and whether there is one. +func (s *Store) NodeKey(node string) (string, bool, error) { + if !namePart.MatchString(node) { + return "", false, fmt.Errorf("%q is not a machine's name", node) + } + raw, err := os.ReadFile(s.nodeKeyPath(node)) + if errors.Is(err, os.ErrNotExist) { + return "", false, nil + } + if err != nil { + return "", false, err + } + var f nodeKeyFile + if err := json.Unmarshal(raw, &f); err != nil || f.Node != node || f.Key == "" { + return "", false, fmt.Errorf("the record of %s's sealing key is not readable", node) + } + return f.Key, true, nil +} + +func (s *Store) setNodeKey(node, key, by string, at time.Time) error { + if err := os.MkdirAll(filepath.Join(s.dir, "nodes"), 0o700); err != nil { + return err + } + raw, _ := json.MarshalIndent(nodeKeyFile{Node: node, Key: key, By: by, Since: at}, "", " ") + return writeWhole(s.nodeKeyPath(node), append(raw, '\n')) +} + +// sealingKey checks a sealing key a node-engine states: one raw X25519 public key in base64, never a PEM, which a +// private key could be. +func sealingKey(key string) (string, error) { + key = strings.TrimSpace(key) + if privateKeyIn(key) || strings.Contains(key, "-----") { + return "", errors.New("a machine's sealing key is stated as its raw X25519 public key in base64, never a PEM " + + "and never a private key; nothing was taken") + } + raw, err := base64.StdEncoding.DecodeString(key) + if err != nil || len(raw) != 32 { + return "", errors.New("the sealing key stated is not a raw X25519 public key in base64; nothing was taken") + } + return key, nil +} + +// recordNodeKey keeps the sealing key node's engine stated, saying so when it replaces another. +func (v *Vault) recordNodeKey(node, key, caller string) error { + held, had, err := v.Store.NodeKey(node) + if err != nil { + return err + } + if had && held == key { + return nil + } + if err := v.Store.setNodeKey(node, key, caller, v.Now().UTC()); err != nil { + return fmt.Errorf("%s's sealing key could not be recorded: %w", node, err) + } + if had { + v.log("%s's sealing key is now %s, stated by %s (it was %s): the core's credentials are sealed to it from now on", + node, key, caller, held) + } else { + v.log("%s's sealing key is %s, stated by %s", node, key, caller) + } + return nil +} + +// coreRefusal says why the controller may not have name sealed to key, or "" when it may: name is a core module's +// own secret and key is its machine's sealing key as that machine's engine stated it. +func (v *Vault) coreRefusal(name, key string) string { + parts := strings.Split(name, "/") + if len(parts) != 4 || parts[0] != "own" { + return fmt.Sprintf("the controller is handed a core module's own secret alone, own//<%s>/, sealed "+ + "to its machine (ADR 0349 §16); %s is not one", strings.Join(CoreModules, "|"), name) + } + node, module := parts[1], parts[2] + if !slices.Contains(CoreModules, module) { + return fmt.Sprintf("%s is not one of the core's modules (%s): its secrets reach its machine through the "+ + "provisioner or its declaration, never through the controller's ask (ADR 0349 §16)", module, + strings.Join(CoreModules, ", ")) + } + stated, held, err := v.Store.NodeKey(node) + if err != nil { + return fmt.Sprintf("%s's sealing key cannot be read: %v", node, err) + } + if !held { + return fmt.Sprintf("%s's node-engine has stated no sealing key to the vault (it does so at the move), so "+ + "there is no key of that machine's own to seal %s to", node, name) + } + want, err := publicPEM(stated) + if err != nil { + return fmt.Sprintf("%s's sealing key as recorded is not usable: %v", node, err) + } + named, err := askerKey(key) + if err != nil { + return err.Error() + } + if named != want { + return fmt.Sprintf("%s is sealed only to %s's sealing key as its node-engine stated it, and the key named "+ + "is another", name, node) + } + return "" +} diff --git a/modules/mesh-vault/cmd/mesh-vault/core_test.go b/modules/mesh-vault/cmd/mesh-vault/core_test.go new file mode 100644 index 00000000..0e98843e --- /dev/null +++ b/modules/mesh-vault/cmd/mesh-vault/core_test.go @@ -0,0 +1,278 @@ +package main + +// Issue 763 (ADR 0349 §16, the lead's decision on issue 755): the core's own credentials travel in the declaration +// sealed by the vault to their node, which the controller carries and cannot open. The vault answers the controller +// for own/// alone, sealed only to the sealing key that node's engine stated in its own name +// at the move; any other module, any other key and any other caller are refused. + +import ( + "crypto/ecdh" + "crypto/rand" + "crypto/x509" + "encoding/base64" + "encoding/json" + "encoding/pem" + "os" + "path/filepath" + "reflect" + "strings" + "sync" + "testing" + "time" + + "git.novox.be/novox/mesh-sdk/go/seal" + "github.com/nats-io/nats-server/v2/server" + "github.com/nats-io/nats.go" +) + +// aSealingKey is a machine's sealing key as its node-engine makes it: the public half raw in base64, the form it +// states, and the private half as the PKCS #8 PEM the ADR 0206 box opens with. +func aSealingKey(t *testing.T) (public, privatePEM string) { + t.Helper() + k, err := ecdh.X25519().GenerateKey(rand.Reader) + if err != nil { + t.Fatal(err) + } + der, err := x509.MarshalPKCS8PrivateKey(k) + if err != nil { + t.Fatal(err) + } + return base64.StdEncoding.EncodeToString(k.PublicKey().Bytes()), + string(pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: der})) +} + +func pemOf(t *testing.T, raw string) string { + t.Helper() + p, err := publicPEM(raw) + if err != nil { + t.Fatal(err) + } + return p +} + +const brokerValue = "pv-anchors-provisioner-broker-password-2025" + +func clientsBroker(sealingKey string) MoveOf { + return MoveOf{Node: "anchor", Module: "mesh-provisioner-client", Name: "provisioner-broker", Origin: "made", + Generation: "g-1", SealingKey: sealingKey} +} + +// The list is the controller's (mesh-controller's core list, ADR 0349 §16): named here so a change to either is seen. +func TestTheCoreIsTheBusTheProvisionerAndItsClient(t *testing.T) { + if want := []string{"mesh-provisioner", "mesh-provisioner-client", "nats"}; !reflect.DeepEqual(CoreModules, want) { + t.Fatalf("the core's modules are %v, want %v", CoreModules, want) + } +} + +// A node-engine states its machine's sealing key in its own name at the move, and only its own machine's. +func TestAMoveRecordsTheSealingKeyItsNodeEngineStates(t *testing.T) { + v := newVault(t, nil) + pub, _ := aSealingKey(t) + if _, err := moveIn(t, v, "node.anchor", clientsBroker(pub), brokerValue); err != nil { + t.Fatal(err) + } + if got, held, err := v.Store.NodeKey("anchor"); err != nil || !held || got != pub { + t.Fatalf("anchor's key is %q, %v, %v", got, held, err) + } + if _, held, _ := v.Store.NodeKey("laptop"); held { + t.Error("a key was recorded for a machine that stated none") + } + // A PEM, which a private key could be, is never taken as a sealing key; nor anything not 32 bytes. + _, private := aSealingKey(t) + for _, bad := range []string{private, pemOf(t, pub), "c2hvcnQ="} { + if _, err := v.MoveOpen("node.anchor", clientsBroker(bad)); err == nil { + t.Errorf("the move took %q as a sealing key", bad[:8]) + } + } + if got, _, _ := v.Store.NodeKey("anchor"); got != pub { + t.Error("a refused key replaced the one stated") + } +} + +// coreOnABus is a vault holding anchor's core credential and a leaf module's secret, both moved in by anchor's +// node-engine, which stated anchor's sealing key; the controller asks it on a real bus. Answers the controller's +// ask (the box as answered), and every byte that crossed. +func coreOnABus(t *testing.T) (ask func(name, key string) (Answer, error), anchorPub, anchorPriv string, + heard func() string) { + t.Helper() + const conf = `listen: 127.0.0.1:-1 +authorization { + users = [ + { user: "novox.mesh-vault", password: "pw", permissions: { + publish: { deny: [">"] } + subscribe: { allow: ["mesh.seat.mesh-vault.call.get.*"] } + allow_responses: true } } + { user: "controller", password: "pw", permissions: { + publish: { allow: ["mesh.seat.mesh-vault.call.get.controller"] } + subscribe: { allow: ["_INBOX.controller.>"] } } } + { user: "tap", password: "pw", permissions: { publish: { deny: [">"] }, subscribe: { allow: [">"] } } } + ] +} +` + file := filepath.Join(t.TempDir(), "bus.conf") + if err := os.WriteFile(file, []byte(conf), 0o600); err != nil { + t.Fatal(err) + } + opts, err := server.ProcessConfigFile(file) + if err != nil { + t.Fatal(err) + } + opts.NoLog, opts.NoSigs = true, true + s, err := server.NewServer(opts) + if err != nil { + t.Fatal(err) + } + go s.Start() + if !s.ReadyForConnections(10 * time.Second) { + t.Fatal("the bus did not come up") + } + t.Cleanup(s.Shutdown) + url := s.ClientURL() + + var mu sync.Mutex + var crossed []string + tap := connectAs(t, url, "tap") + if _, err := tap.Subscribe(">", func(m *nats.Msg) { + mu.Lock() + crossed = append(crossed, m.Subject+" "+string(m.Data)) + mu.Unlock() + }); err != nil { + t.Fatal(err) + } + _ = tap.Flush() + + v := newVault(t, Declared{Read: assignments{}.read}) + anchorPub, anchorPriv = aSealingKey(t) + if _, err := moveIn(t, v, "node.anchor", clientsBroker(anchorPub), brokerValue); err != nil { + t.Fatal(err) + } + if _, err := moveIn(t, v, "node.anchor", superuser(), movedValue); err != nil { + t.Fatal(err) + } + + vaultConn := connectAs(t, url, "novox.mesh-vault") + if _, err := vaultConn.Subscribe("mesh.seat.mesh-vault.call.get.*", func(m *nats.Msg) { + parts := strings.Split(m.Subject, ".") + var args struct { + Name string `json:"name"` + PublicKey string `json:"public_key"` + } + _ = json.Unmarshal(m.Data, &args) + answer := busAnswer{} + got, err := v.Get(untoken(parts[len(parts)-1]), args.Name, args.PublicKey) + if err != nil { + answer.Error = err.Error() + } else { + answer.Result, _ = json.Marshal(got) + } + raw, _ := json.Marshal(answer) + _ = m.Respond(raw) + }); err != nil { + t.Fatal(err) + } + _ = vaultConn.Flush() + + controller := connectAs(t, url, "controller") + ask = func(name, key string) (Answer, error) { + raw, _ := json.Marshal(map[string]string{"name": name, "public_key": key}) + msg, err := controller.Request("mesh.seat.mesh-vault.call.get.controller", raw, 3*time.Second) + if err != nil { + return Answer{}, err + } + var a busAnswer + if err := json.Unmarshal(msg.Data, &a); err != nil { + t.Fatal(err) + } + if a.Error != "" { + return Answer{}, errString(a.Error) + } + var got Answer + if err := json.Unmarshal(a.Result, &got); err != nil { + t.Fatal(err) + } + return got, nil + } + heard = func() string { + _ = tap.Flush() + time.Sleep(100 * time.Millisecond) + mu.Lock() + defer mu.Unlock() + return strings.Join(crossed, "\n") + } + return ask, anchorPub, anchorPriv, heard +} + +type errString string + +func (e errString) Error() string { return string(e) } + +func TestOnARealBusTheControllerGetsACoreCredentialSealedOnlyToItsNodesKey(t *testing.T) { + ask, anchorPub, anchorPriv, heard := coreOnABus(t) + + got, err := ask("own/anchor/mesh-provisioner-client/provisioner-broker", pemOf(t, anchorPub)) + if err != nil { + t.Fatal(err) + } + if got.Name != "own/anchor/mesh-provisioner-client/provisioner-broker" || got.Generation != 1 { + t.Errorf("answered %+v", got) + } + opened, err := seal.Open(got.Sealed, anchorPriv) + if err != nil || opened != brokerValue { + t.Fatalf("anchor's key opens %q, %v", opened, err) + } + + // The controller's own key, or any key but anchor's, is refused: it carries the box and never opens it. + mine, err := seal.GenerateKeyPair() + if err != nil { + t.Fatal(err) + } + if got, err := ask("own/anchor/mesh-provisioner-client/provisioner-broker", mine.PublicKey); err == nil { + t.Errorf("the controller was handed anchor's core credential sealed to its own key: %+v", got) + } else if !strings.Contains(err.Error(), "sealing key") { + t.Errorf("refused for another reason: %v", err) + } + // A module that is not the core's is refused, though anchor's key is the one named. + if got, err := ask("own/anchor/postgres/superuser-password", pemOf(t, anchorPub)); err == nil { + t.Errorf("the controller was handed a leaf module's secret: %+v", got) + } else if !strings.Contains(err.Error(), "core") { + t.Errorf("refused for another reason: %v", err) + } + // A machine whose engine stated no key: nothing to seal to, so nothing is handed. + laptopPub, _ := aSealingKey(t) + if _, err := ask("own/laptop/nats/broker", pemOf(t, laptopPub)); err == nil { + t.Error("the controller was handed a core credential for a machine whose engine stated no key") + } + + all := heard() + if !strings.Contains(all, "mesh.seat.mesh-vault.call.get.controller") { + t.Fatalf("the listener heard nothing of the asks:\n%s", all) + } + for _, v := range []string{brokerValue, movedValue} { + for _, form := range []string{v, base64.StdEncoding.EncodeToString([]byte(v))} { + if strings.Contains(all, form) { + t.Fatal("a value crossed the bus in the clear") + } + } + } +} + +// Only the controller is answered by this rule: a machine's client asking for a core credential, which no plan +// names, is still refused by the running rule, and a node-engine is not an asker. +func TestTheCoreRuleAnswersTheControllerAlone(t *testing.T) { + v := newVault(t, Declared{Read: assignments{}.read}) + pub, _ := aSealingKey(t) + if _, err := moveIn(t, v, "node.anchor", clientsBroker(pub), brokerValue); err != nil { + t.Fatal(err) + } + for _, caller := range []string{"provisioner.anchor", "node.anchor", "anchor.mesh-provisioner-client"} { + if _, err := v.Get(caller, "own/anchor/mesh-provisioner-client/provisioner-broker", pemOf(t, pub)); err == nil { + t.Errorf("%s was handed a core credential", caller) + } + } + // The controller is answered for nothing else: not a pair credential, not a provision's. + for _, name := range []string{"pair/anchor/postgres/anchor/gitea", "provision/anchor/postgres"} { + if _, err := v.Get(ControllerCaller, name, pemOf(t, pub)); err == nil { + t.Errorf("the controller was handed %s", name) + } + } +} diff --git a/modules/mesh-vault/cmd/mesh-vault/followups_test.go b/modules/mesh-vault/cmd/mesh-vault/followups_test.go new file mode 100644 index 00000000..aa5fbb8d --- /dev/null +++ b/modules/mesh-vault/cmd/mesh-vault/followups_test.go @@ -0,0 +1,162 @@ +package main + +// The four follow-ups V1's third review (review 263 on mesh-catalog#286) left for this package: the tool layer +// passes an act's words through; a rolled record of hand-outs is said in the journal; the export's fingerprint +// and vault_status are what they say; and an act's unproven `by` is kept beside the proven caller. + +import ( + "fmt" + "sort" + "strings" + "sync" + "testing" + + "git.novox.be/novox/mesh-sdk/go/seal" +) + +func anyOf(list []string) []any { + out := make([]any, len(list)) + for i, s := range list { + out[i] = s + } + return out +} + +// (1) and (4): the retirement tools pass consumers, why, by, via and confirm through to the custody ledger, and +// a `by` naming somebody other than the proven caller is kept beside it. +func TestTheRetirementToolsPassTheActThrough(t *testing.T) { + c := newCustody(t) + c.ask(apps(5)) + c.passes(1) + c.ask(keepOnly(apps(5), 1)) + c.stable() + set := waitingNames(c.Retirement()) + sort.Strings(set) + defs := definitions(&Env{Custody: c.Custody}) + approve := toolNamed(t, defs, "provisioner_retire_approve") + if _, err := approve.runAs(ControllerCaller, map[string]any{"consumers": anyOf(set[:2]), "why": "x"}); err == nil { + t.Error("the tool took a set other than the one waiting") + } + got, err := approve.runAs(ControllerCaller, map[string]any{"consumers": anyOf(set), "why": "the apps are gone", + "by": "person.jochen", "via": "telegram"}) + if err != nil { + t.Fatal(err) + } + if done := got.(map[string]any)["retired"].([]string); len(done) != 4 { + t.Errorf("retired %v", done) + } + last := c.bus.events[len(c.bus.events)-5]["body"].(map[string]any) + for _, e := range c.bus.events { + if b := e["body"].(map[string]any); b["change"] == "approved" { + last = b + } + } + if last["why"] != "the apps are gone" || last["by"] != "person.jochen" || last["via"] != "telegram, called by controller" { + t.Errorf("the approval said %v", last) + } + + // Delete: consumer and confirm through, the act's words with them. + r := retiredC(t) + del := toolNamed(t, definitions(&Env{Custody: r.Custody}), "provisioner_delete") + if _, err := del.runAs("person.jochen", map[string]any{"consumer": "d-app", "confirm": "c-app", "why": "x"}); err == nil { + t.Error("a confirm naming another was taken through the tool") + } + if _, err := del.runAs("person.jochen", map[string]any{"consumer": "d-app", "confirm": "d-app", "why": "gone"}); err != nil { + t.Fatal(err) + } + body := r.bus.events[len(r.bus.events)-1]["body"].(map[string]any) + if body["change"] != "deleted" || body["by"] != "person.jochen" || body["via"] != nil || body["why"] != "gone" { + t.Errorf("the deletion said %v", body) + } + + // Reject, by the proven caller alone: nothing more is added to via. + k := newCustody(t) + k.ask(apps(5)) + k.passes(1) + k.ask(keepOnly(apps(5), 1)) + k.stable() + waiting := waitingNames(k.Retirement()) + if _, err := toolNamed(t, definitions(&Env{Custody: k.Custody}), "provisioner_retire_reject").runAs("person.jochen", + map[string]any{"consumers": anyOf(waiting), "why": "they come back"}); err != nil { + t.Fatal(err) + } + if body := k.bus.events[len(k.bus.events)-1]["body"].(map[string]any); body["by"] != "person.jochen" || body["via"] != nil { + t.Errorf("the rejection said %v", body) + } +} + +// (2): a roll of the record of hand-outs is said in the journal, loudly. +func TestARolledRecordOfHandOutsIsSaid(t *testing.T) { + v := newVault(t, nil) + var mu sync.Mutex + var said []string + v.Log = func(format string, args ...any) { + mu.Lock() + defer mu.Unlock() + said = append(said, fmt.Sprintf(format, args...)) + } + v.Store.HandedMax = 600 + asker, _ := seal.GenerateKeyPair() + for i := 0; i < 20; i++ { + _, _ = v.Get("g14.flood", "pair/postgres/gitea", asker.PublicKey) + } + rolled := 0 + for _, s := range said { + if strings.Contains(s, "THE RECORD OF HAND-OUTS WAS ROLLED") { + rolled++ + } + } + if rolled == 0 || rolled != v.Store.Rolled() { + t.Errorf("rolled %d times, said %d times", v.Store.Rolled(), rolled) + } +} + +// (3): the vault's export names the operator key's fingerprint as the controller's secrets.Fingerprint says it, +// and vault_status says what needs a person. +func TestTheExportsFingerprintAndVaultStatus(t *testing.T) { + v := newVault(t, nil) + if _, err := v.Put("pair/postgres/gitea", "x", "the test"); err != nil { + t.Fatal(err) + } + answer, err := toolNamed(t, definitions(&Env{Vault: v.Vault}), "secret_vault_export").Run(map[string]any{}) + if err != nil { + t.Fatal(err) + } + doc := answer.(map[string]any) + // sha256 of the key, its first eight bytes in hex: the controller's secrets.Fingerprint. + if fp := doc["fingerprint"].(string); !strings.HasPrefix(fp, "sha256:") || len(fp) != len("sha256:")+16 || + fp != keyFingerprint(v.operatorPublic) || doc["operator-key"] != v.operatorPublic { + t.Errorf("the export says %v", doc) + } + if keyFingerprint("") != "" { + t.Error("no key has a fingerprint") + } + + env := &Env{Vault: v.Vault} + status := func() map[string]any { + got, err := toolNamed(t, definitions(env), "vault_status").Run(map[string]any{}) + if err != nil { + t.Fatal(err) + } + return got.(map[string]any) + } + if s := status(); s["ok"] != true || s["copies_on_an_earlier_key"] != 0 || s["operator_key"] != v.operatorPublic { + t.Errorf("a vault in order says %v", s) + } + env.refuseKey("a private key was offered") + if s := status(); s["ok"] != false || s["operator_key_refused"] != "a private key was offered" { + t.Errorf("a refused key says %v", s) + } + env.refuseKey("") + // A new operator key whose resealing fails leaves copies on the earlier key, and the status says so. + pub, _ := operatorKey(t) + v.Store.failWrite = func(Record) error { return fmt.Errorf("disk full") } + _, _ = v.SetOperatorKey(asPEM(t, pub), "the test") + v.Store.failWrite = nil + if s := status(); s["ok"] != false || s["copies_on_an_earlier_key"] != 1 { + t.Errorf("a copy on an earlier key says %v", s) + } + if s, _ := toolNamed(t, definitions(&Env{}), "vault_status").Run(map[string]any{}); s.(map[string]any)["ready"] != false { + t.Errorf("no vault says %v", s) + } +} diff --git a/modules/mesh-vault/cmd/mesh-vault/form_test.go b/modules/mesh-vault/cmd/mesh-vault/form_test.go new file mode 100644 index 00000000..7c7d6e1d --- /dev/null +++ b/modules/mesh-vault/cmd/mesh-vault/form_test.go @@ -0,0 +1,221 @@ +package main + +// A provider the vault holds no row of yet gets its first value in the form its offer declares (issue 768): the +// controller names the form in each consumer's plan entry (`credential-form`, from the provider's manifest), and +// the vault reads it strictly. A provider the vault already holds rows of is unchanged by it. + +import ( + "reflect" + "strings" + "testing" +) + +func TestANewSharedProviderGetsOneFirstValueForEveryHolder(t *testing.T) { + mirror := assignments{ + "anchor.stand-in": handedWith("anchor", "stand-in"), + "g14.app": handedWith("g14", "app", "stand-in-api@anchor/stand-in+shared"), + "ace.other": handedWith("ace", "other", "stand-in-api@anchor/stand-in+shared"), + } + v := pairVault(t, mirror) + if o := pass(t, v, standIn); o.Outcome != PairMade || o.Generation != 1 { + t.Fatalf("a new shared provider: %+v", o) + } + value, _, err := asks(v, "provisioner.anchor", standIn) + if err != nil || value == "" { + t.Fatalf("the provider's machine was refused its first value: %v", err) + } + for _, c := range []string{"provisioner.g14", "provisioner.ace"} { + if got, _, err := asks(v, c, standIn); err != nil || got != value { + t.Fatalf("%s got %q (%v), not the one value", c, got, err) + } + } + if changedSaid(t, v, standIn, value) != 1 { + t.Fatal("secret-changed was not said once for the first value") + } + rows, _ := v.pairRows() + if len(rows[pairKey{"anchor", "stand-in-api"}]) != 0 { + t.Fatal("a login per consumer was made for a provider whose offer shares one credential") + } + if o := pass(t, v, standIn); o.Outcome != PairUnchanged { + t.Fatalf("the pass after the first value: %+v", o) + } +} + +func TestANewPerLoginProviderGetsAFirstLoginOfItsOwnForEachConsumer(t *testing.T) { + const pg = "provision/novox/postgres-database" + mirror := assignments{ + "novox.postgres": handedWith("novox", "postgres"), + "g14.gitea": handedWith("g14", "gitea", "postgres-database@novox/postgres+pair"), + "ace.umami": handedWith("ace", "umami", "postgres-database@novox/postgres#stats+pair"), + } + v := pairVault(t, mirror) + if o := pass(t, v, pg); o.Outcome != PairMade { + t.Fatalf("a new per-login provider: %+v", o) + } + if _, held, _ := v.Store.Read(pg); held { + t.Fatal("one shared value was made for a provider whose consumers each keep a login of their own") + } + gitea, umami := "pair/novox/postgres-database/g14/gitea", "pair/novox/postgres-database/ace/umami/stats" + ofGitea, _, err := asks(v, "provisioner.g14", gitea) + if err != nil || ofGitea == "" { + t.Fatalf("gitea's machine was refused its first login: %v", err) + } + ofUmami, _, err := asks(v, "provisioner.ace", umami) + if err != nil || ofUmami == "" { + t.Fatalf("umami's machine was refused its first login: %v", err) + } + if ofGitea == ofUmami { + t.Fatal("two consumers of a per-login provider were given one value") + } + if _, _, err := asks(v, "provisioner.g14", umami); err == nil { + t.Fatal("gitea's machine was handed umami's login") + } + for name, want := range map[string]string{gitea: ofGitea, umami: ofUmami} { + if got, _, err := asks(v, "provisioner.novox", name); err != nil || got != want { + t.Fatalf("the provider's machine got %q for %s (%v)", got, name, err) + } + } + // The logins are rows now: the next pass remakes none of them. + if o := pass(t, v, pg); o.Outcome != PairPerPair { + t.Fatalf("the pass after the first logins: %+v", o) + } + if got, gen, _ := asks(v, "provisioner.g14", gitea); got != ofGitea || gen != 1 { + t.Fatal("a consumer's login was remade by the next pass") + } +} + +// A provider the vault holds rows of keeps what it had, whatever form the plans name. +func TestAProviderWithRowsIsUnchangedByTheFormItsPlansName(t *testing.T) { + t.Run("stamped rows", func(t *testing.T) { + mirror := assignments{ + "anchor.stand-in": handedWith("anchor", "stand-in"), + "g14.app": handedWith("g14", "app", "stand-in-api@anchor/stand-in+shared"), + } + v := pairVault(t, mirror) + movedRow(t, v, "g14", "app", "the-value-they-share", "stamp-1", OriginMade) + if o := pass(t, v, standIn); o.Outcome != PairAdopted { + t.Fatalf("stamped rows: %+v", o) + } + if got, gen, _ := asks(v, "provisioner.g14", standIn); got != "the-value-they-share" || gen != 1 { + t.Fatal("the adopted value is not the move's") + } + }) + t.Run("logins of their own", func(t *testing.T) { + mirror := assignments{ + "anchor.stand-in": handedWith("anchor", "stand-in"), + "g14.app": handedWith("g14", "app", "stand-in-api@anchor/stand-in+shared"), + "ace.other": handedWith("ace", "other", "stand-in-api@anchor/stand-in+pair"), + } + v := pairVault(t, mirror) + movedRow(t, v, "g14", "app", "login-of-app", "", OriginMade) + movedRow(t, v, "ace", "other", "login-of-other", "", OriginMade) + if o := pass(t, v, standIn); o.Outcome != PairPerPair { + t.Fatalf("unstamped rows: %+v", o) + } + if _, held, _ := v.Store.Read(standIn); held { + t.Fatal("a shared value was made for a provider whose rows are logins of their own") + } + if got, gen, _ := asks(v, "provisioner.g14", "pair/anchor/stand-in-api/g14/app"); got != "login-of-app" || + gen != 1 { + t.Fatal("a consumer's own login changed") + } + }) +} + +// A form the vault does not make (issued by the provider's software, or given by a person), a form it does not know, +// or plans that disagree: nothing is made, and the pass says why. The rule that no holder is left behind still holds. +func TestTheVaultMakesNothingForAFormItDoesNotMakeOrCannotTrust(t *testing.T) { + for form, said := range map[string]string{ + "issued": "issued by the provider's software", + "given": "given by a person", + "one-for-all": "which is not a credential form", + "shared|pair": "disagree", + "shared|": "no plan says whether", + "shared@unmoved": "stand-in on anchor", + } { + t.Run(form, func(t *testing.T) { + provider := handedWith("anchor", "stand-in") + first, second := form, form + if a, b, ok := strings.Cut(form, "|"); ok { + first, second = a, b + } + if a, _, ok := strings.Cut(form, "@"); ok { + first, second, provider = a, a, unmoved("anchor", "stand-in") + } + entry := func(f string) string { + if f == "" { + return "stand-in-api@anchor/stand-in" + } + return "stand-in-api@anchor/stand-in+" + f + } + mirror := assignments{ + "anchor.stand-in": provider, + "g14.app": handedWith("g14", "app", entry(first)), + "ace.other": handedWith("ace", "other", entry(second)), + } + v := pairVault(t, mirror) + if o := pass(t, v, standIn); o.Outcome != PairHeld || !strings.Contains(o.Why, said) { + t.Fatalf("form %s: %+v", form, o) + } + if _, held, _ := v.Store.Read(standIn); held { + t.Fatalf("form %s: a shared value was made", form) + } + if rows, _ := v.pairRows(); len(rows[pairKey{"anchor", "stand-in-api"}]) != 0 { + t.Fatalf("form %s: a login was made", form) + } + }) + } +} + +// A consumer that binds after the move to a per-login provider the vault already holds rows of gets a login of its +// own (issue 771): the rows that exist are never touched, and `secret-changed` is said for the new login only. +func TestANewConsumerOfAPerLoginProviderWithRowsGetsALoginOfItsOwn(t *testing.T) { + mirror := assignments{ + "anchor.stand-in": handedWith("anchor", "stand-in"), + "g14.app": handedWith("g14", "app", "stand-in-api@anchor/stand-in+pair"), + "ace.other": handedWith("ace", "other", "stand-in-api@anchor/stand-in+pair"), + } + v := pairVault(t, mirror) + movedRow(t, v, "g14", "app", "login-of-app", "", OriginMade) + movedRow(t, v, "ace", "other", "login-of-other", "", OriginMade) + app, other := "pair/anchor/stand-in-api/g14/app", "pair/anchor/stand-in-api/ace/other" + third := "pair/anchor/stand-in-api/novox/third" + before, said := map[string]Record{}, map[string]int{} + for _, n := range []string{app, other} { + r, _, _ := v.Store.Read(n) + before[n], said[n] = r, changedSaid(t, v, n, "") + } + + mirror["novox.third"] = handedWith("novox", "third", "stand-in-api@anchor/stand-in+pair") + if o := pass(t, v, standIn); o.Outcome != PairMade { + t.Fatalf("a third consumer binding: %+v", o) + } + got, _, err := asks(v, "provisioner.novox", third) + if err != nil || got == "" || got == "login-of-app" || got == "login-of-other" { + t.Fatalf("the third consumer has no login of its own: %q (%v)", got, err) + } + if got, _, err := asks(v, "provisioner.anchor", third); err != nil || got == "" { + t.Fatalf("the provider's machine was refused the new login: %v", err) + } + for n, was := range before { + now, _, _ := v.Store.Read(n) + if !reflect.DeepEqual(now, was) { + t.Fatalf("%s was touched: %+v, was %+v", n, now, was) + } + if changedSaid(t, v, n, "") != said[n] { + t.Fatalf("secret-changed was said for %s", n) + } + } + if changedSaid(t, v, third, "") != 1 { + t.Fatal("secret-changed was not said once for the new login") + } + if _, held, _ := v.Store.Read(standIn); held { + t.Fatal("a shared value was made") + } + if o := pass(t, v, standIn); o.Outcome != PairPerPair { + t.Fatalf("the pass after the new login: %+v", o) + } + if got2, _, _ := asks(v, "provisioner.novox", third); got2 != got { + t.Fatal("the new login was remade by the next pass") + } +} diff --git a/modules/mesh-vault/cmd/mesh-vault/intake.go b/modules/mesh-vault/cmd/mesh-vault/intake.go new file mode 100644 index 00000000..d7898574 --- /dev/null +++ b/modules/mesh-vault/cmd/mesh-vault/intake.go @@ -0,0 +1,760 @@ +package main + +// What the vault takes in, and how it rotates (novox/hq ADR 0349 clauses 8 to 10; ADR 0277 §22–§27 and ADR 0305 +// with the vault in the controller's place; design 57's V2, issue 675). +// +// Four verbs of the seat: `accept` (a value given at the controller's terminal), `secret-ask` (a value the +// operator types at a desk), `issued` (a value a provider's software issued) and `rotate`. +// +// **A value comes in sealed to a key the vault made for that one call, and opens only here.** Each call opens with +// the public half of a key made for it (`seal_to`, a raw X25519 key in base64, the NaCl anonymous box the desk's +// prompt and a provider's tool already seal to) and ends with what was sealed to that key, which the vault opens +// with the private half: it never leaves this process and is forgotten once used or after two minutes. +// +// **The vault asks the desk and the provider itself** (issue 723). The desk's prompt and a provider's issuing tool +// are asked by the vault, in its own name, under the bus account its own runtime runs on (ADR 0259 §8's runs-as): +// the bus lets that account alone call them naming the vault, so the prompt and the tool know the vault asks, and +// what they seal to the call's key opens here and nowhere else (ADR 0349 clauses 9 and 10, ADR 0277 §23). The +// controller asks `secret-ask` and `issued` once, carrying no key and no sealed value; it reaches the desk and the +// tool itself only for a module still filled the old way, until R1 (issue 682). Only `accept` stays two steps: +// the value is given at the controller's terminal, which seals it there to the call's key. +// +// Which secret each one is, by name in the vault's store: +// +// own/// a module's own secret on one machine +// provision// the one credential of a provider assignment and provision (clause 8) + +import ( + "crypto/rand" + "encoding/base64" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "regexp" + "slices" + "strings" + "time" + + stdio "git.novox.be/novox/mesh-sdk/go" + "golang.org/x/crypto/nacl/box" +) + +// ControllerCaller is the controller's bus user: the one caller that carries a value in (broker.ControllerName). +const ControllerCaller = "controller" + +// Where a value came from. +const ( + OriginMade = "made" + OriginAccepted = "accepted" + OriginTyped = "typed" + OriginIssued = "issued" +) + +// CallOpenFor is how long a call's key waits for its sealed value: longer than any prompt or tool waits. +const CallOpenFor = 2 * time.Minute + +// The bounds of asking for one secret (ADR 0277 §24, the controller's inventory.SecretAsk* until now). +const ( + // SecretAskOpenFor is how long an ask that has not ended counts as open. + SecretAskOpenFor = 2 * time.Minute + // SecretAsksPerHour is how many asks for one secret an hour takes. + SecretAsksPerHour = 3 +) + +var namePart = regexp.MustCompile(`^[a-z0-9][a-z0-9._-]{0,62}$`) + +// OwnSecret is the vault's name for a module's own secret on one machine. +func OwnSecret(node, module, name string) (string, error) { + for what, v := range map[string]string{"the machine": node, "the module": module, "the secret": name} { + if !namePart.MatchString(v) { + return "", fmt.Errorf("%s %q is not a name the mesh writes", what, v) + } + } + return "own/" + node + "/" + module + "/" + name, nil +} + +// ProvisionSecret is the vault's name for the one credential of a provider assignment and provision. +func ProvisionSecret(provider, provision string) (string, error) { + for what, v := range map[string]string{"the provider's machine": provider, "the provision": provision} { + if !namePart.MatchString(v) { + return "", fmt.Errorf("%s %q is not a name the mesh writes", what, v) + } + } + return "provision/" + provider + "/" + provision, nil +} + +// callKey is one call's key and what the call takes in. Never written anywhere. +type callKey struct { + verb, secret, by string + // origin is what an `accept` call keeps the value as: accepted, or issued for a covered account (issue 787). + origin string + // ask is the ask's record for `secret-ask`, which the call's id is. + ask *AskRecord + // move is what `move-in` takes: the row of the controller's store (move.go). + move *MoveOf + public [32]byte + private [32]byte + until time.Time +} + +// Opened is a first step's answer: the call, and the key to seal its value to. Never a value. +type Opened struct { + Call string `json:"call"` + SealTo string `json:"seal_to"` + Secret string `json:"secret"` + AskedBy string `json:"asked_by,omitempty"` +} + +// fromTheController refuses every caller but the controller: only it asks the vault to take a value in, and only +// it carries a value given at its terminal. +func fromTheController(caller, verb string) error { + if caller != ControllerCaller { + if caller == "" { + caller = stdio.Unproven + } + return fmt.Errorf("%s takes a value from the controller alone, which carries it sealed to this vault's key "+ + "for one call; %s is not the controller", verb, caller) + } + return nil +} + +// openCall makes a key for one call and keeps its private half here until the call is taken or two minutes +// have passed. +func (v *Vault) openCall(verb, secret, by string, ask *AskRecord) (string, string, error) { + pub, priv, err := box.GenerateKey(rand.Reader) + if err != nil { + return "", "", err + } + idRaw := make([]byte, 12) + if _, err := rand.Read(idRaw); err != nil { + return "", "", err + } + id := hex.EncodeToString(idRaw) + if ask != nil { + id = ask.ID + } + v.callsMu.Lock() + defer v.callsMu.Unlock() + if v.pending == nil { + v.pending = map[string]*callKey{} + } + now := v.Now() + for k, c := range v.pending { + if now.After(c.until) { + delete(v.pending, k) + } + } + v.pending[id] = &callKey{verb: verb, secret: secret, by: by, ask: ask, public: *pub, private: *priv, + until: now.Add(CallOpenFor)} + return id, base64.StdEncoding.EncodeToString(pub[:]), nil +} + +// takeCall ends a call: its key is forgotten whatever follows. +func (v *Vault) takeCall(verb, id string) (*callKey, error) { + v.callsMu.Lock() + defer v.callsMu.Unlock() + c, ok := v.pending[id] + if !ok || c.verb != verb { + return nil, fmt.Errorf("no %s call %q is open in the vault: it ended, it was never opened, or the vault "+ + "started again; open it anew", verb, id) + } + delete(v.pending, id) + if v.Now().After(c.until) { + return nil, fmt.Errorf("the %s call %q waited longer than %s and is closed; open it anew", verb, id, CallOpenFor) + } + return c, nil +} + +// open opens what was sealed to the call's key, and forgets the key. +func (c *callKey) open(sealed string) ([]byte, error) { + defer func() { c.private = [32]byte{} }() + return c.openOne(sealed) +} + +// openOne opens one value sealed to the call's key and keeps the key, for a call that takes several (one per +// holder, perholder.go); the caller forgets the key. +func (c *callKey) openOne(sealed string) ([]byte, error) { + blob, err := base64.StdEncoding.DecodeString(strings.TrimSpace(sealed)) + if err != nil { + return nil, errors.New("the value is not sealed to this call; nothing was taken") + } + out, ok := box.OpenAnonymous(nil, blob, &c.public, &c.private) + if !ok { + // Never the value, never what failed to open: only that it was not sealed to this call. + return nil, errors.New("the value is not sealed to this call; nothing was taken") + } + return out, nil +} + +func zero(b []byte) { + for i := range b { + b[i] = 0 + } +} + +// ---- accept ------------------------------------------------------------------------------------------------- + +// CoveredSecrets are the names, in a module's own secrets, of the bus accounts the controller mints for a core module +// or a module marked moved and hands to the vault (novox/hq ADR 0374, issue 787): the module's account on the bus +// (`broker`) and, for the machine's provisioner client, its own (`provisioner-broker`). +var CoveredSecrets = []string{"broker", "provisioner-broker"} + +// Covered says whether secret is a covered account: a module's own secret named one of CoveredSecrets. The vault +// cannot tell core or moved from a module's manifest; the controller, which can, is the only caller that may hand +// one in, and the name keeps what it hands to these. +func Covered(secret string) bool { + parts := strings.Split(secret, "/") + return len(parts) == 4 && parts[0] == "own" && slices.Contains(CoveredSecrets, parts[3]) +} + +// AcceptOpen opens a call to take a value given at the controller's terminal for secret, which the terminal +// seals to the call's key before it leaves it (ADR 0349 clause 8: the vault alone opens an accepted value). +// +// **With origin issued it is a covered account the controller minted** (ADR 0374, issue 787): the controller alone +// hands it, for a covered secret alone, and a re-issue replaces the value in place. Without origin it is a value +// given at the terminal, as before. +func (v *Vault) AcceptOpen(caller, secret, by, origin string) (Opened, error) { + if err := fromTheController(caller, "accept"); err != nil { + return Opened{}, err + } + if err := validName(secret); err != nil { + return Opened{}, err + } + kept := OriginAccepted + switch origin { + case "": + if rec, held, err := v.Store.Read(secret); err != nil { + return Opened{}, err + } else if held && rec.Origin == OriginIssued && !Covered(secret) { + return Opened{}, fmt.Errorf("%s is issued by its provider's software, and `secret issued` is the only way "+ + "it enters the vault (ADR 0305 §10); nothing is taken", secret) + } + case OriginIssued: + if !Covered(secret) { + return Opened{}, fmt.Errorf("%s is not a bus account the controller mints for a core or moved module "+ + "(%s), so it is not handed with origin issued; nothing is taken", secret, strings.Join(CoveredSecrets, ", ")) + } + kept = OriginIssued + default: + return Opened{}, fmt.Errorf("the origin %q is not one accept takes (issued, or none for a value given at the "+ + "terminal); nothing is taken", origin) + } + id, key, err := v.openCall("accept", secret, by, nil) + if err != nil { + return Opened{}, err + } + v.callsMu.Lock() + if c := v.pending[id]; c != nil { + c.origin = kept + } + v.callsMu.Unlock() + return Opened{Call: id, SealTo: key, Secret: secret}, nil +} + +// AcceptTake opens the value sealed to the call's key and keeps it, accepted: never remade by the vault. +func (v *Vault) AcceptTake(caller, call, sealed string) (Changed, error) { + if err := fromTheController(caller, "accept"); err != nil { + return Changed{}, err + } + c, err := v.takeCall("accept", call) + if err != nil { + return Changed{}, err + } + value, err := c.open(sealed) + if err != nil { + return Changed{}, err + } + defer zero(value) + if strings.TrimSpace(string(value)) == "" { + return Changed{}, errors.New("the value given is empty; nothing was taken") + } + origin := c.origin + if origin == "" { + origin = OriginAccepted + } + return v.keep(c.secret, string(value), someoneOr(c.by, caller), caller, origin) +} + +// ---- secret-ask --------------------------------------------------------------------------------------------- + +// AskOf is what an ask is for, and where. +type AskOf struct { + Node, Module, Name, Desk string + // AskedBy is who asked the controller, as the bus named its caller; the controller's word, taken from the + // controller alone. + AskedBy string +} + +// AskOpen records an ask for a module's own secret before any prompt opens, holding its bounds — one open ask +// per secret, three an hour (ADR 0277 §24, §25) — and answers the key the desk's prompt seals what is typed to, +// and who asked as the record says it, which the prompt names (§26). +func (v *Vault) AskOpen(caller string, a AskOf) (Opened, error) { + if err := fromTheController(caller, "secret-ask"); err != nil { + return Opened{}, err + } + secret, err := OwnSecret(a.Node, a.Module, a.Name) + if err != nil { + return Opened{}, fmt.Errorf("nobody was asked to type anything: %w", err) + } + if !namePart.MatchString(a.Desk) { + return Opened{}, fmt.Errorf("nobody was asked to type anything: the desk %q is not a machine's name", a.Desk) + } + askedBy := askedByName(a.AskedBy) + v.Store.asksMu.Lock() + defer v.Store.asksMu.Unlock() + now := v.Now().UTC() + recent, err := v.Store.AsksSince(now.Add(-time.Hour)) + if err != nil { + return Opened{}, fmt.Errorf("nobody was asked to type anything: the record of asks cannot be read: %w", err) + } + ended := map[string]bool{} + for _, r := range recent { + if r.Event == AskEnded { + ended[r.ID] = true + } + } + lastHour, openBy := 0, "" + for _, r := range recent { + if r.Event != AskOpened || r.Secret != secret { + continue + } + lastHour++ + if !ended[r.ID] && r.At.After(now.Add(-SecretAskOpenFor)) { + openBy = r.AskedBy + } + } + if openBy != "" { + return Opened{}, fmt.Errorf("an ask for %s of %s on %s is still open (asked by %s): one prompt at a time for a "+ + "secret, and this one ends within %s", a.Name, a.Module, a.Node, openBy, SecretAskOpenFor) + } + if lastHour >= SecretAsksPerHour { + return Opened{}, fmt.Errorf("%s of %s on %s was asked for %d times in the last hour, and an hour takes %d asks "+ + "for one secret: the operator is not kept at a prompt", a.Name, a.Module, a.Node, lastHour, SecretAsksPerHour) + } + idRaw := make([]byte, 12) + if _, err := rand.Read(idRaw); err != nil { + return Opened{}, err + } + rec := AskRecord{ID: hex.EncodeToString(idRaw), At: now, Event: AskOpened, Secret: secret, Node: a.Node, + Module: a.Module, Name: a.Name, Desk: a.Desk, AskedBy: askedBy, Caller: caller} + if err := v.recordAsk(rec); err != nil { + return Opened{}, fmt.Errorf("nobody was asked to type anything: the ask could not be recorded: %w", err) + } + id, key, err := v.openCall("secret-ask", secret, askedBy, &rec) + if err != nil { + _ = v.recordAsk(AskRecord{ID: rec.ID, At: v.Now().UTC(), Event: AskEnded, Secret: secret, Outcome: "failed"}) + return Opened{}, err + } + return Opened{Call: id, SealTo: key, Secret: secret, AskedBy: askedBy}, nil +} + +// The outcomes an ask ends with, as the controller's record had them. +var askOutcomes = map[string]bool{"dismissed": true, "timed-out": true, "empty": true, "refused": true, "failed": true} + +// AskEnd ends an ask: with what the desk sealed to its key, which the vault opens and keeps as typed, or with +// the outcome the desk answered instead. How it ended is recorded whatever happens. +func (v *Vault) AskEnd(caller, call, sealed, outcome string) (Changed, error) { + if err := fromTheController(caller, "secret-ask"); err != nil { + return Changed{}, err + } + if sealed == "" && !askOutcomes[outcome] { + return Changed{}, fmt.Errorf("an ask ends with what was sealed to its key, or with one of dismissed, " + + "timed-out, empty, refused or failed") + } + c, err := v.takeCall("secret-ask", call) + if err != nil { + return Changed{}, err + } + end := func(how string) { + v.Store.asksMu.Lock() + defer v.Store.asksMu.Unlock() + if err := v.recordAsk(AskRecord{ID: c.ask.ID, At: v.Now().UTC(), Event: AskEnded, Secret: c.secret, + Outcome: how}); err != nil { + v.log("the end of ask %s (%s) could not be recorded: %v", c.ask.ID, how, err) + } + } + if sealed == "" { + end(outcome) + return Changed{Name: c.secret, Unchanged: true}, nil + } + opened, err := c.open(sealed) + if err != nil { + end("failed") + return Changed{}, err + } + value := strings.TrimRight(string(opened), "\r\n") + zero(opened) + if strings.TrimSpace(value) == "" { + end("empty") + return Changed{}, fmt.Errorf("%w: the prompt on %s was answered empty", ErrNothingGiven, c.ask.Desk) + } + changed, err := v.keep(c.secret, value, c.ask.AskedBy, caller, OriginTyped) + value = "" + if err != nil { + end("failed") + return Changed{}, err + } + end("given") + return changed, nil +} + +// recordAsk appends one line to the record of asks. The caller holds Store.asksMu, so no two writers roll the +// record at once and no ask is counted against its bounds while another line is half written. +func (v *Vault) recordAsk(r AskRecord) error { + rolled, err := v.Store.RecordAsk(r) + if rolled { + v.log("THE RECORD OF ASKS WAS ROLLED: it reached its bound of %d bytes; the oldest of %d kept files is "+ + "dropped. Many asks in little time is worth a look: secret_asks", v.Store.HandedBound(), HandedKept) + } + return err +} + +// askedByName is who asked as the prompt and the record name it: the first clause of what the bus said, in the +// characters a name has, at most 80 (the controller's own rule, ADR 0277 §26), so no words of the caller's own +// reach the prompt. +func askedByName(caller string) string { + first, _, _ := strings.Cut(caller, ",") + var b strings.Builder + for _, r := range strings.TrimSpace(first) { + if b.Len() >= 80 { + break + } + switch { + case r >= 'a' && r <= 'z', r >= 'A' && r <= 'Z', r >= '0' && r <= '9', r == '.', r == '_', r == '/', r == '@', + r == '-', r == ' ': + b.WriteRune(r) + default: + b.WriteRune('-') + } + } + name := strings.TrimSpace(b.String()) + if name == "" || strings.HasPrefix(name, "-") { + return "an unnamed caller" + } + return name +} + +// ---- issued ------------------------------------------------------------------------------------------------- + +// IssuedOpen opens a call to take the credential a provider's software issued for a provision (ADR 0349 clause +// 10, ADR 0305 with the vault in the controller's place): the controller asks the provider's tool, on the +// provider's machine, with the call's key, and passes on what it answered. +func (v *Vault) IssuedOpen(caller, provision, provider, module string) (Opened, error) { + if err := fromTheController(caller, "issued"); err != nil { + return Opened{}, err + } + secret, err := ProvisionSecret(provider, provision) + if err != nil { + return Opened{}, err + } + if !namePart.MatchString(module) { + return Opened{}, fmt.Errorf("the provider's module %q is not a name the mesh writes", module) + } + id, key, err := v.openCall("issued", secret, module+" on "+provider, nil) + if err != nil { + return Opened{}, err + } + return Opened{Call: id, SealTo: key, Secret: secret}, nil +} + +// IssuedTake opens what the provider's tool sealed to the call's key and keeps it as the provision's one +// credential. The same value again writes nothing (ADR 0305 §5): every holder that has it is left as it is. +func (v *Vault) IssuedTake(caller, call, sealed string) (Changed, error) { + if err := fromTheController(caller, "issued"); err != nil { + return Changed{}, err + } + c, err := v.takeCall("issued", call) + if err != nil { + return Changed{}, err + } + opened, err := c.open(sealed) + if err != nil { + return Changed{}, err + } + value := strings.TrimSpace(string(opened)) + zero(opened) + if value == "" { + return Changed{}, fmt.Errorf("%s answered an empty value; nothing was taken", c.by) + } + return v.keep(c.secret, value, c.by, caller, OriginIssued) +} + +// ---- the vault's own calls (issue 723) ----------------------------------------------------------------------- + +// DeskPromptFor is how long the desk's prompt waits for the operator when the vault asks: under the thirty seconds +// a call through the runtime waits, and under the controller's wait for `secret-ask`. +const DeskPromptFor = 20 + +// DeskPrompt is the desk's secret prompt as the vault asks it: the seat's verb on the desk's machine. +func DeskPrompt(desk string) string { return "seat:node-launcher.secret@" + desk } + +// Asked is how an ask at a desk ended: what was kept, or why nothing was. Never the value. +type Asked struct { + Secret string `json:"secret"` + Generation int `json:"generation,omitempty"` + AskedBy string `json:"asked_by"` + // Outcome is given, dismissed, timed-out, empty, refused or failed. + Outcome string `json:"outcome"` + Unchanged bool `json:"unchanged,omitempty"` +} + +// AskAtDesk is `secret-ask` (ADR 0349 clause 9, ADR 0277 §22–§27): asked by the controller alone, which names who +// asked it. The ask is recorded and its bounds held before any prompt opens; the vault asks the desk's prompt +// itself, in its own name, naming the asker as recorded and giving a key made for this one call; what is typed +// comes back sealed to that key and opens here. How the prompt ended is the answer and the record's. +func (v *Vault) AskAtDesk(caller string, a AskOf) (Asked, error) { + if v.Ask == nil { + return Asked{}, errors.New("nobody was asked to type anything: this vault reaches no desk") + } + opened, err := v.AskOpen(caller, a) + if err != nil { + return Asked{}, err + } + out := Asked{Secret: opened.Secret, AskedBy: opened.AskedBy} + end := func(outcome string) (Asked, error) { + out.Outcome = outcome + if _, err := v.AskEnd(caller, opened.Call, "", outcome); err != nil { + return out, err + } + return out, nil + } + raw, err := v.Ask(DeskPrompt(a.Desk), map[string]any{"module": a.Module, "secret": a.Name, "node": a.Node, + "asked_by": opened.AskedBy, "seal_to": opened.SealTo, "timeout_seconds": DeskPromptFor}) + if err != nil { + if _, ended := end("refused"); ended != nil { + v.log("%v", ended) + } + return Asked{}, fmt.Errorf("the desk on %s could not be asked: %w", a.Desk, err) + } + var answer struct { + Sealed string `json:"sealed"` + Cancelled bool `json:"cancelled"` + TimedOut bool `json:"timed_out"` + } + if err := json.Unmarshal(raw, &answer); err != nil { + return end("failed") + } + switch { + case answer.TimedOut: + return end("timed-out") + case answer.Cancelled: + return end("dismissed") + case answer.Sealed == "": + return end("failed") + } + changed, err := v.AskEnd(caller, opened.Call, answer.Sealed, "") + if err != nil { + // AskEnd recorded how it ended; an empty answer is the operator's, not a failure. + if errors.Is(err, ErrNothingGiven) { + out.Outcome = "empty" + return out, nil + } + return Asked{}, err + } + out.Outcome, out.Generation, out.Unchanged = "given", changed.Generation, changed.Unchanged + return out, nil +} + +// ErrNothingGiven is a prompt answered empty: the operator's answer, not a failure. +var ErrNothingGiven = errors.New("nothing was given") + +// Issued is what the vault kept of a value a provider's software issued. Never the value. For a tool that answers a +// value per holder (perholder.go), Secret is the provision's name, Holders what was kept for each and NotAnswered +// the holders asked about that the tool gave nothing for; Unchanged is then whether no holder's value changed. +type Issued struct { + Secret string `json:"secret"` + Generation int `json:"generation"` + Unchanged bool `json:"unchanged,omitempty"` + Holders []IssuedHolder `json:"holders,omitempty"` + NotAnswered []Holder `json:"not_answered,omitempty"` +} + +// TakeIssued is `issued` (ADR 0349 clause 10, ADR 0305 with the vault in the controller's place): asked by the +// controller alone, naming the provision and the provider's machine, module and tool. The vault asks that tool +// itself, in its own name, with a key made for this one call; the tool answers its software's value sealed to that +// key, which opens here and is kept as the provision's one credential. The bus lets the vault's account call no +// tool but an issuing one, so a tool named here that is not one is refused there. +func (v *Vault) TakeIssued(caller, provision, provider, module, tool string) (Issued, error) { + return v.TakeIssuedFor(caller, provision, provider, module, tool, nil) +} + +// TakeIssuedFor is TakeIssued naming the provision's holders from that provider, as the controller knows them (issue +// 749): the tool is asked with them, and may answer one value for the provision (`sealed`, kept as its one +// credential, as before) or one per holder (`holders`: node, module and sealed each, kept as that holder's own and +// handed to it alone, perholder.go). Never both. +func (v *Vault) TakeIssuedFor(caller, provision, provider, module, tool string, holders []Holder) (Issued, error) { + if v.Ask == nil { + return Issued{}, errors.New("nothing was taken: this vault reaches no provider") + } + if !toolName.MatchString(tool) { + return Issued{}, fmt.Errorf("the provider's tool %q is not a tool's name", tool) + } + if err := checkHolders(holders); err != nil { + return Issued{}, fmt.Errorf("nothing was asked: %w", err) + } + opened, err := v.IssuedOpen(caller, provision, provider, module) + if err != nil { + return Issued{}, err + } + args := map[string]any{"provision": provision, "seal_to": opened.SealTo} + if len(holders) > 0 { + args["holders"] = holders + } + raw, err := v.Ask(module+"."+tool+"@"+provider, args) + if err != nil { + v.dropCall("issued", opened.Call) + return Issued{}, fmt.Errorf("its tool %s could not be asked: %w", tool, err) + } + var given struct { + Sealed string `json:"sealed"` + Holders []sealedFor `json:"holders"` + } + if err := json.Unmarshal(raw, &given); err != nil || (given.Sealed == "") == (len(given.Holders) == 0) { + v.dropCall("issued", opened.Call) + return Issued{}, fmt.Errorf("its tool %s answered no sealed value, or both one value and a value per holder", tool) + } + if len(given.Holders) > 0 { + return v.takePerHolder(caller, opened.Call, provider, provision, holders, given.Holders) + } + changed, err := v.IssuedTake(caller, opened.Call, given.Sealed) + if err != nil { + return Issued{}, err + } + return Issued{Secret: changed.Name, Generation: changed.Generation, Unchanged: changed.Unchanged}, nil +} + +var toolName = regexp.MustCompile(`^[a-z0-9][a-z0-9_-]{0,62}$`) + +// dropCall forgets a call's key that will not be answered. +func (v *Vault) dropCall(verb, id string) { + v.callsMu.Lock() + defer v.callsMu.Unlock() + if c, ok := v.pending[id]; ok && c.verb == verb { + c.private = [32]byte{} + delete(v.pending, id) + } +} + +// ---- rotate ------------------------------------------------------------------------------------------------- + +// Rotate makes a secret anew (ADR 0114, ADR 0228). Refused: a secret the vault does not hold; a value a +// provider's software issued, which the software rotates (ADR 0305 §9); and a provision's accepted credential, +// which the vault does not remake (ADR 0349 clause 8, ADR 0158 §8). A module's own value given by hand is +// rotated like a made one (ADR 0228). Who asked is the person the controller names, beside the controller that +// carried the act — both are kept — and why is said in the vault's log. +// +// **From the controller alone, and with a why** (the review of issue 675): only the controller knows whether a +// module's secrets are filled from the vault yet (ADR 0349 clause 17); a rotation asked of the vault for a module +// still filled the old way would change a value its machine never reads, and drift from the one the controller +// holds. And a rotation is an act by hand, which says why. +func (v *Vault) Rotate(caller, secret, by, why string) (Changed, error) { + return v.RotateFor(caller, secret, Narrow{}, by, why) +} + +// Narrow is the one consumer a rotation of a per-login provider's credential is for (issue 767): the controller's +// `rotate --consumer --module `, either or both. Empty is every consumer. +type Narrow struct { + Consumer string `json:"consumer,omitempty"` + ConsumerModule string `json:"consumer_module,omitempty"` +} + +func (n Narrow) empty() bool { return n.Consumer == "" && n.ConsumerModule == "" } + +func (n Narrow) holds(r pairRow) bool { + return (n.Consumer == "" || r.consumer == n.Consumer) && (n.ConsumerModule == "" || r.module == n.ConsumerModule) +} + +func (n Narrow) String() string { + switch { + case n.Consumer == "": + return n.ConsumerModule + case n.ConsumerModule == "": + return n.Consumer + } + return n.ConsumerModule + " on " + n.Consumer +} + +// RotateFor is Rotate, narrowed to one consumer's own login of a per-login provider (ADR 0158 §11, issue 767). A +// value shared by every holder (ADR 0158) is one value, so it is never rotated for one consumer alone: refused, and +// nothing changes. A module's own secret names no consumer. +func (v *Vault) RotateFor(caller, secret string, one Narrow, by, why string) (Changed, error) { + if caller != ControllerCaller { + return Changed{}, fmt.Errorf("the vault rotates for the controller alone, which knows whether the module's "+ + "secrets are filled from the vault yet; %s is not the controller: `secret rotate` at the controller", caller) + } + if strings.TrimSpace(why) == "" { + return Changed{}, errors.New("not rotated: a rotation says why it was asked (`why`)") + } + if err := validName(secret); err != nil { + return Changed{}, err + } + rec, held, err := v.Store.Read(secret) + if err != nil { + return Changed{}, err + } + parts := strings.Split(secret, "/") + shared := len(parts) == 3 && parts[0] == "provision" + switch { + case !one.empty() && !shared: + return Changed{}, fmt.Errorf("not rotated: %s is not a provision's credential, and only a provider's "+ + "consumers are named in a rotation", secret) + case !one.empty() && held: + return Changed{}, fmt.Errorf("not rotated: %s is one value every holder shares (ADR 0158), so it cannot be "+ + "rotated for %s alone; rotated without a consumer, it is made anew for every holder", secret, one) + case !held && shared: + // A provider whose consumers keep logins of their own: each one's own is remade, never one shared value. + changed, err := v.rotatePerPair(secret, one, someoneOr(by, caller)) + if err == nil { + v.log("rotated each consumer's own login of %s (%d), asked by %s via %s: %s", secret, len(changed.Each), + someoneOr(by, caller), caller, strings.TrimSpace(why)) + } + return changed, err + case !held: + return Changed{}, fmt.Errorf("not rotated: the vault holds no secret named %s", secret) + case Covered(secret): + return Changed{}, fmt.Errorf("not rotated: %s is a bus account the controller mints and hands here (ADR 0374 "+ + "clause 4): only a password whose hash the bus's user list holds can log in, so the vault makes no new "+ + "value for it; `module issue` again hands a new one and replaces this in place", secret) + case rec.Origin == OriginIssued: + return Changed{}, fmt.Errorf("not rotated: %s is issued by its provider's software, which the mesh cannot "+ + "make or rotate (ADR 0305 §9): change it in that software, and `secret issued` takes the value it holds now", + secret) + case rec.Origin == OriginAccepted && strings.HasPrefix(secret, "provision/"): + return Changed{}, fmt.Errorf("not rotated: %s was accepted, and an accepted credential is not remade by the "+ + "vault (ADR 0349 clause 8): accept its new value instead", secret) + } + if shared || (parts[0] == "pair" && (len(parts) == 5 || len(parts) == 6)) { + if why := v.rotationHeldBack(strings.Join(append([]string{"provision"}, parts[1:3]...), "/")); why != "" { + return Changed{}, fmt.Errorf("not rotated: %s", why) + } + } + if by == "" { + by = caller + } + raw := make([]byte, 30) + if _, err := rand.Read(raw); err != nil { + return Changed{}, err + } + changed, err := v.keep(secret, base64.RawURLEncoding.EncodeToString(raw), by, caller, OriginMade) + if err == nil && shared { + if sh, recorded, rerr := v.Store.ReadShared(secret); rerr == nil && recorded { + sh.Generation, sh.At = changed.Generation, v.Now().UTC() + if werr := v.Store.WriteShared(sh); werr != nil { + v.log("%s was rotated, but the record of who holds it keeps the earlier generation: %v", secret, werr) + } + } + } + if err == nil { + v.log("rotated %s (generation %d), asked by %s via %s: %s", secret, changed.Generation, by, caller, + strings.TrimSpace(why)) + } + return changed, err +} + +func someoneOr(by, caller string) string { + if strings.TrimSpace(by) == "" { + return caller + } + return by +} diff --git a/modules/mesh-vault/cmd/mesh-vault/intake_test.go b/modules/mesh-vault/cmd/mesh-vault/intake_test.go new file mode 100644 index 00000000..e795bad7 --- /dev/null +++ b/modules/mesh-vault/cmd/mesh-vault/intake_test.go @@ -0,0 +1,552 @@ +package main + +// ADR 0349's tests for what the vault takes in (design 57's V2, issue 675): the vault's `secret-ask` test, the +// issuing provider's test with a stand-in provider (plex's own tool moves with the media catalogue, after +// release 57), an accepted value opened in the vault alone and never remade, and rotate. +// +// The two-step tests below drive the steps the vault's own calls are made of (owncalls_test.go, issue 723): "the +// controller" there is a test function that holds no private key, standing where the vault's call to the desk or the +// provider stands, and carries only what was sealed to the key the vault made for the call. + +import ( + "crypto/rand" + "encoding/base64" + "encoding/json" + "errors" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "git.novox.be/novox/mesh-sdk/go/seal" + "golang.org/x/crypto/nacl/box" +) + +// sealToCall seals value to a call's key as the desk's prompt and a provider's tool do: the NaCl anonymous box. +func sealToCall(t *testing.T, sealTo, value string) string { + t.Helper() + raw, err := base64.StdEncoding.DecodeString(sealTo) + if err != nil || len(raw) != 32 { + t.Fatalf("seal_to %q is not a key", sealTo) + } + var key [32]byte + copy(key[:], raw) + sealed, err := box.SealAnonymous(nil, []byte(value), &key, rand.Reader) + if err != nil { + t.Fatal(err) + } + return base64.StdEncoding.EncodeToString(sealed) +} + +// handedTo opens what `get` hands an allowed asker: the value the vault holds. +func handedTo(t *testing.T, v testVault, asker, name string) string { + t.Helper() + key, _ := seal.GenerateKeyPair() + a, err := v.Get(asker, name, key.PublicKey) + if err != nil { + t.Fatal(err) + } + value, err := seal.Open(a.Sealed, key.PrivateKey) + if err != nil { + t.Fatal(err) + } + return value +} + +// everything is all the vault said and wrote during a test, as text. +func everything(t *testing.T, v testVault, more ...any) string { + t.Helper() + var b strings.Builder + for _, said := range v.bus.raw { + b.WriteString(said) + } + for _, content := range filesUnder(t, v.dir) { + b.WriteString(content) + } + for _, m := range more { + raw, _ := json.Marshal(m) + b.Write(raw) + } + return b.String() +} + +const typedAtTheDesk = "123456789:AAEhBP0av28P4XFQnIuR-o-7Xnz1kkUzW3g" + +func telegramAsk(by string) AskOf { + return AskOf{Node: "anchor", Module: "telegram", Name: "telegram-token", Desk: "laptop", AskedBy: by} +} + +// ---- the vault's secret-ask test (ADR 0349 "How it is checked") --------------------------------------------- + +// An ask is recorded in the vault's store before the prompt opens; its prompt names the asker as the bus named +// it; the typed value opens only in the vault; the controller carries ciphertext only. +func TestAnAskIsRecordedBeforeThePromptAndTheTypedValueOpensOnlyInTheVault(t *testing.T) { + v := newVault(t, AllowList{"anchor.node-provisioner": {"own/anchor/telegram/telegram-token"}}) + opened, err := v.AskOpen(ControllerCaller, telegramAsk("g14/claude-code, via the console")) + if err != nil { + t.Fatal(err) + } + // Recorded before any prompt opens: the record holds the ask now, with no end. + asks, _ := v.Store.Asks(10) + if len(asks) != 1 || asks[0].Event != AskOpened || asks[0].ID != opened.Call || asks[0].AskedBy != "g14/claude-code" || + asks[0].Caller != ControllerCaller || asks[0].Desk != "laptop" || asks[0].Secret != "own/anchor/telegram/telegram-token" { + t.Fatalf("the record before the prompt: %+v", asks) + } + // The prompt names the asker as the bus named it, cut to a name: what the vault answers is what it recorded. + if opened.AskedBy != "g14/claude-code" { + t.Errorf("the prompt would name %q", opened.AskedBy) + } + // The desk seals what is typed to the call's key; the controller passes it on. + changed, err := v.AskEnd(ControllerCaller, opened.Call, sealToCall(t, opened.SealTo, typedAtTheDesk+"\n"), "") + if err != nil { + t.Fatal(err) + } + if changed.Generation != 1 || changed.By != "g14/claude-code" { + t.Errorf("kept as %+v", changed) + } + if got := handedTo(t, v, "anchor.node-provisioner", "own/anchor/telegram/telegram-token"); got != typedAtTheDesk { + t.Errorf("the vault holds %q, not what was typed with its line ending taken off", got) + } + rec, _, _ := v.Store.Read("own/anchor/telegram/telegram-token") + if rec.Origin != OriginTyped || rec.ChangedBy != "g14/claude-code" || rec.ChangedVia != ControllerCaller { + t.Errorf("the record says origin %q, by %q, via %q", rec.Origin, rec.ChangedBy, rec.ChangedVia) + } + asks, _ = v.Store.Asks(10) + if len(asks) != 2 || asks[0].Event != AskEnded || asks[0].Outcome != "given" || asks[0].ID != opened.Call { + t.Errorf("how it ended: %+v", asks) + } + if strings.Contains(everything(t, v, opened, changed, asks), typedAtTheDesk) { + t.Fatal("the typed value is in something the vault said, answered or wrote in the clear") + } +} + +// One open ask per secret, and three an hour (ADR 0277 §24), read from the vault's own record. +func TestAnAskIsBoundedToOneOpenPerSecretAndThreeAnHour(t *testing.T) { + v := newVault(t, nil) + first, err := v.AskOpen(ControllerCaller, telegramAsk("g14.claude-code")) + if err != nil { + t.Fatal(err) + } + if _, err := v.AskOpen(ControllerCaller, telegramAsk("ace.claude-code")); err == nil || + !strings.Contains(err.Error(), "still open (asked by g14.claude-code)") { + t.Fatalf("a second ask while one is open: %v", err) + } + // Another secret is not held back by it. + other := telegramAsk("ace.claude-code") + other.Name = "webhook-secret" + if _, err := v.AskOpen(ControllerCaller, other); err != nil { + t.Fatalf("another secret's ask was held back: %v", err) + } + if _, err := v.AskEnd(ControllerCaller, first.Call, "", "dismissed"); err != nil { + t.Fatal(err) + } + for i := 0; i < 2; i++ { + o, err := v.AskOpen(ControllerCaller, telegramAsk("g14.claude-code")) + if err != nil { + t.Fatalf("ask %d of the hour: %v", i+2, err) + } + if _, err := v.AskEnd(ControllerCaller, o.Call, "", "timed-out"); err != nil { + t.Fatal(err) + } + } + if _, err := v.AskOpen(ControllerCaller, telegramAsk("g14.claude-code")); err == nil || + !strings.Contains(err.Error(), "3 times in the last hour") { + t.Fatalf("a fourth ask in the hour: %v", err) + } + asks, _ := v.Store.Asks(50) + outcomes := map[string]int{} + for _, a := range asks { + outcomes[a.Outcome]++ + } + if outcomes["dismissed"] != 1 || outcomes["timed-out"] != 2 { + t.Errorf("the record's outcomes: %v", outcomes) + } + // An ask left open counts as open for SecretAskOpenFor only: a prompt that died does not hold the secret. + w := newVault(t, nil) + clock := time.Date(2026, 10, 14, 9, 0, 0, 0, time.UTC) + w.Now = func() time.Time { return clock } + if _, err := w.AskOpen(ControllerCaller, telegramAsk("g14.claude-code")); err != nil { + t.Fatal(err) + } + clock = clock.Add(SecretAskOpenFor + time.Second) + if _, err := w.AskOpen(ControllerCaller, telegramAsk("g14.claude-code")); err != nil { + t.Errorf("an ask left open past its time still held the secret: %v", err) + } +} + +// Only the controller reaches a desk's prompt, so only it opens and ends an ask; a value not sealed to the +// call's key, or a call that waited too long, keeps nothing. +func TestAnAskIsTakenFromTheControllerAloneAndOnlySealedToItsCall(t *testing.T) { + v := newVault(t, nil) + for _, caller := range []string{"g14.node-tools", "unproven", "", "person.jochen"} { + if _, err := v.AskOpen(caller, telegramAsk("x")); err == nil { + t.Errorf("%q opened an ask", caller) + } + } + if asks, _ := v.Store.Asks(10); len(asks) != 0 { + t.Fatalf("a refused caller filled the record: %+v", asks) + } + opened, err := v.AskOpen(ControllerCaller, telegramAsk("g14.claude-code")) + if err != nil { + t.Fatal(err) + } + if _, err := v.AskEnd("g14.node-tools", opened.Call, sealToCall(t, opened.SealTo, "x"), ""); err == nil { + t.Error("another caller ended the ask") + } + other, _, _ := box.GenerateKey(rand.Reader) + if _, err := v.AskEnd(ControllerCaller, opened.Call, sealToCall(t, base64.StdEncoding.EncodeToString(other[:]), "x"), ""); err == nil || + !strings.Contains(err.Error(), "not sealed to this call") { + t.Errorf("a value sealed to another key: %v", err) + } + if _, held, _ := v.Store.Read("own/anchor/telegram/telegram-token"); held { + t.Error("a value not sealed to the call was kept") + } + // The call's key is forgotten once used: the same call cannot be answered twice. + if _, err := v.AskEnd(ControllerCaller, opened.Call, sealToCall(t, opened.SealTo, "x"), ""); err == nil { + t.Error("a call was answered twice") + } + asks, _ := v.Store.Asks(1) + if asks[0].Outcome != "failed" { + t.Errorf("the ask that failed ended %+v", asks[0]) + } + // A call that waited longer than its time is closed. + clock := time.Date(2026, 10, 14, 9, 0, 0, 0, time.UTC) + v.Now = func() time.Time { return clock } + late, err := v.IssuedOpen(ControllerCaller, "plex-api", "anchor", "plex") + if err != nil { + t.Fatal(err) + } + clock = clock.Add(CallOpenFor + time.Second) + if _, err := v.IssuedTake(ControllerCaller, late.Call, sealToCall(t, late.SealTo, "x")); err == nil { + t.Error("a call that waited too long was taken") + } + // An empty answer gives nothing and says so. + clock = clock.Add(time.Hour) + empty, _ := v.AskOpen(ControllerCaller, telegramAsk("g14.claude-code")) + if _, err := v.AskEnd(ControllerCaller, empty.Call, sealToCall(t, empty.SealTo, " \n"), ""); err == nil { + t.Error("an empty answer was kept") + } + if asks, _ := v.Store.Asks(1); asks[0].Outcome != "empty" { + t.Errorf("an empty answer ended %+v", asks[0]) + } +} + +// ---- the issuing provider's test, with a stand-in provider ------------------------------------------------- + +// standInProvider is a provider's tool for an `issued-by: software` offer, as plex's must answer (ADR 0305 §2 +// with the vault in the controller's place): it answers the vault's own account, which asks it itself (issue 723), +// and the controller, whose path for a provider filled the old way stays until R1 (issue 682); it refuses every +// other caller, and answers only the software's value sealed to the call's key. +func standInProvider(software *string) func(caller string, args map[string]any) (map[string]any, error) { + return func(caller string, args map[string]any) (map[string]any, error) { + if caller != ControllerCaller && !strings.HasSuffix(caller, ".mesh-vault") { + return nil, errors.New("this tool answers the vault and, until R1, the controller alone") + } + key, _ := args["seal_to"].(string) + raw, err := base64.StdEncoding.DecodeString(key) + if err != nil || len(raw) != 32 { + return nil, errors.New("seal_to is not a sealing key") + } + var to [32]byte + copy(to[:], raw) + sealed, err := box.SealAnonymous(nil, []byte(*software), &to, rand.Reader) + if err != nil { + return nil, err + } + return map[string]any{"sealed": base64.StdEncoding.EncodeToString(sealed)}, nil + } +} + +// issue is the controller's `secret issued` for one provider: one ask of the vault, which asks the provider's +// tool itself, in its own name. +func issue(t *testing.T, v testVault, provider func(string, map[string]any) (map[string]any, error)) (Changed, error) { + t.Helper() + v.Ask = (&aMesh{tools: map[string]func(string, map[string]any) (map[string]any, error){ + "plex.plex_issued_credential@anchor": provider}}).ask + got, err := v.TakeIssued(ControllerCaller, "plex-api", "anchor", "plex", "plex_issued_credential") + if err != nil { + return Changed{}, err + } + rec, _, _ := v.Store.Read(got.Secret) + return Changed{Name: got.Secret, Generation: got.Generation, By: rec.ChangedBy, Unchanged: got.Unchanged}, nil +} + +func TestAnIssuingProviderAnswersOnlyTheVaultsCallSealedToTheVault(t *testing.T) { + v := newVault(t, AllowList{"anchor.node-provisioner": {"provision/anchor/plex-api"}}) + software := "plex-token-from-the-vendor" + provider := standInProvider(&software) + + // The provider's tool refuses every caller but the vault's account and the controller, whatever it asks. + for _, caller := range []string{"anchor.node-tools", "g14.claude-code", "unproven", ""} { + if _, err := provider(caller, map[string]any{"provision": "plex-api", "seal_to": "x"}); err == nil { + t.Errorf("the provider's tool answered %q", caller) + } + } + // The vault takes the steps from the controller alone. + if _, err := v.IssuedOpen("anchor.node-tools", "plex-api", "anchor", "plex"); err == nil { + t.Error("the vault opened an issued call for another caller") + } + + changed, err := issue(t, v, provider) + if err != nil { + t.Fatal(err) + } + if changed.Name != "provision/anchor/plex-api" || changed.Generation != 1 || changed.By != "plex on anchor" { + t.Errorf("kept as %+v", changed) + } + if got := handedTo(t, v, "anchor.node-provisioner", "provision/anchor/plex-api"); got != software { + t.Errorf("the vault holds %q", got) + } + rec, _, _ := v.Store.Read("provision/anchor/plex-api") + if rec.Origin != OriginIssued || rec.ChangedVia != ControllerCaller { + t.Errorf("the record: origin %q via %q", rec.Origin, rec.ChangedVia) + } + + // Nothing rewritten for a value every holder has (ADR 0305 §5): no new generation, no write, nothing said. + path := filepath.Join(v.dir, "store", "secrets", "provision%2Fanchor%2Fplex-api.json") + before, _ := os.ReadFile(path) + said := len(v.bus.raw) + again, err := issue(t, v, provider) + if err != nil { + t.Fatal(err) + } + after, _ := os.ReadFile(path) + if !again.Unchanged || again.Generation != 1 || string(before) != string(after) || len(v.bus.raw) != said { + t.Errorf("the same value again rewrote something: %+v", again) + } + + // The software changes its value: the next ask keeps it as a new generation, and says so without it. + software = "plex-token-renewed" + renewed, err := issue(t, v, provider) + if err != nil || renewed.Generation != 2 { + t.Fatalf("%+v %v", renewed, err) + } + if got := handedTo(t, v, "anchor.node-provisioner", "provision/anchor/plex-api"); got != software { + t.Errorf("after the change the vault holds %q", got) + } + + // A value sealed to any other key is refused, and nothing changes. + opened, _ := v.IssuedOpen(ControllerCaller, "plex-api", "anchor", "plex") + stranger, _, _ := box.GenerateKey(rand.Reader) + if _, err := v.IssuedTake(ControllerCaller, opened.Call, sealToCall(t, base64.StdEncoding.EncodeToString(stranger[:]), "forged")); err == nil { + t.Error("a value not sealed to the call was taken") + } + + // The software rotates it, not the mesh (ADR 0305 §9); and no other path puts it into the vault (§10). + if _, err := v.Rotate(ControllerCaller, "provision/anchor/plex-api", "person.jochen", "leaked"); err == nil || + !strings.Contains(err.Error(), "software") { + t.Errorf("rotate of an issued value: %v", err) + } + if _, err := v.AcceptOpen(ControllerCaller, "provision/anchor/plex-api", "person.jochen", ""); err == nil { + t.Error("an issued value was taken by accept") + } + if strings.Contains(everything(t, v), "plex-token") { + t.Fatal("the issued value is in something the vault said or wrote in the clear") + } +} + +// ---- accept and rotate ------------------------------------------------------------------------------------- + +// A value given at the controller's terminal is sealed there to the vault's call and opened here alone; an +// accepted provision's credential is never remade; a module's own given value rotates as a made one (ADR 0228). +func TestAnAcceptedValueOpensInTheVaultAloneAndIsNotRemade(t *testing.T) { + v := newVault(t, AllowList{"g14.node-provisioner": {"provision/novox/postgres", "own/g14/dunst/token"}}) + if _, err := v.AcceptOpen("g14.claude-code", "provision/novox/postgres", "", ""); err == nil { + t.Error("accept was opened for a caller that is not the controller") + } + for name, value := range map[string]string{"provision/novox/postgres": "pw-of-last-year", "own/g14/dunst/token": "tok"} { + opened, err := v.AcceptOpen(ControllerCaller, name, "jochen", "") + if err != nil { + t.Fatal(err) + } + changed, err := v.AcceptTake(ControllerCaller, opened.Call, sealToCall(t, opened.SealTo, value)) + if err != nil { + t.Fatal(err) + } + if changed.By != "jochen" { + t.Errorf("%s accepted by %q", name, changed.By) + } + if got := handedTo(t, v, "g14.node-provisioner", name); got != value { + t.Errorf("%s holds %q", name, got) + } + rec, _, _ := v.Store.Read(name) + if rec.Origin != OriginAccepted || rec.ChangedVia != ControllerCaller { + t.Errorf("%s: origin %q via %q", name, rec.Origin, rec.ChangedVia) + } + } + if _, err := v.Rotate(ControllerCaller, "provision/novox/postgres", "jochen", "leaked"); err == nil || + !strings.Contains(err.Error(), "not remade") { + t.Errorf("rotate of an accepted credential: %v", err) + } + rotated, err := v.Rotate(ControllerCaller, "own/g14/dunst/token", "jochen", "leaked") + if err != nil || rotated.Generation != 2 { + t.Fatalf("rotate of a module's own given value: %+v %v", rotated, err) + } + if got := handedTo(t, v, "g14.node-provisioner", "own/g14/dunst/token"); got == "tok" || len(got) != 40 { + t.Errorf("rotated to %q", got) + } + if strings.Contains(everything(t, v), "pw-of-last-year") { + t.Fatal("an accepted value is in something the vault said or wrote in the clear") + } +} + +// Rotate: refused for a secret not held and for an unproven caller; who asked is the proven caller, or the +// person the controller names when it carries the act — both kept. +func TestRotateKeepsWhoAskedAndWhoCarriedIt(t *testing.T) { + v := newVault(t, nil) + if _, err := v.Rotate(ControllerCaller, "own/g14/dunst/token", "jochen", "leaked"); err == nil { + t.Error("a secret not held was rotated") + } + if _, err := v.Mint("own/g14/dunst/token", "the test"); err != nil { + t.Fatal(err) + } + if _, err := v.Rotate("unproven", "own/g14/dunst/token", "", "leaked"); err == nil { + t.Error("an unproven caller rotated") + } + if _, err := v.Rotate(ControllerCaller, "own/g14/dunst/token", "jochen", "leaked"); err != nil { + t.Fatal(err) + } + rec, _, _ := v.Store.Read("own/g14/dunst/token") + if rec.ChangedBy != "jochen" || rec.ChangedVia != ControllerCaller || rec.Generation != 2 { + t.Errorf("by %q via %q generation %d", rec.ChangedBy, rec.ChangedVia, rec.Generation) + } +} + +// Rotate is the controller's alone, and says why (the review of issue 675): only the controller knows whether a +// module is filled from the vault yet, so a rotation asked by any other caller — a proven one included — would +// drift from the value the controller holds; and a rotation without a reason is refused. Neither changes anything. +func TestRotateIsTheControllersAloneAndSaysWhy(t *testing.T) { + v := newVault(t, nil) + if _, err := v.Mint("own/g14/dunst/token", "the test"); err != nil { + t.Fatal(err) + } + for _, caller := range []string{"g14.claude-code", "g14.node-provisioner", "unproven", ""} { + if _, err := v.Rotate(caller, "own/g14/dunst/token", "jochen", "leaked"); err == nil || + !strings.Contains(err.Error(), "controller alone") { + t.Errorf("%q rotated: %v", caller, err) + } + } + for _, why := range []string{"", " "} { + if _, err := v.Rotate(ControllerCaller, "own/g14/dunst/token", "jochen", why); err == nil || + !strings.Contains(err.Error(), "why") { + t.Errorf("a rotation without a why (%q): %v", why, err) + } + } + if rec, _, _ := v.Store.Read("own/g14/dunst/token"); rec.Generation != 1 { + t.Fatalf("a refused rotation changed the secret: generation %d", rec.Generation) + } +} + +// The names the vault keeps them under are names, never paths. +func TestTheNamesOfWhatIsTakenIn(t *testing.T) { + if n, err := OwnSecret("g14", "dunst", "token"); err != nil || n != "own/g14/dunst/token" { + t.Errorf("%q %v", n, err) + } + if n, err := ProvisionSecret("novox", "postgres"); err != nil || n != "provision/novox/postgres" { + t.Errorf("%q %v", n, err) + } + for _, bad := range [][3]string{{"..", "x", "y"}, {"g14", "a/b", "y"}, {"g14", "x", ""}, {"G14", "x", "y"}} { + if _, err := OwnSecret(bad[0], bad[1], bad[2]); err == nil { + t.Errorf("%v was taken", bad) + } + } +} + +// ---- the verbs as the runtime calls them ------------------------------------------------------------------ + +// Each verb takes its caller from the runtime, never from its arguments, and names its secret one way only. +func TestTheVerbsTakeTheirCallerFromTheRuntime(t *testing.T) { + v := newVault(t, nil) + env := &Env{Vault: v.Vault} + defs := definitions(env) + var asked []map[string]any + software := "tok" + v.Ask = (&aMesh{desk: aDesk("typed", &asked), tools: map[string]func(string, map[string]any) (map[string]any, error){ + "plex.plex_issued_credential@anchor": standInProvider(&software)}}).ask + ask := toolNamed(t, defs, Seat+".secret-ask") + args := map[string]any{"node": "anchor", "module": "telegram", "name": "telegram-token", "desk": "laptop", + "asked_by": "g14.claude-code", "caller": ControllerCaller} + if _, err := ask.Run(args); err == nil { + t.Error("an ask opened for a caller named in the arguments") + } + answer, err := ask.runAs(ControllerCaller, args) + if err != nil || answer.(Asked).Outcome != "given" || len(asked) != 1 { + t.Fatalf("secret-ask: %+v %v", answer, err) + } + listed, err := toolNamed(t, defs, "secret_asks").Run(map[string]any{}) + if err != nil || len(listed.([]AskRecord)) != 2 { + t.Fatalf("secret_asks: %v %v", listed, err) + } + + accept := toolNamed(t, defs, Seat+".accept") + if _, err := accept.runAs(ControllerCaller, map[string]any{"node": "g14", "provision": "postgres"}); err == nil { + t.Error("accept took a name both ways") + } + answer, err = accept.runAs(ControllerCaller, map[string]any{"provider": "novox", "provision": "postgres", "by": "jochen"}) + if err != nil { + t.Fatal(err) + } + o := answer.(Opened) + if _, err := accept.runAs(ControllerCaller, map[string]any{"call": o.Call, "sealed": sealToCall(t, o.SealTo, "pw")}); err != nil { + t.Fatal(err) + } + rotate := toolNamed(t, defs, Seat+".rotate") + if _, err := rotate.runAs("g14.claude-code", map[string]any{"node": "g14", "module": "dunst", "name": "token"}); err == nil { + t.Error("a secret not held was rotated through the verb") + } + + issued := toolNamed(t, defs, Seat+".issued") + answer, err = issued.runAs(ControllerCaller, map[string]any{"provision": "plex-api", "provider": "anchor", + "module": "plex", "tool": "plex_issued_credential"}) + if err != nil || answer.(Issued).Secret != "provision/anchor/plex-api" { + t.Fatalf("issued: %+v %v", answer, err) + } + broken := &Env{Broken: errors.New("no store")} + if _, err := toolNamed(t, definitions(broken), Seat+".rotate").runAs(ControllerCaller, map[string]any{}); err == nil || + err.Error() != "no store" { + t.Errorf("a broken vault said %v", err) + } +} + +// How an ask ended is written under the record's lock (the review of issue 675), as its opening is: two writers +// never roll asks.jsonl at once, and an ask opened meanwhile never counts its bounds from a half-written record. +// With the lock held elsewhere, AskEnd waits, and writes nothing until it is let go. +func TestTheEndOfAnAskIsWrittenUnderTheRecordsLock(t *testing.T) { + v := newVault(t, nil) + opened, err := v.AskOpen(ControllerCaller, telegramAsk("g14.claude-code")) + if err != nil { + t.Fatal(err) + } + ended := func() int { + t.Helper() + raw, err := os.ReadFile(filepath.Join(v.Store.dir, "asks.jsonl")) + if err != nil { + t.Fatal(err) + } + return strings.Count(string(raw), `"`+string(AskEnded)+`"`) + } + v.Store.asksMu.Lock() + done := make(chan error, 1) + go func() { + _, err := v.AskEnd(ControllerCaller, opened.Call, "", "dismissed") + done <- err + }() + select { + case err := <-done: + v.Store.asksMu.Unlock() + t.Fatalf("the end of the ask was written while another writer held the record (%v)", err) + case <-time.After(200 * time.Millisecond): + } + if n := ended(); n != 0 { + v.Store.asksMu.Unlock() + t.Fatalf("%d end(s) written under another writer's lock", n) + } + v.Store.asksMu.Unlock() + if err := <-done; err != nil { + t.Fatal(err) + } + if n := ended(); n != 1 { + t.Fatalf("%d end(s) written once the lock was let go", n) + } +} diff --git a/modules/mesh-vault/cmd/mesh-vault/issued_account_test.go b/modules/mesh-vault/cmd/mesh-vault/issued_account_test.go new file mode 100644 index 00000000..01e99e8f --- /dev/null +++ b/modules/mesh-vault/cmd/mesh-vault/issued_account_test.go @@ -0,0 +1,99 @@ +package main + +import ( + "strings" + "testing" +) + +// A bus account the controller minted for a core or moved module is handed to the vault with origin issued (novox/hq +// ADR 0374, issue 787): kept as that module's own secret on that machine, served to the controller's core get and to +// the moved module's client, replaced in place on a re-issue with `secret-changed`, and never made anew by the vault. + +func handIssued(t *testing.T, v testVault, caller, secret, value string) (Changed, error) { + t.Helper() + opened, err := v.AcceptOpen(caller, secret, "module issue", OriginIssued) + if err != nil { + return Changed{}, err + } + return v.AcceptTake(caller, opened.Call, sealToCall(t, opened.SealTo, value)) +} + +func TestAHandedBusAccountIsKeptServedReplacedAndNeverMadeByTheVault(t *testing.T) { + const moved = "own/g14/moved-bot/broker" + v := newVault(t, AllowList{"provisioner.g14": {moved}}) + + first, err := handIssued(t, v, ControllerCaller, moved, `{"password":"first-pw"}`) + if err != nil || first.Generation != 1 { + t.Fatalf("the handed account: %+v %v", first, err) + } + rec, held, _ := v.Store.Read(moved) + if !held || rec.Origin != OriginIssued || rec.ChangedVia != ControllerCaller { + t.Fatalf("kept as origin %q via %q (held %v), want issued via the controller", rec.Origin, rec.ChangedVia, held) + } + if got := handedTo(t, v, "provisioner.g14", moved); got != `{"password":"first-pw"}` { + t.Errorf("the moved module's client is handed %q", got) + } + + // A re-issue replaces it in place, a new generation, and the vault says it changed. + before := len(v.bus.events) + second, err := handIssued(t, v, ControllerCaller, moved, `{"password":"second-pw"}`) + if err != nil || second.Generation != 2 || second.Unchanged { + t.Fatalf("the re-issue: %+v %v", second, err) + } + if got := handedTo(t, v, "provisioner.g14", moved); got != `{"password":"second-pw"}` { + t.Errorf("after the re-issue the client is handed %q: a stale value stayed", got) + } + changedEvents := 0 + for _, e := range v.bus.events[before:] { + if e["event"] == ChangedEvent { + changedEvents++ + } + } + if changedEvents != 1 { + t.Errorf("the re-issue raised %d secret-changed events, want 1", changedEvents) + } + + // The vault makes no value for it: not a rotation, and not an accept without origin over it either way round. + if _, err := v.Rotate(ControllerCaller, moved, "jochen", "leaked"); err == nil || !strings.Contains(err.Error(), "ADR 0374") { + t.Errorf("the vault rotated a covered account: %v", err) + } + if _, err := v.Rotate(ControllerCaller, "own/g14/moved-bot/provisioner-broker", "jochen", "leaked"); err == nil { + t.Error("the vault made a first value for a covered account by rotating it") + } + if got := handedTo(t, v, "provisioner.g14", moved); got != `{"password":"second-pw"}` { + t.Errorf("a refused rotation changed the value to %q", got) + } + + // A secret that is not a covered account is not handed with origin issued; and only the controller hands one. + if _, err := v.AcceptOpen(ControllerCaller, "own/g14/moved-bot/token", "x", OriginIssued); err == nil { + t.Error("origin issued was taken for a secret that is not a bus account") + } + if _, err := v.AcceptOpen(ControllerCaller, "provision/anchor/postgres", "x", OriginIssued); err == nil { + t.Error("origin issued was taken for a provision's credential") + } + if _, err := v.AcceptOpen("g14.moved-bot", moved, "x", OriginIssued); err == nil { + t.Error("a caller that is not the controller handed a covered account") + } + if _, err := v.AcceptOpen(ControllerCaller, moved, "x", "made"); err == nil { + t.Error("an origin accept does not take was taken") + } + if strings.Contains(everything(t, v), "first-pw") || strings.Contains(everything(t, v), "second-pw") { + t.Fatal("a handed account is in something the vault said or wrote in the clear") + } +} + +// A core module's handed account is the one the controller's core get is answered, sealed to its machine's key. +func TestAHandedCoreAccountIsKeptAsTheModulesOwnSecret(t *testing.T) { + const natsBroker = "own/anchor/nats/broker" + v := newVault(t, nil) + if _, err := handIssued(t, v, ControllerCaller, natsBroker, `{"password":"core-pw"}`); err != nil { + t.Fatal(err) + } + rec, held, _ := v.Store.Read(natsBroker) + if !held || rec.Origin != OriginIssued { + t.Fatalf("the core's account is held %v as %q", held, rec.Origin) + } + if !Covered(natsBroker) || Covered("own/anchor/nats/token") || Covered("pair/a/b/c/broker") { + t.Error("the covered names are not the module's broker and provisioner-broker own secrets") + } +} diff --git a/modules/mesh-vault/cmd/mesh-vault/keys.go b/modules/mesh-vault/cmd/mesh-vault/keys.go new file mode 100644 index 00000000..033d324d --- /dev/null +++ b/modules/mesh-vault/cmd/mesh-vault/keys.go @@ -0,0 +1,168 @@ +package main + +// The keys the vault meets, and the one it holds (novox/hq ADR 0113 §2, ADR 0206). +// +// - Its own key: an X25519 pair made once in its state directory, the private half never leaving it. Every +// secret is kept sealed to it at rest. +// - An asker's key: the public half only, as SPKI PEM — the form of the ADR 0206 box (mesh-sdk go/seal). +// A raw 32-byte key is refused, because a raw private key looks the same and the vault takes none. +// - The operator key: the public half, taken as SPKI PEM only and kept raw in base64, the form `operator key +// make` writes and the NaCl box the operator opens uses. +// +// No private key is ever accepted: not as a key to seal to, not as a secret to hold. + +import ( + "crypto/ecdh" + "crypto/rand" + "crypto/x509" + "encoding/base64" + "encoding/json" + "encoding/pem" + "errors" + "fmt" + "os" + "path/filepath" + "regexp" + "strings" + + "git.novox.be/novox/mesh-sdk/go/seal" + "golang.org/x/crypto/nacl/box" +) + +// OwnKeyFile is where the vault's own key lives in its state directory. +const OwnKeyFile = "vault-key.json" + +// OwnKey is the vault's own key pair, made once in dir. Written whole and synced, then linked into place and +// the directory synced, so a second process reads the first's key and never half of it. +// +// **A key is made only for an empty store.** When the key file is missing and the store holds secrets, the +// key they are sealed to is lost or misplaced, and a new key would open none of them: the vault refuses to +// start, loudly, rather than seal new secrets to a key beside old ones nothing opens. The operator copies +// are what re-raise it (ADR 0113 §21). +func OwnKey(dir string, store *Store) (seal.KeyPair, error) { + if dir == "" { + return seal.KeyPair{}, errors.New("the vault has no state directory: MESH_VAULT_STATE is not set") + } + if err := os.MkdirAll(dir, 0o700); err != nil { + return seal.KeyPair{}, err + } + file := filepath.Join(dir, OwnKeyFile) + if _, err := os.Stat(file); errors.Is(err, os.ErrNotExist) { + if store != nil { + empty, err := store.Empty() + if err != nil { + return seal.KeyPair{}, fmt.Errorf("the vault's key is missing and its store cannot be read: %w", err) + } + if !empty { + return seal.KeyPair{}, fmt.Errorf("THE VAULT'S KEY IS MISSING (%s) WHILE ITS STORE HOLDS SECRETS SEALED TO IT: "+ + "the vault does not start and makes no new key, which would open none of them — put the key back, "+ + "or re-raise the vault from the operator copies (novox/hq ADR 0113 §21)", file) + } + } + k, err := seal.GenerateKeyPair() + if err != nil { + return seal.KeyPair{}, err + } + raw, _ := json.Marshal(k) + tmp := filepath.Join(dir, fmt.Sprintf(".vault-key.%d.json", os.Getpid())) + if err := writeSynced(tmp, raw); err != nil { + return seal.KeyPair{}, err + } + _ = os.Link(tmp, file) // fails when another made it first, which is right + _ = os.Remove(tmp) + if err := syncDir(dir); err != nil { + return seal.KeyPair{}, err + } + } + raw, err := os.ReadFile(file) + if err != nil { + return seal.KeyPair{}, err + } + var k seal.KeyPair + if err := json.Unmarshal(raw, &k); err != nil || k.PublicKey == "" || k.PrivateKey == "" { + return seal.KeyPair{}, fmt.Errorf("%s is not the vault's key", file) + } + return k, nil +} + +var privateKeyPEM = regexp.MustCompile(`-----BEGIN [A-Z0-9 ]*PRIVATE KEY-----`) + +// privateKeyIn says whether text carries a private key. +func privateKeyIn(text string) bool { + return privateKeyPEM.MatchString(text) || strings.Contains(text, "PRIVATE KEY") +} + +// askerKey is an asker's public key, checked and as PEM: SPKI, X25519, and never a private key. +func askerKey(key string) (string, error) { + if privateKeyIn(key) { + return "", errors.New("that is a private key, and a private key is never sent to the vault (ADR 0113 §2): send the public half") + } + block, rest := pem.Decode([]byte(strings.TrimSpace(key))) + if block == nil || block.Type != "PUBLIC KEY" || strings.TrimSpace(string(rest)) != "" { + return "", errors.New("the key to seal to is one X25519 public key as SPKI PEM (`-----BEGIN PUBLIC KEY-----`), the form of the sealed box (ADR 0206)") + } + k, err := x509.ParsePKIXPublicKey(block.Bytes) + if err != nil { + return "", fmt.Errorf("the key to seal to is not a public key: %w", err) + } + pub, ok := k.(*ecdh.PublicKey) + if !ok || pub.Curve() != ecdh.X25519() { + return "", errors.New("the key to seal to is not an X25519 key") + } + return string(pem.EncodeToMemory(block)), nil +} + +// checkOperatorKey takes the operator's public key as SPKI PEM — never a private key, and never a raw key, +// which cannot be told from a private one — and gives it in the raw base64 form the operator's NaCl box and +// `secret recover` use. +func checkOperatorKey(key string) (string, error) { + if privateKeyIn(key) { + return "", errors.New("that is a private key: the operator key's private half never enters the mesh (ADR 0113 §2)") + } + block, rest := pem.Decode([]byte(strings.TrimSpace(key))) + if block == nil || block.Type != "PUBLIC KEY" || strings.TrimSpace(string(rest)) != "" { + return "", errors.New("the operator key is one X25519 public key as SPKI PEM (`-----BEGIN PUBLIC KEY-----`); " + + "a raw key is refused, because a raw private key looks the same") + } + k, err := x509.ParsePKIXPublicKey(block.Bytes) + if err != nil { + return "", fmt.Errorf("the operator key is not a public key: %w", err) + } + pub, ok := k.(*ecdh.PublicKey) + if !ok || pub.Curve() != ecdh.X25519() { + return "", errors.New("the operator key is not an X25519 key") + } + return base64.StdEncoding.EncodeToString(pub.Bytes()), nil +} + +// publicPEM wraps a raw X25519 public key, base64, as SPKI PEM. +func publicPEM(raw string) (string, error) { + b, err := base64.StdEncoding.DecodeString(strings.TrimSpace(raw)) + if err != nil || len(b) != 32 { + return "", errors.New("not a raw X25519 key in base64") + } + pub, err := ecdh.X25519().NewPublicKey(b) + if err != nil { + return "", err + } + der, err := x509.MarshalPKIXPublicKey(pub) + if err != nil { + return "", err + } + return string(pem.EncodeToMemory(&pem.Block{Type: "PUBLIC KEY", Bytes: der})), nil +} + +// operatorSeal seals value to the operator key in the box the operator opens today. +func operatorSeal(key string, value []byte) (string, error) { + raw, err := base64.StdEncoding.DecodeString(key) + if err != nil || len(raw) != 32 { + return "", errors.New("the operator key is not usable") + } + var pub [32]byte + copy(pub[:], raw) + sealed, err := box.SealAnonymous(nil, value, &pub, rand.Reader) + if err != nil { + return "", err + } + return base64.StdEncoding.EncodeToString(sealed), nil +} diff --git a/modules/mesh-vault/cmd/mesh-vault/ledger.go b/modules/mesh-vault/cmd/mesh-vault/ledger.go new file mode 100644 index 00000000..17465276 --- /dev/null +++ b/modules/mesh-vault/cmd/mesh-vault/ledger.go @@ -0,0 +1,251 @@ +package main + +// The custody ledger (novox/hq ADR 0085, design 24), carried over from the TypeScript vault: for each pair +// credential the controller delivers to this vault as the `secret` provider, who holds it, since when, its +// fingerprint and every rotation — and never the value. The controller still mints and seals those until +// the secrets move to the vault (design 57's V3, V4); the pass records what it delivered, so +// `secret_holders` and `secret_verify` answer as before. Its files are the same JSON, read as they are. +// +// **A holder the mesh stops asking for is retired, never deleted by a pass** (novox/hq ADR 0230). The vault +// is a mark-only provider: it has no access to switch off, so retiring writes the mark — when and why — into +// the holder's own ledger file, which keeps its rotation history. A holder is retired only once the same set +// has gone unasked in five consecutive passes that read the contributions file and for ten minutes; a set +// over the bound (more than three, or more than half of those held) waits for a person; asked for again, a +// retired holder is re-enabled at once; a holder is deleted only by `provisioner_delete`, a person's act +// with a why. Because the mark is in the ledger file, a restart forgets nothing. + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "regexp" + "sort" + "strings" + "time" +) + +// isoMillis is JavaScript's toISOString, the form the ledger's times were first written in. +const isoMillis = "2006-01-02T15:04:05.000Z" + +// Held is one holder of a secret this vault provides: everything the vault knows of it, and no value. +type Held struct { + As string `json:"as"` + Consumer string `json:"consumer"` + Fingerprint string `json:"fingerprint"` + Length int `json:"length"` + Since string `json:"since"` + Changed string `json:"changed"` + Rotations int `json:"rotations"` + History []Earlier `json:"history"` + // Retired is the mark a retirement writes; nil while the holder is active. + Retired *RetiredMark `json:"retired,omitempty"` +} + +// RetiredMark is when and why a holder was retired. +type RetiredMark struct { + At string `json:"at"` + Why string `json:"why"` +} + +// Earlier is a fingerprint a rotation replaced, and when. +type Earlier struct { + Fingerprint string `json:"fingerprint"` + Until string `json:"until"` +} + +// Outcome is what recording a delivery found. +type Outcome string + +const ( + Granted Outcome = "granted" + Rotated Outcome = "rotated" + Unchanged Outcome = "unchanged" +) + +// Fingerprint is sha256 of a value, as `sha256:`: the one thing about a secret that may be said. +func Fingerprint(value string) string { + sum := sha256.Sum256([]byte(value)) + return "sha256:" + hex.EncodeToString(sum[:]) +} + +// Ledger is the directory of holders. +type Ledger struct{ dir string } + +// NewLedger opens (and makes) the ledger at dir. +func NewLedger(dir string) (*Ledger, error) { + if dir == "" { + return nil, errors.New("MESH_VAULT_LEDGER is not set: the vault has nowhere to keep its ledger") + } + if err := os.MkdirAll(dir, 0o700); err != nil { + return nil, err + } + return &Ledger{dir: dir}, nil +} + +var login = regexp.MustCompile(`^[a-z0-9][a-z0-9_.-]*$`) + +func (l *Ledger) pathOf(as string) (string, error) { + if !login.MatchString(as) || strings.Contains(as, "..") { + return "", fmt.Errorf("a login is a name, not a path: %q", as) + } + return filepath.Join(l.dir, as+".json"), nil +} + +// Record says the mesh delivered value for as: the same value again changes nothing, another is a rotation. +// The value is fingerprinted here and goes no further. A retirement mark is left as it is. +func (l *Ledger) Record(as, consumer, value string, now time.Time) (Held, Outcome, error) { + path, err := l.pathOf(as) + if err != nil { + return Held{}, "", err + } + fp, at := Fingerprint(value), now.UTC().Format(isoMillis) + before, ok := l.Get(as) + if !ok { + h := Held{As: as, Consumer: consumer, Fingerprint: fp, Length: len(value), Since: at, Changed: at, History: []Earlier{}} + return h, Granted, l.write(path, h) + } + if before.Fingerprint == fp && before.Length == len(value) { + return before, Unchanged, nil + } + h := before + h.Consumer, h.Fingerprint, h.Length, h.Changed = consumer, fp, len(value), at + h.Rotations++ + h.History = append(append([]Earlier{}, before.History...), Earlier{Fingerprint: before.Fingerprint, Until: at}) + return h, Rotated, l.write(path, h) +} + +// Mark writes or clears a holder's retirement mark. +func (l *Ledger) Mark(as string, mark *RetiredMark) error { + path, err := l.pathOf(as) + if err != nil { + return err + } + h, ok := l.Get(as) + if !ok { + return fmt.Errorf("the ledger holds nothing for %s", as) + } + h.Retired = mark + return l.write(path, h) +} + +// Delete forgets a holder, saying whether there was one. Only a person's `provisioner_delete` reaches it. +func (l *Ledger) Delete(as string) bool { + path, err := l.pathOf(as) + if err != nil { + return false + } + if os.Remove(path) != nil { + return false + } + _ = syncDir(l.dir) + return true +} + +// Get is one holder. +func (l *Ledger) Get(as string) (Held, bool) { + path, err := l.pathOf(as) + if err != nil { + return Held{}, false + } + raw, err := os.ReadFile(path) + if err != nil { + return Held{}, false + } + var h Held + if json.Unmarshal(raw, &h) != nil { + return Held{}, false + } + return h, true +} + +// List is every holder, by login. +func (l *Ledger) List() []Held { + entries, err := os.ReadDir(l.dir) + if err != nil { + return nil + } + var out []Held + for _, e := range entries { + if strings.HasSuffix(e.Name(), ".json") { + if h, ok := l.Get(strings.TrimSuffix(e.Name(), ".json")); ok { + out = append(out, h) + } + } + } + sort.Slice(out, func(i, j int) bool { return out[i].As < out[j].As }) + return out +} + +func (l *Ledger) write(path string, h Held) error { + raw, err := json.MarshalIndent(h, "", " ") + if err != nil { + return err + } + return writeWhole(path, append(raw, '\n')) +} + +// Contribution is one entry of the contributions file the mesh delivers to the vault. +type Contribution struct { + From string `json:"from,omitempty"` + Node string `json:"node,omitempty"` + As string `json:"as"` + Secret string `json:"secret"` +} + +// Contributions is what the mesh asks this vault to serve: the entries with a login and a secret file. +// +// **A file that does not say who is asked for is not "nobody".** One that is unreadable, not JSON, without a +// `given` list (missing or null), or written for another requirement than `secret` is refused, so a pass +// over it retires nobody. An empty `given` list is an answer: nobody is asked for. +func Contributions(receives string) ([]Contribution, error) { + raw, err := os.ReadFile(receives) + if err != nil { + return nil, err + } + var doc struct { + Requirement *string `json:"requirement"` + Given json.RawMessage `json:"given"` + } + if err := json.Unmarshal(raw, &doc); err != nil { + return nil, fmt.Errorf("the contributions file is not JSON: %w", err) + } + if doc.Requirement != nil && *doc.Requirement != "secret" { + return nil, fmt.Errorf("the contributions file is for %q, not for secret", *doc.Requirement) + } + if len(doc.Given) == 0 || string(doc.Given) == "null" { + return nil, errors.New("the contributions file has no `given` list, so it says nothing of who is asked for") + } + var given []Contribution + if err := json.Unmarshal(doc.Given, &given); err != nil { + return nil, fmt.Errorf("the contributions file's `given` is not a list of contributions: %w", err) + } + var out []Contribution + for _, g := range given { + if g.As != "" && g.Secret != "" { + out = append(out, g) + } + } + return out, nil +} + +// delivered is the value the mesh delivered for c, without the newline a file may end with. +func delivered(c Contribution) (string, error) { + raw, err := os.ReadFile(c.Secret) + if err != nil { + return "", fmt.Errorf("the delivered secret is not readable: %v", err) + } + return strings.TrimSuffix(string(raw), "\n"), nil +} + +// DeliveredFingerprint is the fingerprint and length of what the mesh currently delivers for c. +func DeliveredFingerprint(c Contribution) (string, int, error) { + v, err := delivered(c) + if err != nil { + return "", 0, err + } + return Fingerprint(v), len(v), nil +} diff --git a/modules/mesh-vault/cmd/mesh-vault/ledger_test.go b/modules/mesh-vault/cmd/mesh-vault/ledger_test.go new file mode 100644 index 00000000..74aaa70e --- /dev/null +++ b/modules/mesh-vault/cmd/mesh-vault/ledger_test.go @@ -0,0 +1,129 @@ +package main + +// The custody ledger the TypeScript vault kept (its client.ts and test/ledger.test.ts), carried over whole: +// who holds a secret the controller delivered for this vault, since when, and its fingerprint — never the +// value. Its files are the same JSON, so the ledger on the vault's disk is read as it is. + +import ( + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + "time" +) + +func TestAFirstDeliveryIsAGrantTheSameAgainNothingANewValueARotation(t *testing.T) { + l, _ := NewLedger(t.TempDir()) + t0 := time.Date(2026, 9, 20, 10, 0, 0, 0, time.UTC) + t1 := time.Date(2026, 9, 21, 10, 0, 0, 0, time.UTC) + + held, outcome, err := l.Record("anchor-redis", "anchor", "first-value", t0) + if err != nil || outcome != Granted || held.Rotations != 0 || held.Since != t0.Format(isoMillis) || + held.Fingerprint != Fingerprint("first-value") { + t.Fatalf("%+v %s %v", held, outcome, err) + } + if _, outcome, _ := l.Record("anchor-redis", "anchor", "first-value", t1); outcome != Unchanged { + t.Errorf("the same value again is %s", outcome) + } + held, outcome, _ = l.Record("anchor-redis", "anchor", "second-value", t1) + if outcome != Rotated || held.Rotations != 1 || held.Since != t0.Format(isoMillis) || held.Changed != t1.Format(isoMillis) { + t.Errorf("%+v %s", held, outcome) + } + if len(held.History) != 1 || held.History[0].Fingerprint != Fingerprint("first-value") || held.History[0].Until != t1.Format(isoMillis) { + t.Errorf("history %+v", held.History) + } +} + +func TestTheLedgerHoldsFingerprintsNeverTheValue(t *testing.T) { + dir := t.TempDir() + l, _ := NewLedger(dir) + _, _, _ = l.Record("anchor-redis", "anchor", "the-actual-password", time.Now()) + _, _, _ = l.Record("anchor-redis", "anchor", "the-rotated-password", time.Now()) + entries, _ := os.ReadDir(dir) + for _, e := range entries { + raw, _ := os.ReadFile(filepath.Join(dir, e.Name())) + if strings.Contains(string(raw), "the-actual-password") || strings.Contains(string(raw), "the-rotated-password") { + t.Errorf("%s holds a value", e.Name()) + } + info, _ := e.Info() + if info.Mode().Perm() != 0o600 { + t.Errorf("%s is %v", e.Name(), info.Mode()) + } + } +} + +func TestDeletingForgetsAHolderAndListingIsByLogin(t *testing.T) { + l, _ := NewLedger(t.TempDir()) + _, _, _ = l.Record("b-app", "b", "x", time.Now()) + _, _, _ = l.Record("a-app", "a", "y", time.Now()) + if got := logins(l.List()); got != "a-app,b-app" { + t.Errorf("listed %s", got) + } + if !l.Delete("a-app") || l.Delete("a-app") { + t.Error("deleting found the wrong thing") + } + if got := logins(l.List()); got != "b-app" { + t.Errorf("listed %s", got) + } +} + +func TestALoginIsANameNotAPath(t *testing.T) { + l, _ := NewLedger(t.TempDir()) + if _, _, err := l.Record("../etc/passwd", "n", "v", time.Now()); err == nil { + t.Error("a path was taken for a login") + } +} + +// A ledger the TypeScript vault wrote is read as it is. +func TestALedgerTheTypeScriptVaultWroteIsRead(t *testing.T) { + dir := t.TempDir() + written := `{ + "as": "anchor-redis", + "consumer": "anchor", + "fingerprint": "sha256:abc", + "length": 40, + "since": "2026-09-20T10:00:00.000Z", + "changed": "2026-09-21T10:00:00.000Z", + "rotations": 1, + "history": [{"fingerprint": "sha256:old", "until": "2026-09-21T10:00:00.000Z"}] +} +` + _ = os.WriteFile(filepath.Join(dir, "anchor-redis.json"), []byte(written), 0o600) + l, _ := NewLedger(dir) + h, ok := l.Get("anchor-redis") + if !ok || h.Rotations != 1 || h.Length != 40 || h.History[0].Fingerprint != "sha256:old" { + t.Fatalf("%+v %v", h, ok) + } +} + +func TestWhatTheMeshDeliversIsFingerprintedNeverReturned(t *testing.T) { + dir := t.TempDir() + secret := filepath.Join(dir, "anchor.redis.secret") + _ = os.WriteFile(secret, []byte("minted-value\n"), 0o600) + receives := filepath.Join(dir, "mesh.json") + doc, _ := json.Marshal(map[string]any{"requirement": "secret", "given": []map[string]string{ + {"from": "redis", "node": "anchor", "as": "anchor-redis", "secret": secret}, + {"from": "offer-only", "node": "anchor"}, + }}) + _ = os.WriteFile(receives, doc, 0o600) + asked, err := Contributions(receives) + if err != nil || len(asked) != 1 || asked[0].As != "anchor-redis" { + t.Fatalf("%+v %v", asked, err) + } + fp, length, err := DeliveredFingerprint(asked[0]) + if err != nil || fp != Fingerprint("minted-value") || length != len("minted-value") { + t.Errorf("%s %d %v", fp, length, err) + } + if _, _, err := DeliveredFingerprint(Contribution{As: "x", Secret: filepath.Join(dir, "missing")}); err == nil || !strings.Contains(err.Error(), "not readable") { + t.Errorf("a missing file said %v", err) + } +} + +func logins(hs []Held) string { + var out []string + for _, h := range hs { + out = append(out, h.As) + } + return strings.Join(out, ",") +} diff --git a/modules/mesh-vault/cmd/mesh-vault/main.go b/modules/mesh-vault/cmd/mesh-vault/main.go new file mode 100644 index 00000000..d8f09abd --- /dev/null +++ b/modules/mesh-vault/cmd/mesh-vault/main.go @@ -0,0 +1,118 @@ +// mesh-vault (novox/hq ADR 0085, ADR 0113, ADR 0349 §7 and §11): one binary, launched by its node's runtime +// and speaking MCP to it over stdio through the Go SDK. It holds the `mesh-vault` seat: it keeps every +// secret in its own store, sealed at rest and with a copy sealed to the operator key; it answers the seat's +// verb `get` with a secret sealed to the asker's key; it raises `secret-changed` without a value. Beside +// that it keeps the custody ledger of the pair credentials the controller still delivers to it. +// +// stdout is the MCP channel; everything this module says, it says on stderr. +package main + +import ( + "errors" + "fmt" + "os" + "time" + + stdio "git.novox.be/novox/mesh-sdk/go" +) + +func say(format string, args ...any) { + fmt.Fprintf(os.Stderr, "[mesh-vault] "+format+"\n", args...) +} + +func emit(event string, body map[string]any) error { return stdio.Emit(event, body) } + +// build makes the vault from the module's words: directories, never values. +func build() *Env { + env := &Env{Receives: os.Getenv("MESH_RECEIVES"), Root: os.Getenv("MESH_VAULT_ROOT")} + if l, err := NewLedger(os.Getenv("MESH_VAULT_LEDGER")); err == nil { + env.Ledger = l + } else { + say("%v", err) + } + if env.Ledger != nil && env.Receives != "" { + env.Custody = &Custody{Ledger: env.Ledger, Receives: env.Receives, Resource: "secret", Node: os.Getenv("MESH_NODE"), + Emit: emit, Now: time.Now, Log: say} + } + store, err := OpenStore(os.Getenv("MESH_VAULT_STORE")) + if err != nil { + env.Broken = fmt.Errorf("the vault could not open its store: %w", err) + say("%v", env.Broken) + return env + } + own, err := OwnKey(os.Getenv("MESH_VAULT_STATE"), store) + if err != nil { + env.Broken = fmt.Errorf("the vault could not open its key: %w", err) + say("%v", env.Broken) + return env + } + // Who may ask (askers.go): a node's provisioner client, for what its module's plan there names. + // Where it sees a consumer bind a provider (pair.go): the same mirror, every key. + env.Vault = &Vault{Store: store, Own: own, Askers: Declared{Read: readAssignment, Store: store}, + Assignments: Mirror{Keys: assignmentKeys, Read: readAssignment}, Emit: emit, Ask: stdio.Ask, + Now: time.Now, Log: say} + return env +} + +// adoptOperatorKey takes the operator key from the export the mesh writes to the vault's root (what +// `operator key set` recorded; the controller keeps only a public half there), and seals again any +// operator copy not yet sealed to the current key. Run every minute, so a failure partway is finished. It +// returns why the pass did not re-seal everything, or nil. +func adoptOperatorKey(env *Env) error { + if env.Vault == nil { + if env.Broken != nil { + return env.Broken + } + return errors.New("the vault was not built") + } + if raw := exportedOperatorKey(env.Root); env.Root != "" && raw != "" { + key, err := publicPEM(raw) + if err == nil { + _, err = env.Vault.SetOperatorKey(key, "the mesh's export") + } + env.refuseKey("") + if err != nil { + env.refuseKey(err.Error()) + say("the operator key in the mesh's export is not taken, or not every copy is sealed to it yet: %v", err) + } + return err + } + _, err := env.Vault.Reseal() + if err != nil { + say("%v", err) + } + return err +} + +// readAssignment is one value of the controller's assignments mirror, read through the runtime under the vault's +// own account (its manifest `reads` it), or nil when there is none. +func readAssignment(key string) ([]byte, error) { + e, err := stdio.State(AssignmentsRead).Get(key) + if err != nil || e == nil { + return nil, err + } + return e.Value, nil +} + +// assignmentKeys is every key of the controller's assignments mirror, read the same way. +func assignmentKeys() ([]string, error) { return stdio.State(AssignmentsRead).Keys() } + +func main() { + env := build() + go func() { + tick := time.NewTicker(time.Minute) + defer tick.Stop() + for { + env.pass() + env.pairPass() + if env.Custody != nil { + env.Custody.Pass() + } + <-tick.C + } + }() + if err := stdio.Serve("", tools(env)); err != nil { + say("%v", err) + os.Exit(1) + } +} diff --git a/modules/mesh-vault/cmd/mesh-vault/move.go b/modules/mesh-vault/cmd/mesh-vault/move.go new file mode 100644 index 00000000..8db5f853 --- /dev/null +++ b/modules/mesh-vault/cmd/mesh-vault/move.go @@ -0,0 +1,369 @@ +package main + +// The move of the controller's secrets into the vault (novox/hq ADR 0349 clause 7; design 57's V3, issue 676; +// the operator's decision of 2026-10-14 on that issue). +// +// **Only a machine the value was sealed to can open it.** The controller's store holds every value sealed to a +// machine's sealing key (and, for some, to the operator key); the controller cannot open one. So each machine's +// node-engine opens what was sealed to it and hands it in, in the two steps of the intake (intake.go): `move-in` +// names the secret, where it came from and its generation in the controller's store, and is answered a call and +// a key made for it; the second step carries the value sealed to that key and a key of the node-engine's own, +// and is answered what the vault kept, read back from its own store and sealed to that key — the node-engine +// opens it and compares it with what it handed in, which is the move's proof. +// +// **From the node-engine of a machine the value was sealed to, and no other caller.** The caller is the bus user +// the runtime proved, `node.` (the controller's broker names a node-engine so): a module's own secret +// was sealed to its one machine, a pair credential to both its ends. +// +// **Copied as it is, and never over a value the vault holds.** Nothing is rotated (ADR 0349's rule for a move): +// the value keeps its origin, the controller's generation and when it was made. The same value again changes +// nothing — the other end of a pair, or the move run again — and a different one is refused, so a value the +// vault holds, rotated or given since, is never put back to the controller's older one. +// +// The controller's record of asks at a desk holds no value, and the controller copies it as it is (`move-asks`). +// +// Which secret each row of the controller's store is, by name in the vault's store: +// +// own/// a row of module_secret +// pair////[/] a row of secret, its local name last when it has one + +import ( + "crypto/rand" + "encoding/base64" + "errors" + "fmt" + "slices" + "strconv" + "strings" + "time" + + stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/seal" + "golang.org/x/crypto/nacl/box" +) + +// NodeEnginePrefix starts the bus user of a machine's node-engine: `node.`. +const NodeEnginePrefix = "node." + +// MovedFrom is where a value the move took came from: the node-engine that handed it in, and the generation the +// controller's store stamped it with (ADR 0158, ADR 0305 §5), kept as it was. Never a value. +type MovedFrom struct { + By string `json:"by"` + At time.Time `json:"at"` + Generation string `json:"generation,omitempty"` +} + +// MoveOf is what one row of the controller's store is: a module's own secret (node, module, name) or a pair +// credential (provider, provision, consumer, consumer module, local name), where it came from, its generation +// and when it was made. +type MoveOf struct { + Node, Module, Name string + Provider, Provision, Consumer, ConsumerModule, Local string + Origin, Generation string + MadeAt time.Time + // SealingKey is the caller's machine's sealing key as its node-engine states it (a raw X25519 public key in + // base64), kept as that machine's: what the core's credentials are sealed to (core.go, issue 763). Optional. + SealingKey string +} + +// MovedIn is what the vault kept, and that value read back from its store sealed to the node-engine's key. +type MovedIn struct { + Name string `json:"name"` + Generation int `json:"generation"` + Unchanged bool `json:"unchanged,omitempty"` + // ReadBack is the value the vault's store holds now, sealed to the key the second step sent (NaCl anonymous + // box, base64). + ReadBack string `json:"read_back"` +} + +// PairSecret is the vault's name for one row of the controller's pair credentials. +func PairSecret(provider, provision, consumer, consumerModule, local string) (string, error) { + parts := map[string]string{"the provider's machine": provider, "the provision": provision, + "the consumer's machine": consumer, "the consuming module": consumerModule} + if local != "" { + parts["the local name"] = local + } + for what, v := range parts { + if !namePart.MatchString(v) { + return "", fmt.Errorf("%s %q is not a name the mesh writes", what, v) + } + } + name := "pair/" + provider + "/" + provision + "/" + consumer + "/" + consumerModule + if local != "" { + name += "/" + local + } + return name, nil +} + +// secret is the row's name in the vault and the machines it was sealed to. +func (m MoveOf) secret() (string, []string, error) { + own := m.Node != "" || m.Module != "" || m.Name != "" + pair := m.Provider != "" || m.Provision != "" || m.Consumer != "" || m.ConsumerModule != "" || m.Local != "" + switch { + case own && !pair: + name, err := OwnSecret(m.Node, m.Module, m.Name) + return name, []string{m.Node}, err + case pair && !own: + name, err := PairSecret(m.Provider, m.Provision, m.Consumer, m.ConsumerModule, m.Local) + return name, []string{m.Provider, m.Consumer}, err + } + return "", nil, errors.New("name a module's own secret (node, module, name) or a pair credential (provider, " + + "provision, consumer, consumer_module and local when it has one), one of the two") +} + +// movedOrigins are the controller's words for where a value came from, as the vault's. +var movedOrigins = map[string]string{"made": OriginMade, "accepted": OriginAccepted, "issued": OriginIssued} + +// nodeEngineOf is the machine whose node-engine caller is, if it is one. +func nodeEngineOf(caller string) (string, bool) { + node, ok := strings.CutPrefix(caller, NodeEnginePrefix) + return node, ok && namePart.MatchString(node) && !strings.Contains(node, ".") +} + +// MoveOpen opens a call to take one value of the controller's store from the node-engine of a machine it was +// sealed to. +func (v *Vault) MoveOpen(caller string, m MoveOf) (Opened, error) { + node, ok := nodeEngineOf(caller) + if !ok { + if caller == "" { + caller = "an unproven caller" + } + return Opened{}, fmt.Errorf("move-in takes a value from a machine's node-engine alone, which opens what was "+ + "sealed to its machine; %s is not a node-engine", caller) + } + secret, sealedTo, err := m.secret() + if err != nil { + return Opened{}, err + } + if !slices.Contains(sealedTo, node) { + return Opened{}, fmt.Errorf("%s was sealed to %s, not to %s: only a machine it was sealed to moves it in", + secret, strings.Join(sealedTo, " and "), node) + } + if movedOrigins[m.Origin] == "" { + return Opened{}, fmt.Errorf("the origin %q is not one the controller's store holds (made, accepted or "+ + "issued); nothing is taken", m.Origin) + } + if len(m.Generation) > 120 || strings.ContainsAny(m.Generation, "\r\n\x00") { + return Opened{}, errors.New("the generation is not one short line; nothing is taken") + } + if m.SealingKey != "" { + stated, err := sealingKey(m.SealingKey) + if err != nil { + return Opened{}, err + } + if err := v.recordNodeKey(node, stated, caller); err != nil { + return Opened{}, err + } + } + id, key, err := v.openCall("move-in", secret, caller, nil) + if err != nil { + return Opened{}, err + } + v.callsMu.Lock() + if c := v.pending[id]; c != nil { + moved := m + c.move = &moved + } + v.callsMu.Unlock() + return Opened{Call: id, SealTo: key, Secret: secret}, nil +} + +// readBackKey is the key a read-back is sealed to: a raw X25519 public key in base64, the box the node-engine's +// own sealing key opens. A PEM, which a private key could be, is refused. +func readBackKey(key string) (*[32]byte, error) { + if privateKeyIn(key) || strings.Contains(key, "-----") { + return nil, errors.New("the key to read back to is a raw X25519 public key in base64, never a PEM and never " + + "a private key; nothing was taken") + } + raw, err := base64.StdEncoding.DecodeString(strings.TrimSpace(key)) + if err != nil || len(raw) != 32 { + return nil, errors.New("the second step names no key to read back to (`read_back_to`, a raw X25519 public " + + "key in base64); nothing was taken") + } + var pub [32]byte + copy(pub[:], raw) + return &pub, nil +} + +// MoveTake opens the value sealed to the call's key and keeps it as it is, unless the vault holds another; then +// answers what the store holds, read back and sealed to readBackTo. +func (v *Vault) MoveTake(caller, call, sealed, readBackTo string) (MovedIn, error) { + if _, ok := nodeEngineOf(caller); !ok { + return MovedIn{}, fmt.Errorf("move-in takes a value from a machine's node-engine alone; %q is not one", caller) + } + to, err := readBackKey(readBackTo) + if err != nil { + return MovedIn{}, err + } + c, err := v.takeCall("move-in", call) + if err != nil { + return MovedIn{}, err + } + if c.by != caller || c.move == nil { + return MovedIn{}, fmt.Errorf("the move-in call %q was opened by another caller; nothing was taken", call) + } + value, err := c.open(sealed) + if err != nil { + return MovedIn{}, err + } + defer zero(value) + if len(value) == 0 { + return MovedIn{}, errors.New("the value handed in is empty; nothing was taken") + } + m := c.move + changed, err := v.keepWith(c.secret, string(value), caller, "", movedOrigins[m.Origin], keepOptions{ + neverReplace: true, madeAt: m.MadeAt, movedFrom: &MovedFrom{By: caller, Generation: m.Generation}}) + if err != nil { + return MovedIn{}, err + } + rec, held, err := v.Store.Read(c.secret) + if err != nil || !held { + return MovedIn{}, fmt.Errorf("%s was kept but cannot be read back from the vault's store: %v", c.secret, err) + } + back, err := seal.Open(rec.AtRest, v.Own.PrivateKey) + if err != nil { + return MovedIn{}, fmt.Errorf("%s was kept but does not open with the vault's key: %w", c.secret, err) + } + boxed, err := box.SealAnonymous(nil, []byte(back), to, rand.Reader) + back = "" + if err != nil { + return MovedIn{}, err + } + v.log("moved in %s (generation %d) from %s, origin %s, the controller's generation %q", c.secret, + changed.Generation, caller, movedOrigins[m.Origin], m.Generation) + return MovedIn{Name: c.secret, Generation: rec.Generation, Unchanged: changed.Unchanged, + ReadBack: base64.StdEncoding.EncodeToString(boxed)}, nil +} + +// ---- move-asks ---------------------------------------------------------------------------------------------- + +// MovedAsk is one row of the controller's record of asks at a desk (its secret_ask table): no value. +type MovedAsk struct { + ID int64 `json:"id"` + Node string `json:"node"` + Module string `json:"module"` + Name string `json:"name"` + Desk string `json:"desk"` + AskedBy string `json:"asked_by"` + OpenedAt time.Time `json:"opened_at"` + EndedAt *time.Time `json:"ended_at,omitempty"` + Outcome string `json:"outcome,omitempty"` +} + +// AsksMoved is what a copy of the record of asks did. +type AsksMoved struct { + Copied int `json:"copied"` + Already int `json:"already"` +} + +// MovedAsksAtOnce bounds one call: the controller sends its record in parts. +const MovedAsksAtOnce = 500 + +// movedAskID is the vault's id of a row of the controller's record. +func movedAskID(id int64) string { return "controller-" + strconv.FormatInt(id, 10) } + +// MoveAsks copies rows of the controller's record of asks into the vault's, as they are: each ask's opening and, +// when it ended, its end — once, so a copy run again adds nothing. From the controller alone, whose record it is. +// Every row is judged before any is written. +func (v *Vault) MoveAsks(caller string, asks []MovedAsk) (AsksMoved, error) { + if err := fromTheController(caller, "move-asks"); err != nil { + return AsksMoved{}, err + } + if len(asks) > MovedAsksAtOnce { + return AsksMoved{}, fmt.Errorf("%d asks at once is more than %d; send them in parts", len(asks), MovedAsksAtOnce) + } + type lines struct{ opened, ended AskRecord } + var all []lines + for _, a := range asks { + secret, err := OwnSecret(a.Node, a.Module, a.Name) + if err != nil { + return AsksMoved{}, fmt.Errorf("ask %d: %w; nothing was copied", a.ID, err) + } + if !namePart.MatchString(a.Desk) || a.OpenedAt.IsZero() || a.ID <= 0 { + return AsksMoved{}, fmt.Errorf("ask %d names no desk, no time it opened or no id; nothing was copied", a.ID) + } + l := lines{opened: AskRecord{ID: movedAskID(a.ID), At: a.OpenedAt.UTC(), Event: AskOpened, Secret: secret, + Node: a.Node, Module: a.Module, Name: a.Name, Desk: a.Desk, AskedBy: askedByName(a.AskedBy), + Caller: ControllerCaller}} + if a.EndedAt != nil { + if a.Outcome != "given" && !askOutcomes[a.Outcome] { + return AsksMoved{}, fmt.Errorf("ask %d ended %q, which is no outcome the record knows; nothing was "+ + "copied", a.ID, a.Outcome) + } + l.ended = AskRecord{ID: movedAskID(a.ID), At: a.EndedAt.UTC(), Event: AskEnded, Secret: secret, + Outcome: a.Outcome} + } + all = append(all, l) + } + v.Store.asksMu.Lock() + defer v.Store.asksMu.Unlock() + held, err := v.Store.asks() + if err != nil { + return AsksMoved{}, fmt.Errorf("the record of asks cannot be read: %w; nothing was copied", err) + } + seen := map[string]bool{} + for _, r := range held { + if r.Event == AskOpened { + seen[r.ID] = true + } + } + out := AsksMoved{} + for _, l := range all { + if seen[l.opened.ID] { + out.Already++ + continue + } + if err := v.recordAsk(l.opened); err != nil { + return out, fmt.Errorf("ask %s could not be copied, after %d: %w", l.opened.ID, out.Copied, err) + } + if l.ended.ID != "" { + if err := v.recordAsk(l.ended); err != nil { + return out, fmt.Errorf("the end of ask %s could not be copied, after %d: %w", l.opened.ID, out.Copied, err) + } + } + seen[l.opened.ID] = true + out.Copied++ + } + return out, nil +} + +// StateNodeKey records the sealing key the node-engine of caller's machine states in its own name (novox/hq ADR 0372, +// issue 786), at its start and whenever the key changes, whether or not it ever moved a secret in. The machine is the +// caller's, as the bus proved it (`node.`), never an argument. A first key is handled like a changed one: the +// machine's core secrets, sealed to no key until now or to the old one, are said changed so the controller asks again +// (ADR 0349 §16). The same key again changes nothing and says nothing. +func (v *Vault) StateNodeKey(caller, key string) error { + node, ok := nodeEngineOf(caller) + if !ok { + if caller == "" || caller == stdio.Unproven { + caller = "an unproven caller" + } + return fmt.Errorf("node-key takes a machine's sealing key from that machine's node-engine alone, in its own "+ + "name; %s is not a node-engine", caller) + } + stated, err := sealingKey(key) + if err != nil { + return err + } + held, had, err := v.Store.NodeKey(node) + if err != nil { + return err + } + if had && held == stated { + return nil + } + if err := v.recordNodeKey(node, stated, caller); err != nil { + return err + } + all, err := v.Store.List() + if err != nil { + return fmt.Errorf("%s's key is recorded, but its core secrets could not be listed to say them changed: %w", node, err) + } + for _, r := range all { + parts := strings.Split(r.Name, "/") + if len(parts) == 4 && parts[0] == "own" && parts[1] == node && slices.Contains(CoreModules, parts[2]) { + v.announce(Changed{Name: r.Name, Generation: r.Generation, At: v.Now().UTC(), By: caller}) + } + } + return nil +} diff --git a/modules/mesh-vault/cmd/mesh-vault/move_test.go b/modules/mesh-vault/cmd/mesh-vault/move_test.go new file mode 100644 index 00000000..d2e41c1b --- /dev/null +++ b/modules/mesh-vault/cmd/mesh-vault/move_test.go @@ -0,0 +1,321 @@ +package main + +// The move of the controller's secrets into the vault (design 57's V3, issue 676, the operator's decision of +// 2026-10-14): each machine's node-engine opens what was sealed to it and hands it in, sealed to a key the vault +// made for the call, keeping where it came from and its generation; the vault takes that step only from the +// node-engine of a machine the value was sealed to, never replaces a value it holds, and answers what it kept +// read back from its own store, sealed to a key the node-engine sent. The controller copies its record of asks +// as it is. Nothing is rotated. + +import ( + "crypto/rand" + "encoding/base64" + "strings" + "testing" + "time" + + "golang.org/x/crypto/nacl/box" +) + +// aReadBackKey is a key the node-engine makes for one read-back: its public half to send, its private to open. +func aReadBackKey(t *testing.T) (string, *[32]byte, *[32]byte) { + t.Helper() + pub, priv, err := box.GenerateKey(rand.Reader) + if err != nil { + t.Fatal(err) + } + return base64.StdEncoding.EncodeToString(pub[:]), pub, priv +} + +// openReadBack opens what the vault read back, as the node-engine does. +func openReadBack(t *testing.T, sealed string, pub, priv *[32]byte) string { + t.Helper() + blob, err := base64.StdEncoding.DecodeString(sealed) + if err != nil { + t.Fatal(err) + } + out, ok := box.OpenAnonymous(nil, blob, pub, priv) + if !ok { + t.Fatal("the read-back does not open with the key sent for it") + } + return string(out) +} + +// moveIn is one node-engine's two steps for one value. +func moveIn(t *testing.T, v testVault, caller string, m MoveOf, value string) (MovedIn, error) { + t.Helper() + opened, err := v.MoveOpen(caller, m) + if err != nil { + return MovedIn{}, err + } + key, _, _ := aReadBackKey(t) + return v.MoveTake(caller, opened.Call, sealToCall(t, opened.SealTo, value), key) +} + +const movedValue = "Xq3-the-superusers-password-made-in-2025" + +func superuser() MoveOf { + return MoveOf{Node: "anchor", Module: "postgres", Name: "superuser-password", Origin: "made", + Generation: "g-2025-11-02T10:00:00Z-1", MadeAt: time.Date(2025, 11, 2, 10, 0, 0, 0, time.UTC)} +} + +// A machine's own value moves as it is: opened in the vault alone, kept with its origin, the controller's +// generation and when it was made, and read back from the vault's store to the node-engine, sealed to its key. +func TestANodeEngineMovesInAValueSealedToItsMachineAsItIs(t *testing.T) { + v := newVault(t, nil) + opened, err := v.MoveOpen("node.anchor", superuser()) + if err != nil { + t.Fatal(err) + } + if opened.Secret != "own/anchor/postgres/superuser-password" || opened.SealTo == "" || opened.Call == "" { + t.Fatalf("opened %+v", opened) + } + key, pub, priv := aReadBackKey(t) + moved, err := v.MoveTake("node.anchor", opened.Call, sealToCall(t, opened.SealTo, movedValue), key) + if err != nil { + t.Fatal(err) + } + if moved.Name != opened.Secret || moved.Generation != 1 || moved.Unchanged { + t.Errorf("moved %+v", moved) + } + if got := openReadBack(t, moved.ReadBack, pub, priv); got != movedValue { + t.Errorf("the read-back opens to %q", got) + } + rec, held, _ := v.Store.Read(opened.Secret) + if !held || rec.Origin != OriginMade || rec.ChangedBy != "node.anchor" || rec.MovedFrom == nil || + rec.MovedFrom.Generation != "g-2025-11-02T10:00:00Z-1" || rec.MovedFrom.By != "node.anchor" || + !rec.MadeAt.Equal(superuser().MadeAt) { + t.Errorf("the record: %+v moved from %+v", rec, rec.MovedFrom) + } + // Kept as every value is: a copy for the operator. + if got := openOperator(t, v.operatorPublic, v.operatorPrivate, rec.Operator.Sealed); got != movedValue { + t.Errorf("the operator copy opens to %q", got) + } + if strings.Contains(everything(t, v, opened, moved), movedValue) { + t.Fatal("the value is in something the vault said, answered or wrote in the clear") + } +} + +// Each origin the controller's store holds is kept as the vault's word for it; any other is refused. +func TestAMovedValueKeepsWhereItCameFrom(t *testing.T) { + v := newVault(t, nil) + for origin, want := range map[string]string{"made": OriginMade, "accepted": OriginAccepted, "issued": OriginIssued} { + m := superuser() + m.Name, m.Origin = "from-"+origin, origin + if _, err := moveIn(t, v, "node.anchor", m, "value-"+origin); err != nil { + t.Fatalf("%s: %v", origin, err) + } + rec, _, _ := v.Store.Read("own/anchor/postgres/from-" + origin) + if rec.Origin != want { + t.Errorf("%s was kept as %q", origin, rec.Origin) + } + } + m := superuser() + m.Origin = "invented" + if _, err := v.MoveOpen("node.anchor", m); err == nil || !strings.Contains(err.Error(), "origin") { + t.Errorf("an unknown origin: %v", err) + } +} + +// Only the node-engine of a machine the value was sealed to takes the step: not the controller, not a module's +// tools, not another machine's node-engine. +func TestOnlyTheNodeEngineOfAMachineTheValueWasSealedToMovesIt(t *testing.T) { + v := newVault(t, nil) + for _, caller := range []string{ControllerCaller, "anchor.node-tools", "anchor.mesh-vault", "node.shanks", "", + "node.", "node.anchor.x"} { + if _, err := v.MoveOpen(caller, superuser()); err == nil { + t.Errorf("%q opened a move of anchor's value", caller) + } + } + // A pair credential was sealed to both its ends: either may move it, and nobody else. + pair := MoveOf{Provider: "anchor", Provision: "database", Consumer: "shanks", ConsumerModule: "gitea", + Origin: "made", Generation: "g-1"} + for _, caller := range []string{"node.anchor", "node.shanks"} { + if _, err := v.MoveOpen(caller, pair); err != nil { + t.Errorf("%s, an end of the pair, was refused: %v", caller, err) + } + } + if _, err := v.MoveOpen("node.g14", pair); err == nil { + t.Error("a machine at neither end opened a move of the pair") + } + // The second step is the first step's caller's alone. + opened, err := v.MoveOpen("node.anchor", superuser()) + if err != nil { + t.Fatal(err) + } + key, _, _ := aReadBackKey(t) + if _, err := v.MoveTake("node.shanks", opened.Call, sealToCall(t, opened.SealTo, movedValue), key); err == nil { + t.Error("another machine's node-engine took the second step") + } + if _, held, _ := v.Store.Read("own/anchor/postgres/superuser-password"); held { + t.Error("a value was kept from the wrong caller") + } +} + +// A pair credential is named by both ends and its local name, so every row of the controller's store is one +// secret in the vault, kept as it is. +func TestAPairCredentialMovesUnderTheNameOfItsRow(t *testing.T) { + v := newVault(t, nil) + pair := MoveOf{Provider: "anchor", Provision: "database", Consumer: "shanks", ConsumerModule: "gitea", + Local: "replica", Origin: "accepted", Generation: "g-7"} + moved, err := moveIn(t, v, "node.shanks", pair, "the-pair-value") + if err != nil { + t.Fatal(err) + } + if moved.Name != "pair/anchor/database/shanks/gitea/replica" { + t.Errorf("named %s", moved.Name) + } + pair.Local = "" + if moved, err = moveIn(t, v, "node.anchor", pair, "the-ordinary-value"); err != nil || + moved.Name != "pair/anchor/database/shanks/gitea" { + t.Errorf("the ordinary one: %+v, %v", moved, err) + } +} + +// The move never replaces a value: the same value again changes nothing (the other end of a pair, or a move run +// twice), and a different one is refused, so a value the vault holds is never put back to an older one. +func TestTheMoveNeverReplacesAValueTheVaultHolds(t *testing.T) { + v := newVault(t, nil) + pair := MoveOf{Provider: "anchor", Provision: "database", Consumer: "shanks", ConsumerModule: "gitea", + Origin: "made", Generation: "g-1"} + if _, err := moveIn(t, v, "node.anchor", pair, "the-pair-value"); err != nil { + t.Fatal(err) + } + opened, err := v.MoveOpen("node.shanks", pair) + if err != nil { + t.Fatal(err) + } + key, pub, priv := aReadBackKey(t) + again, err := v.MoveTake("node.shanks", opened.Call, sealToCall(t, opened.SealTo, "the-pair-value"), key) + if err != nil || !again.Unchanged || again.Generation != 1 { + t.Fatalf("the other end's same value: %+v, %v", again, err) + } + if got := openReadBack(t, again.ReadBack, pub, priv); got != "the-pair-value" { + t.Errorf("read back %q", got) + } + if _, err := v.Rotate(ControllerCaller, "pair/anchor/database/shanks/gitea", "jochen", "a test"); err != nil { + t.Fatal(err) + } + if _, err := moveIn(t, v, "node.anchor", pair, "the-pair-value"); err == nil || + !strings.Contains(err.Error(), "never replaces") { + t.Fatalf("a move over a value the vault changed since: %v", err) + } + rec, _, _ := v.Store.Read("pair/anchor/database/shanks/gitea") + if rec.Generation != 2 || rec.Origin != OriginMade { + t.Errorf("the vault's own value was touched: %+v", rec) + } +} + +// A second step that is not sealed to its call, or carries no key to read back to, takes nothing. +func TestAMoveTakesNothingItCannotOpenOrReadBack(t *testing.T) { + v := newVault(t, nil) + opened, err := v.MoveOpen("node.anchor", superuser()) + if err != nil { + t.Fatal(err) + } + for _, key := range []string{"", "not a key", "-----BEGIN PRIVATE KEY-----"} { + if _, err := v.MoveTake("node.anchor", opened.Call, sealToCall(t, opened.SealTo, movedValue), key); err == nil { + t.Errorf("read-back key %q was taken", key) + } + } + // A refused key leaves the call open for the right one. + other, _ := v.MoveOpen("node.anchor", superuser()) + key, _, _ := aReadBackKey(t) + if _, err := v.MoveTake("node.anchor", other.Call, "bm90IHNlYWxlZA==", key); err == nil { + t.Error("a value not sealed to the call was taken") + } + if _, held, _ := v.Store.Read("own/anchor/postgres/superuser-password"); held { + t.Error("something was kept") + } + if moved, err := v.MoveTake("node.anchor", opened.Call, sealToCall(t, opened.SealTo, movedValue), key); err != nil || + moved.Generation != 1 { + t.Errorf("the call after a refused key: %+v, %v", moved, err) + } +} + +// The controller's record of asks is copied as it is, once: each ask's opening and its end, who asked and at which +// desk, and the bounds then count them. From the controller alone. +func TestTheRecordOfAsksIsCopiedAsItIsOnce(t *testing.T) { + v := newVault(t, nil) + now := v.Now() + ended := now.Add(-30 * time.Second) + asks := []MovedAsk{ + {ID: 41, Node: "anchor", Module: "telegram", Name: "telegram-token", Desk: "laptop", AskedBy: "g14/claude-code", + OpenedAt: now.Add(-50 * time.Minute), EndedAt: &ended, Outcome: "given"}, + {ID: 42, Node: "anchor", Module: "telegram", Name: "telegram-token", Desk: "laptop", AskedBy: "g14/claude-code", + OpenedAt: now.Add(-40 * time.Minute), EndedAt: &ended, Outcome: "dismissed"}, + {ID: 43, Node: "anchor", Module: "telegram", Name: "telegram-token", Desk: "laptop", AskedBy: "ace/agent", + OpenedAt: now.Add(-20 * time.Minute)}, + } + if _, err := v.MoveAsks("node.anchor", asks); err == nil { + t.Error("a node-engine copied the record of asks") + } + copied, err := v.MoveAsks(ControllerCaller, asks) + if err != nil || copied.Copied != 3 || copied.Already != 0 { + t.Fatalf("copied %+v, %v", copied, err) + } + again, err := v.MoveAsks(ControllerCaller, asks) + if err != nil || again.Copied != 0 || again.Already != 3 { + t.Fatalf("copied again %+v, %v", again, err) + } + all, _ := v.Store.Asks(100) + if len(all) != 5 { + t.Fatalf("%d lines: %+v", len(all), all) + } + opened := map[string]AskRecord{} + for _, r := range all { + if r.Event == AskOpened { + opened[r.ID] = r + } + } + if r := opened["controller-41"]; r.Secret != "own/anchor/telegram/telegram-token" || r.AskedBy != "g14/claude-code" || + r.Desk != "laptop" || !r.At.Equal(asks[0].OpenedAt) || r.Caller != ControllerCaller { + t.Errorf("ask 41: %+v", r) + } + // Three asks in the hour, from the controller's record: the vault's bound holds them. + if _, err := v.AskOpen(ControllerCaller, telegramAsk("g14/claude-code")); err == nil || + !strings.Contains(err.Error(), "in the last hour") { + t.Errorf("an ask after three copied in the hour: %v", err) + } + bad := []MovedAsk{{ID: 44, Node: "anchor", Module: "telegram", Name: "x", Desk: "laptop", AskedBy: "a", + OpenedAt: now, EndedAt: &ended, Outcome: "whatever"}} + if _, err := v.MoveAsks(ControllerCaller, bad); err == nil { + t.Error("an ask that ended in no outcome the record knows was copied") + } +} + +// The two steps as the runtime calls the tool: the caller is the one it proved, never an argument. +func TestMoveInThroughTheTool(t *testing.T) { + v := newVault(t, nil) + tool := toolNamed(t, definitions(&Env{Vault: v.Vault}), Seat+".move-in") + first := map[string]any{"node": "anchor", "module": "postgres", "name": "superuser-password", "origin": "made", + "generation": "g-1", "made_at": "2025-11-02T10:00:00Z"} + if _, err := tool.runAs("controller", first); err == nil { + t.Fatal("the controller opened a move") + } + got, err := tool.runAs("node.anchor", first) + if err != nil { + t.Fatal(err) + } + opened := got.(Opened) + key, pub, priv := aReadBackKey(t) + got, err = tool.runAs("node.anchor", map[string]any{"call": opened.Call, + "sealed": sealToCall(t, opened.SealTo, movedValue), "read_back_to": key}) + if err != nil { + t.Fatal(err) + } + if back := openReadBack(t, got.(MovedIn).ReadBack, pub, priv); back != movedValue { + t.Errorf("read back %q", back) + } + rec, _, _ := v.Store.Read("own/anchor/postgres/superuser-password") + if rec.MadeAt.Year() != 2025 || rec.MovedFrom == nil || rec.MovedFrom.Generation != "g-1" { + t.Errorf("kept %+v", rec) + } + asks := toolNamed(t, definitions(&Env{Vault: v.Vault}), Seat+".move-asks") + moved, err := asks.runAs("controller", map[string]any{"asks": []any{map[string]any{"id": 7, "node": "anchor", + "module": "telegram", "name": "telegram-token", "desk": "laptop", "asked_by": "g14/claude-code", + "opened_at": "2026-10-01T10:00:00Z", "ended_at": "2026-10-01T10:00:20Z", "outcome": "given"}}}) + if err != nil || moved.(AsksMoved).Copied != 1 { + t.Fatalf("asks through the tool: %+v, %v", moved, err) + } +} diff --git a/modules/mesh-vault/cmd/mesh-vault/named_pair_test.go b/modules/mesh-vault/cmd/mesh-vault/named_pair_test.go new file mode 100644 index 00000000..779f1a51 --- /dev/null +++ b/modules/mesh-vault/cmd/mesh-vault/named_pair_test.go @@ -0,0 +1,44 @@ +package main + +// A consumer's own login, kept by the vault as its pair row, is named on the consumer's plan entry when the +// controller hands the consumer's secrets file to its machine's client (issue 779): the client asks for the row, and +// the rule hands it to that consumer's machine alone. + +import "testing" + +func TestANamedPairEntryHandsTheConsumersMachineItsOwnLoginAndNoOtherConsumers(t *testing.T) { + const pg = "provision/novox/postgres-database" + mirror := assignments{ + "novox.postgres": handedWith("novox", "postgres"), + "g14.gitea": handedWith("g14", "gitea", "postgres-database@novox/postgres=database+pair"), + "ace.umami": handedWith("ace", "umami", "postgres-database@novox/postgres#stats=database-stats+pair"), + "shanks.other": handedWith("shanks", "other", "postgres-database@novox/postgres=database+pair"), + } + v := pairVault(t, mirror) + if o := pass(t, v, pg); o.Outcome != PairMade { + t.Fatalf("a named pair entry held the provider back: %+v", o) + } + gitea, umami := "pair/novox/postgres-database/g14/gitea", "pair/novox/postgres-database/ace/umami/stats" + ofGitea, _, err := asks(v, "provisioner.g14", gitea) + if err != nil || ofGitea == "" { + t.Fatalf("the consumer's machine was refused its own login: %v", err) + } + if ofUmami, _, err := asks(v, "provisioner.ace", umami); err != nil || ofUmami == "" || ofUmami == ofGitea { + t.Fatalf("the second consumer's login: %q (%v)", ofUmami, err) + } + if _, _, err := asks(v, "provisioner.g14", umami); err == nil { + t.Fatal("a consumer's machine was handed another consumer's login") + } + if _, _, err := asks(v, "provisioner.shanks", gitea); err == nil { + t.Fatal("a machine was handed the login of a module that is not its own") + } + // A named entry of another form is still not a pair row's. + other := assignments{ + "novox.postgres": handedWith("novox", "postgres"), + "g14.gitea": handedWith("g14", "gitea", "postgres-database@novox/postgres=database+issued"), + } + w := pairVault(t, other) + if _, _, err := asks(w, "provisioner.g14", gitea); err == nil { + t.Fatal("a named entry of the issued form was handed a pair row") + } +} diff --git a/modules/mesh-vault/cmd/mesh-vault/nodekey_test.go b/modules/mesh-vault/cmd/mesh-vault/nodekey_test.go new file mode 100644 index 00000000..e266dc3d --- /dev/null +++ b/modules/mesh-vault/cmd/mesh-vault/nodekey_test.go @@ -0,0 +1,191 @@ +package main + +// Issue 786 (ADR 0372): a machine's node-engine states its own sealing key to the vault, in its own name, at start and +// when it changes, whether or not it ever moved a secret in. The caller is the bus user the runtime proved; a changed +// key (or a first one) raises `secret-changed` for the machine's core secrets, the same key says nothing. + +import ( + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/nats-io/nats-server/v2/server" + "github.com/nats-io/nats.go" +) + +func changedNames(p *published) []string { + p.mu.Lock() + defer p.mu.Unlock() + var out []string + for _, e := range p.events { + if e["event"] == ChangedEvent { + out = append(out, e["body"].(map[string]any)["name"].(string)) + } + } + return out +} + +func TestANodeEngineStatesItsOwnKeyAndAChangeRaisesItsCoreSecrets(t *testing.T) { + v := newVault(t, nil) + tool := toolNamed(t, definitions(&Env{Vault: v.Vault}), Seat+".node-key") + // ace holds two core secrets and a leaf module's; g14 holds a core secret too. + old, _ := aSealingKey(t) + for _, m := range []MoveOf{ + {Node: "ace", Module: "mesh-provisioner-client", Name: "provisioner-broker", Origin: "made", Generation: "g-1", SealingKey: old}, + {Node: "ace", Module: "nats", Name: "broker", Origin: "made", Generation: "g-1"}, + {Node: "ace", Module: "postgres", Name: "superuser-password", Origin: "made", Generation: "g-1"}, + {Node: "g14", Module: "nats", Name: "broker", Origin: "made", Generation: "g-1"}, + } { + if _, err := moveIn(t, v, "node."+m.Node, m, movedValue); err != nil { + t.Fatal(err) + } + } + // ace never stated a key in the moves above except through the first row; forget it to be a machine that never moved. + if err := os.Remove(v.Store.nodeKeyPath("ace")); err != nil { + t.Fatal(err) + } + before := len(changedNames(v.bus)) + + pub, _ := aSealingKey(t) + if _, err := tool.runAs("controller", map[string]any{"key": pub}); err == nil { + t.Error("the controller stated a machine's key") + } + if _, err := tool.runAs("", map[string]any{"key": pub}); err == nil { + t.Error("an unproven caller stated a key") + } + if _, err := tool.runAs("node.ace", map[string]any{"key": "c2hvcnQ="}); err == nil { + t.Error("a short key was taken") + } + if _, err := tool.runAs("node.ace", map[string]any{"node": "g14", "key": pub}); err != nil { + t.Fatal(err) + } + // The machine is the caller's; an argument naming another one changes nothing. + if got, held, _ := v.Store.NodeKey("ace"); !held || got != pub { + t.Errorf("ace's key is %q, %v", got, held) + } + if _, held, _ := v.Store.NodeKey("g14"); held { + t.Error("a key was recorded for g14 by ace's engine") + } + got := changedNames(v.bus)[before:] + want := []string{"own/ace/mesh-provisioner-client/provisioner-broker", "own/ace/nats/broker"} + if strings.Join(got, ",") != strings.Join(want, ",") { + t.Errorf("secret-changed was raised for %v, want %v", got, want) + } + + // The same key again changes nothing and says nothing. + if _, err := tool.runAs("node.ace", map[string]any{"key": pub}); err != nil { + t.Fatal(err) + } + if n := len(changedNames(v.bus)); n != before+2 { + t.Errorf("the same key raised %d more", n-before-2) + } + // A different key raises them again. + other, _ := aSealingKey(t) + if _, err := tool.runAs("node.ace", map[string]any{"key": other}); err != nil { + t.Fatal(err) + } + if n := len(changedNames(v.bus)); n != before+4 { + t.Errorf("a changed key raised %d, want 2", n-before-2) + } + if got, _, _ := v.Store.NodeKey("ace"); got != other { + t.Error("the changed key was not recorded") + } +} + +const nodeKeyBus = `listen: 127.0.0.1:-1 +authorization { + users = [ + { user: "novox.mesh-vault", password: "pw", permissions: { + publish: { deny: [">"] } + subscribe: { allow: ["mesh.seat.mesh-vault.call.node-key.*"] } + allow_responses: true } } + { user: "node.ace", password: "pw", permissions: { + publish: { allow: ["mesh.seat.mesh-vault.call.node-key.node_ace"] } + subscribe: { allow: ["_INBOX.node.ace.>"] } } } + { user: "node.g14", password: "pw", permissions: { + publish: { allow: ["mesh.seat.mesh-vault.call.node-key.node_g14"] } + subscribe: { allow: ["_INBOX.node.g14.>"] } } } + ] +} +` + +// On a real bus whose accounts may publish only their own subject: node.ace's statement records ace's key; node.g14 +// cannot speak in ace's name, and what it states is g14's. +func TestOnARealBusANodeEngineStatesOnlyItsOwnKey(t *testing.T) { + file := filepath.Join(t.TempDir(), "bus.conf") + if err := os.WriteFile(file, []byte(nodeKeyBus), 0o600); err != nil { + t.Fatal(err) + } + opts, err := server.ProcessConfigFile(file) + if err != nil { + t.Fatal(err) + } + opts.NoLog, opts.NoSigs = true, true + s, err := server.NewServer(opts) + if err != nil { + t.Fatal(err) + } + go s.Start() + if !s.ReadyForConnections(10 * time.Second) { + t.Fatal("the bus did not come up") + } + t.Cleanup(s.Shutdown) + + v := newVault(t, nil) + tool := toolNamed(t, definitions(&Env{Vault: v.Vault}), Seat+".node-key") + vaultConn := connectAs(t, s.ClientURL(), "novox.mesh-vault") + if _, err := vaultConn.Subscribe("mesh.seat.mesh-vault.call.node-key.*", func(m *nats.Msg) { + parts := strings.Split(m.Subject, ".") + var args map[string]any + _ = json.Unmarshal(m.Data, &args) + answer := busAnswer{} + if got, err := tool.runAs(untoken(parts[len(parts)-1]), args); err != nil { + answer.Error = err.Error() + } else { + answer.Result, _ = json.Marshal(got) + } + raw, _ := json.Marshal(answer) + _ = m.Respond(raw) + }); err != nil { + t.Fatal(err) + } + _ = vaultConn.Flush() + + say := func(user, subject, key string) (string, error) { + raw, _ := json.Marshal(map[string]string{"key": key}) + msg, err := connectAs(t, s.ClientURL(), user).Request(subject, raw, 3*time.Second) + if err != nil { + return "", err + } + var a busAnswer + _ = json.Unmarshal(msg.Data, &a) + if a.Error != "" { + return a.Error, errString(a.Error) + } + return "", nil + } + aceKey, _ := aSealingKey(t) + g14Key, _ := aSealingKey(t) + if _, err := say("node.ace", "mesh.seat.mesh-vault.call.node-key.node_ace", aceKey); err != nil { + t.Fatal(err) + } + if got, held, _ := v.Store.NodeKey("ace"); !held || got != aceKey { + t.Errorf("ace's key is %q, %v", got, held) + } + // g14 may not publish on ace's subject: the bus refuses it, so the vault never hears it. + if _, err := say("node.g14", "mesh.seat.mesh-vault.call.node-key.node_ace", g14Key); err == nil { + t.Error("node.g14 spoke in ace's name") + } + if got, _, _ := v.Store.NodeKey("ace"); got != aceKey { + t.Error("ace's key was replaced by g14's") + } + if _, err := say("node.g14", "mesh.seat.mesh-vault.call.node-key.node_g14", g14Key); err != nil { + t.Fatal(err) + } + if got, held, _ := v.Store.NodeKey("g14"); !held || got != g14Key { + t.Errorf("g14's key is %q, %v", got, held) + } +} diff --git a/modules/mesh-vault/cmd/mesh-vault/owncalls_test.go b/modules/mesh-vault/cmd/mesh-vault/owncalls_test.go new file mode 100644 index 00000000..5f303955 --- /dev/null +++ b/modules/mesh-vault/cmd/mesh-vault/owncalls_test.go @@ -0,0 +1,230 @@ +package main + +// The vault's own calls (issue 723): it asks the desk's prompt and an issuing provider's tool itself, in its own +// name, under the bus account its own runtime runs on. That the bus lets that account alone make these calls, and +// nothing else, is mesh-controller's test on a real server (internal/broker vault_calls_live_test.go); here, what +// the vault sends and takes back on those calls carries keys and sealed values only. + +import ( + "crypto/rand" + "encoding/base64" + "encoding/json" + "errors" + "strings" + "sync" + "testing" + + "golang.org/x/crypto/nacl/box" +) + +// vaultsAccount is the vault's bus user as its own runtime names it to a tool it calls. +const vaultsAccount = "novox.mesh-vault" + +// aMesh stands in for the bus the vault's runtime calls on: each call is answered by the desk's prompt or a +// provider's tool, named by the caller the vault's own account is, and everything that crosses is kept. +type aMesh struct { + mu sync.Mutex + desk func(caller string, args map[string]any) (map[string]any, error) + tools map[string]func(caller string, args map[string]any) (map[string]any, error) + keys []string + crossed []string +} + +func (m *aMesh) ask(key string, body any) (json.RawMessage, error) { + m.mu.Lock() + m.keys = append(m.keys, key) + m.mu.Unlock() + raw, _ := json.Marshal(body) + var args map[string]any + _ = json.Unmarshal(raw, &args) + var answer map[string]any + var err error + switch { + case strings.HasPrefix(key, "seat:node-launcher.secret@") && m.desk != nil: + answer, err = m.desk(vaultsAccount, args) + case m.tools[key] != nil: + answer, err = m.tools[key](vaultsAccount, args) + default: + err = errors.New("503 no responders") + } + out, _ := json.Marshal(answer) + m.mu.Lock() + m.crossed = append(m.crossed, string(raw), string(out)) + m.mu.Unlock() + if err != nil { + return nil, err + } + return out, nil +} + +func (m *aMesh) said() string { + m.mu.Lock() + defer m.mu.Unlock() + return strings.Join(m.crossed, "\n") +} + +// aDesk is the desk's prompt (rofi's node-launcher.secret): it answers what the operator types, sealed to the key +// it is given, and keeps what it was asked. +func aDesk(typed string, asked *[]map[string]any) func(string, map[string]any) (map[string]any, error) { + return func(caller string, args map[string]any) (map[string]any, error) { + *asked = append(*asked, args) + switch typed { + case "": + return map[string]any{"cancelled": true}, nil + case "\x00timeout": + return map[string]any{"cancelled": true, "timed_out": true}, nil + } + key, _ := args["seal_to"].(string) + raw, err := base64.StdEncoding.DecodeString(key) + if err != nil || len(raw) != 32 { + return nil, errors.New("seal_to is not a sealing key") + } + var to [32]byte + copy(to[:], raw) + sealed, err := box.SealAnonymous(nil, []byte(typed+"\n"), &to, rand.Reader) + if err != nil { + return nil, err + } + return map[string]any{"sealed": base64.StdEncoding.EncodeToString(sealed)}, nil + } +} + +// The vault asks the desk itself: recorded first, the prompt named by the vault's call and naming the asker, what +// is typed opened here alone; what crossed is a key and a sealed box. +func TestTheVaultAsksTheDesksPromptItselfAndOnlyASealedValueCrosses(t *testing.T) { + v := newVault(t, AllowList{"anchor.node-provisioner": {"own/anchor/telegram/telegram-token"}}) + var asked []map[string]any + mesh := &aMesh{desk: aDesk(typedAtTheDesk, &asked)} + v.Ask = mesh.ask + + got, err := v.AskAtDesk(ControllerCaller, telegramAsk("g14/claude-code, via the console")) + if err != nil { + t.Fatal(err) + } + if got.Outcome != "given" || got.Generation != 1 || got.AskedBy != "g14/claude-code" || + got.Secret != "own/anchor/telegram/telegram-token" { + t.Errorf("answered %+v", got) + } + if len(mesh.keys) != 1 || mesh.keys[0] != "seat:node-launcher.secret@laptop" { + t.Fatalf("the vault asked %v", mesh.keys) + } + p := asked[0] + if p["asked_by"] != "g14/claude-code" || p["module"] != "telegram" || p["secret"] != "telegram-token" || + p["node"] != "anchor" || p["seal_to"] == "" || p["timeout_seconds"] != float64(DeskPromptFor) { + t.Errorf("the prompt was asked %v", p) + } + if v := handedTo(t, v, "anchor.node-provisioner", "own/anchor/telegram/telegram-token"); v != typedAtTheDesk { + t.Errorf("the vault holds %q", v) + } + rec, _, _ := v.Store.Read("own/anchor/telegram/telegram-token") + if rec.Origin != OriginTyped || rec.ChangedBy != "g14/claude-code" || rec.ChangedVia != ControllerCaller { + t.Errorf("the record: origin %q by %q via %q", rec.Origin, rec.ChangedBy, rec.ChangedVia) + } + asks, _ := v.Store.Asks(10) + if len(asks) != 2 || asks[1].Event != AskOpened || asks[0].Outcome != "given" { + t.Errorf("the record of asks: %+v", asks) + } + if strings.Contains(mesh.said(), typedAtTheDesk) || strings.Contains(everything(t, v, got, asks), typedAtTheDesk) { + t.Fatal("the typed value crossed the bus, or is in something the vault said or wrote, in the clear") + } + + // Dismissed and timed out end the ask so, and keep nothing. + for typed, outcome := range map[string]string{"": "dismissed", "\x00timeout": "timed-out"} { + w := newVault(t, nil) + var a []map[string]any + w.Ask = (&aMesh{desk: aDesk(typed, &a)}).ask + got, err := w.AskAtDesk(ControllerCaller, telegramAsk("g14.claude-code")) + if err != nil || got.Outcome != outcome { + t.Errorf("%s: %+v %v", outcome, got, err) + } + if asks, _ := w.Store.Asks(1); asks[0].Outcome != outcome { + t.Errorf("%s recorded %+v", outcome, asks[0]) + } + if _, held, _ := w.Store.Read("own/anchor/telegram/telegram-token"); held { + t.Errorf("%s kept a value", outcome) + } + } + // Answered empty: the operator's answer, said so, nothing kept. + e := newVault(t, nil) + var ea []map[string]any + e.Ask = (&aMesh{desk: aDesk(" ", &ea)}).ask + if got, err := e.AskAtDesk(ControllerCaller, telegramAsk("g14.claude-code")); err != nil || got.Outcome != "empty" { + t.Errorf("empty: %+v %v", got, err) + } + // The desk cannot be asked: refused, recorded so. + r := newVault(t, nil) + r.Ask = (&aMesh{}).ask + if _, err := r.AskAtDesk(ControllerCaller, telegramAsk("g14.claude-code")); err == nil { + t.Error("an ask nobody answered was said to be done") + } + if asks, _ := r.Store.Asks(1); asks[0].Outcome != "refused" { + t.Errorf("an unanswered desk recorded %+v", asks[0]) + } + // Only the controller asks the vault to ask: anybody else is refused before any prompt. + n := newVault(t, nil) + var na []map[string]any + n.Ask = (&aMesh{desk: aDesk(typedAtTheDesk, &na)}).ask + for _, caller := range []string{"g14.node-tools", "g14.claude-code", "unproven", ""} { + if _, err := n.AskAtDesk(caller, telegramAsk("x")); err == nil { + t.Errorf("%q had the vault ask a desk", caller) + } + } + if len(na) != 0 { + t.Fatal("a desk was asked for a caller that is not the controller") + } +} + +// The vault asks an issuing provider's tool itself: by its name on the provider's machine, with a key made for the +// call; the tool answers the vault's account; the value opens here; what crossed is a key and a sealed box. +func TestTheVaultAsksTheIssuingToolItselfAndOnlyASealedValueCrosses(t *testing.T) { + v := newVault(t, AllowList{"anchor.node-provisioner": {"provision/anchor/plex-api"}}) + software := "plex-token-from-the-vendor" + mesh := &aMesh{tools: map[string]func(string, map[string]any) (map[string]any, error){ + "plex.plex_issued_credential@anchor": standInProvider(&software)}} + v.Ask = mesh.ask + got, err := v.TakeIssued(ControllerCaller, "plex-api", "anchor", "plex", "plex_issued_credential") + if err != nil { + t.Fatal(err) + } + if got.Secret != "provision/anchor/plex-api" || got.Generation != 1 || got.Unchanged { + t.Errorf("answered %+v", got) + } + if got := handedTo(t, v, "anchor.node-provisioner", "provision/anchor/plex-api"); got != software { + t.Errorf("the vault holds %q", got) + } + if strings.Contains(mesh.said(), software) || strings.Contains(everything(t, v, got), software) { + t.Fatal("the issued value crossed the bus, or is in something the vault said or wrote, in the clear") + } + if !strings.Contains(mesh.said(), `"provision":"plex-api"`) || !strings.Contains(mesh.said(), `"seal_to"`) { + t.Errorf("the tool was asked %s", mesh.said()) + } + // The same value again writes nothing. + again, err := v.TakeIssued(ControllerCaller, "plex-api", "anchor", "plex", "plex_issued_credential") + if err != nil || !again.Unchanged || again.Generation != 1 { + t.Errorf("again: %+v %v", again, err) + } + // Only the controller asks the vault to take it; a tool's name that is not one is refused; a tool that is not + // answered, or answers no sealed value, keeps nothing. + for _, caller := range []string{"anchor.node-tools", "g14.claude-code", "unproven", ""} { + if _, err := v.TakeIssued(caller, "plex-api", "anchor", "plex", "plex_issued_credential"); err == nil { + t.Errorf("%q had the vault ask a provider", caller) + } + } + for _, bad := range []string{"", "plex.other", "x@y", "a b"} { + if _, err := v.TakeIssued(ControllerCaller, "plex-api", "anchor", "plex", bad); err == nil { + t.Errorf("the tool %q was asked", bad) + } + } + if _, err := v.TakeIssued(ControllerCaller, "plex-api", "anchor", "plex", "plex_status"); err == nil { + t.Error("a tool nobody answers gave a value") + } + mesh.tools["plex.plex_issued_credential@anchor"] = func(string, map[string]any) (map[string]any, error) { + return map[string]any{"value": "in-the-clear"}, nil + } + if _, err := v.TakeIssued(ControllerCaller, "plex-api", "anchor", "plex", "plex_issued_credential"); err == nil { + t.Error("a tool answering no sealed value gave a value") + } + if len(v.pending) != 0 { + t.Errorf("%d calls' keys are still kept", len(v.pending)) + } +} diff --git a/modules/mesh-vault/cmd/mesh-vault/pair.go b/modules/mesh-vault/cmd/mesh-vault/pair.go new file mode 100644 index 00000000..ac3df924 --- /dev/null +++ b/modules/mesh-vault/cmd/mesh-vault/pair.go @@ -0,0 +1,814 @@ +package main + +// The credential a provider shares with its consumers (novox/hq ADR 0349 clause 8; ADR 0158 §4–§6, §8; design 57's +// V5, issue 677): one value per provider assignment and provision, `provision//`, minted +// and remade by the vault for all its holders together when a consumer binds or unbinds, or when it is rotated. +// +// **Where the vault sees a bind.** A recorded need names no provider (issue 677's note 7). The controller names, in +// the plan of each module it handed to the provisioner, each provision the module requires with the provider +// resolution chose (issue 760): `provisions` in `.` of the assignments mirror, the plan the module's +// client records its needs from. A consumer binds when such an entry appears and unbinds when it goes, or when its +// assignment goes. The vault reads every plan on each pass (once a minute, and when it starts). +// +// **Every holder can get it, or nothing is made** (the lead's rule on issue 677, note 10). The holders are the +// provider on its machine and every consumer bound to it. Only a module handed to the provisioner gets a value from +// the vault; one still filled the old way gets the controller's (ADR 0349 clause 17). So the vault mints or remakes +// a provider's value only when the provider's assignment is handed to the provisioner, and no consumer the vault +// knows to hold it is still filled the old way: a consumer whose pair row the move brought in (V3) and whose +// assignment is still there with no plan. Otherwise it makes nothing, and says why on each pass that changes what +// it says. A consumer filled the old way that bound after the move names no provider on the bus at all; it is the +// provider's move (design 57 B2, B3) that ends that case. +// +// **What the move brought in is adopted, and nothing is rotated early.** The move (V3) copied each pair row as +// `pair////[/]`. Rows a provider shares (ADR 0158: its own secret and +// every consumer's row made in one act) carry one generation stamp; when every row of a provider and provision +// carries the same stamp and the same value, the vault keeps that value as the provision's one credential, with its +// origin, and remakes nothing. Rows of a provider with a login per consumer (ADR 0048) carry no stamp and differ +// per pair: each consumer keeps a value of its own for ever (ADR 0158 §11, issue 677's note 12). The vault never +// makes them one shared value; a rotation (`rotate` of `provision//`) remakes each consumer's +// own apart, and each consumer's machine is handed only its own. +// +// **A provider the vault holds no row of gets its first value in its offer's form** (issue 768). Each plan entry +// names the form the provider's offer declares (`credential-form`, from the offer's `credential` in the provider's +// manifest): `shared`, one value for every holder (ADR 0158), made as `provision//`; `pair`, a +// login of its own for each consumer (ADR 0048, ADR 0158 §11), made as each consumer's pair row; `issued` by the +// provider's software (ADR 0305) or `given` by a person (ADR 0337), which the vault never makes. An entry naming no +// form (a controller from before), plans that disagree, or a form the vault does not know: nothing is made, rather +// than a guess. A provider the vault already holds rows of is unchanged by the form: it is adopted or kept per pair +// as above. +// +// **An accepted or issued value is never remade** (ADR 0158 §8, ADR 0305 §9): the vault records who holds it and +// hands it to them, and leaves the value alone. A provision whose plans name a credential per holder (issue 757, +// an entry with a name) has no one shared value and is not the vault's to make here. + +import ( + "crypto/rand" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "net/url" + "os" + "path/filepath" + "slices" + "sort" + "strings" + "time" + + "git.novox.be/novox/mesh-sdk/go/seal" +) + +// Mirror is the controller's assignments mirror as the vault reads it: every key, and one key's value. +type Mirror struct { + Keys func() ([]string, error) + Read AssignmentReader +} + +// Shared is what the vault last made, adopted or recorded the provision's one credential for: its holders and the +// generation they hold. Never a value. +type Shared struct { + Name string `json:"name"` + Provider string `json:"provider"` + Provision string `json:"provision"` + ProviderModule string `json:"provider-module,omitempty"` + // Consumers are the binds holding it: `/`, and `/` when bound under a local name. + Consumers []string `json:"consumers"` + Generation int `json:"generation"` + At time.Time `json:"at"` + // Adopted is the controller's generation stamp of the pair rows the value was adopted from, when it was. + Adopted string `json:"adopted,omitempty"` +} + +// holderNodes are the machines holding it: the provider's and every consumer's. +func (s Shared) holderNodes() []string { + out := []string{s.Provider} + for _, c := range s.Consumers { + out = append(out, strings.SplitN(c, "/", 2)[0]) + } + return out +} + +func (s *Store) sharedPath(name string) string { + return filepath.Join(s.dir, "shared", url.PathEscape(name)+".json") +} + +// ReadShared is the record of who holds the provision's one credential name, and whether there is one. +func (s *Store) ReadShared(name string) (Shared, bool, error) { + if err := validName(name); err != nil { + return Shared{}, false, err + } + raw, err := os.ReadFile(s.sharedPath(name)) + if errors.Is(err, os.ErrNotExist) { + return Shared{}, false, nil + } + if err != nil { + return Shared{}, false, err + } + var sh Shared + if err := json.Unmarshal(raw, &sh); err != nil { + return Shared{}, false, fmt.Errorf("the record of who holds %s is not readable: %w", name, err) + } + return sh, true, nil +} + +// WriteShared keeps the record of who holds a provision's one credential. +func (s *Store) WriteShared(sh Shared) error { + if err := validName(sh.Name); err != nil { + return err + } + if err := os.MkdirAll(filepath.Join(s.dir, "shared"), 0o700); err != nil { + return err + } + sort.Strings(sh.Consumers) + raw, err := json.MarshalIndent(sh, "", " ") + if err != nil { + return err + } + return writeWhole(s.sharedPath(sh.Name), append(raw, '\n')) +} + +// pairKey is one provider assignment and provision. +type pairKey struct{ provider, provision string } + +func (k pairKey) name() string { return "provision/" + k.provider + "/" + k.provision } + +// binds is what the plans say of one provider and provision: its module, who binds it, whether a plan names a +// credential per holder for it, and the credential forms the plans name for it ("" for an entry naming none). +type binds struct { + providerModules map[string]bool + consumers map[string]bool + perHolder bool + forms map[string]bool +} + +// The credential forms a plan entry names (issue 768), as the provider's offer declares them. +const ( + // FormShared is one value every holder shares (ADR 0158): the offer's credential is one of its own secrets. + FormShared = "shared" + // FormPair is a login of its own for each consumer (ADR 0048, ADR 0158 §11): the offer says no credential. + FormPair = "pair" + // FormIssued is a value the provider's software issues (ADR 0305): the mesh never makes it. + FormIssued = "issued" + // FormGiven is a value only a person gives (ADR 0337): the mesh never makes it. + FormGiven = "given" +) + +// credentialForms are every form a plan entry may name. +var credentialForms = []string{FormShared, FormPair, FormIssued, FormGiven} + +// form is the one form the plans name for k, or why the vault cannot trust one. +func (b *binds) form(k pairKey) (string, string) { + if b == nil || len(b.forms) == 0 || b.forms[""] { + return "", unknownForm(k) + } + var named []string + for f := range b.forms { + if !slices.Contains(credentialForms, f) { + return "", fmt.Sprintf("a plan names %q for %s from %s, which is not a credential form the vault knows "+ + "(%s); it makes nothing", f, k.provision, k.provider, strings.Join(credentialForms, ", ")) + } + named = append(named, f) + } + if len(named) > 1 { + sort.Strings(named) + return "", fmt.Sprintf("the plans binding %s from %s disagree on its credential form (%s); it makes nothing", + k.provision, k.provider, strings.Join(named, ", ")) + } + return named[0], "" +} + +// pairRow is one row the move brought in. +type pairRow struct { + consumer, module string + rec Record +} + +// bindings is what one pass read: every plan's bound provisions, which assignments there are, and which of them +// are handed to the provisioner. +type bindings struct { + bound map[pairKey]*binds + assigned map[string]bool + handed map[string]bool +} + +// plannedProvisions is as much of an assignment as the pass reads. +type plannedProvisions struct { + Node string `json:"node"` + Module string `json:"module"` + Provisioning *struct { + Provisions []struct { + Name string `json:"name"` + Provision string `json:"provision"` + Local string `json:"local"` + Provider string `json:"provider"` + ProviderModule string `json:"provider-module"` + CredentialForm string `json:"credential-form"` + // Contributes is what the consumer tells its provider (issue 776); the vault does not act on it. + Contributes map[string]any `json:"contributes"` + } `json:"provisions"` + } `json:"provisioning"` +} + +// readBindings reads every assignment. An assignment that cannot be read stops the pass: a bind missed would read +// as an unbind, and remake a value its holder still uses. +func readBindings(m Mirror) (bindings, error) { + b := bindings{bound: map[pairKey]*binds{}, assigned: map[string]bool{}, handed: map[string]bool{}} + if m.Keys == nil || m.Read == nil { + return b, errors.New("the vault cannot read the assignments, so it cannot see who binds a provider") + } + keys, err := m.Keys() + if err != nil { + return b, fmt.Errorf("the assignments' keys could not be read: %w", err) + } + for _, key := range keys { + raw, err := m.Read(key) + if err != nil { + return b, fmt.Errorf("the assignment %s could not be read: %w", key, err) + } + if raw == nil { + continue + } + var a plannedProvisions + if err := json.Unmarshal(raw, &a); err != nil || a.Node+"."+a.Module != key { + return b, fmt.Errorf("the assignment %s is not the controller's", key) + } + b.assigned[key] = true + if a.Provisioning == nil { + continue + } + b.handed[key] = true + for _, p := range a.Provisioning.Provisions { + if !namePart.MatchString(p.Provider) || !namePart.MatchString(p.Provision) { + continue + } + k := pairKey{p.Provider, p.Provision} + if b.bound[k] == nil { + b.bound[k] = &binds{providerModules: map[string]bool{}, consumers: map[string]bool{}, + forms: map[string]bool{}} + } + b.bound[k].forms[p.CredentialForm] = true + if p.ProviderModule != "" { + b.bound[k].providerModules[p.ProviderModule] = true + } + // A name on an entry of the login-per-consumer form is that login's file (issue 779), not a value issued + // for one holder: the provider still gets its logins as rows. + if p.Name != "" && p.CredentialForm != FormPair { + b.bound[k].perHolder = true + } + b.bound[k].consumers[consumerOf(a.Node, a.Module, p.Local)] = true + } + } + return b, nil +} + +func consumerOf(node, module, local string) string { + if local == "" { + return node + "/" + module + } + return node + "/" + module + "/" + local +} + +// pairRows is every row the move brought in, by provider and provision. +func (v *Vault) pairRows() (map[pairKey][]pairRow, error) { + all, err := v.Store.List() + if err != nil { + return nil, err + } + out := map[pairKey][]pairRow{} + for _, rec := range all { + parts := strings.Split(rec.Name, "/") + if parts[0] != "pair" || (len(parts) != 5 && len(parts) != 6) { + continue + } + k := pairKey{parts[1], parts[2]} + out[k] = append(out[k], pairRow{consumer: parts[3], module: parts[4], rec: rec}) + } + return out, nil +} + +// PairOutcome is what one pass did, or did not do, for one provider and provision. +type PairOutcome struct { + Name string `json:"name"` + Outcome string `json:"outcome"` + Generation int `json:"generation,omitempty"` + Why string `json:"why,omitempty"` +} + +// What a pass did for a provider and provision. +const ( + PairMade = "made" + PairRemade = "remade" + PairAdopted = "adopted" + PairRecorded = "recorded" + PairUnchanged = "unchanged" + PairHeld = "held" + PairPerPair = "waits-for-rotation" + PairPerHolder = "per-holder" +) + +// sharedHolders is who holds a provider's value now: its provider module and the consumers bound to it. +func sharedHolders(k pairKey, b *binds) (string, []string) { + var modules, consumers []string + if b != nil { + for m := range b.providerModules { + modules = append(modules, m) + } + for c := range b.consumers { + consumers = append(consumers, c) + } + } + sort.Strings(modules) + sort.Strings(consumers) + return strings.Join(modules, ","), consumers +} + +// heldBack says why the vault may not make a value for k now, or "": a holder that would never get it. +func heldBack(k pairKey, b *binds, rows []pairRow, seen bindings) string { + if b == nil || len(b.providerModules) == 0 { + return fmt.Sprintf("no plan names the module serving %s on %s, so the vault cannot tell the provider gets "+ + "a value it makes", k.provision, k.provider) + } + for m := range b.providerModules { + if !seen.handed[k.provider+"."+m] { + return fmt.Sprintf("the provider %s on %s is not handed to the provisioner, so it would never get a "+ + "value the vault makes (ADR 0349 clause 17)", m, k.provider) + } + } + for _, r := range rows { + key := r.consumer + "." + r.module + if seen.assigned[key] && !seen.handed[key] { + return fmt.Sprintf("%s on %s holds %s from %s and is not handed to the provisioner, so it would never "+ + "get a value the vault makes (ADR 0349 clause 17)", r.module, r.consumer, k.provision, k.provider) + } + } + return "" +} + +// oneStamp is the generation stamp every row shares, with their one origin and value, when they all share one; or +// why not. +func (v *Vault) oneStamp(rows []pairRow) (stamp, origin, value string, madeAt time.Time, why string) { + for i, r := range rows { + if r.rec.MovedFrom == nil || r.rec.MovedFrom.Generation == "" { + return "", "", "", time.Time{}, fmt.Sprintf("%s carries no generation stamp: a login of its own (ADR 0048)", + r.rec.Name) + } + opened, err := seal.Open(r.rec.AtRest, v.Own.PrivateKey) + if err != nil { + return "", "", "", time.Time{}, fmt.Sprintf("%s does not open with the vault's key: %v", r.rec.Name, err) + } + o := r.rec.Origin + if o == "" { + o = OriginMade + } + if i == 0 { + stamp, origin, value, madeAt = r.rec.MovedFrom.Generation, o, opened, r.rec.MadeAt + continue + } + if r.rec.MovedFrom.Generation != stamp || opened != value || o != origin { + return "", "", "", time.Time{}, fmt.Sprintf("the rows of one provider and provision carry different "+ + "generations, values or origins (%s among them): logins of their own (ADR 0048)", r.rec.Name) + } + if r.rec.MadeAt.Before(madeAt) { + madeAt = r.rec.MadeAt + } + } + return stamp, origin, value, madeAt, "" +} + +// PairPass looks at every provider and provision the plans bind or the move brought in, and makes, remakes or +// adopts each one's shared value as the rules above say. It returns what it did for each; an error is a pass that +// could not read what it needs, and did nothing. +func (v *Vault) PairPass() ([]PairOutcome, error) { + seen, err := readBindings(v.Assignments) + if err != nil { + return nil, err + } + rows, err := v.pairRows() + if err != nil { + return nil, err + } + keys := map[pairKey]bool{} + for k := range seen.bound { + keys[k] = true + } + for k := range rows { + keys[k] = true + } + all, err := v.Store.List() + if err != nil { + return nil, err + } + for _, rec := range all { + parts := strings.Split(rec.Name, "/") + if len(parts) == 3 && parts[0] == "provision" { + if _, recorded, _ := v.Store.ReadShared(rec.Name); recorded { + keys[pairKey{parts[1], parts[2]}] = true + } + } + } + ordered := make([]pairKey, 0, len(keys)) + for k := range keys { + ordered = append(ordered, k) + } + sort.Slice(ordered, func(i, j int) bool { return ordered[i].name() < ordered[j].name() }) + var out []PairOutcome + var failed []error + for _, k := range ordered { + o, err := v.pairOne(k, seen.bound[k], rows[k], seen) + if err != nil { + failed = append(failed, fmt.Errorf("%s: %w", k.name(), err)) + o = PairOutcome{Name: k.name(), Outcome: PairHeld, Why: err.Error()} + } + out = append(out, o) + } + return out, errors.Join(failed...) +} + +// pairOne is one provider and provision's part of a pass. +func (v *Vault) pairOne(k pairKey, b *binds, rows []pairRow, seen bindings) (PairOutcome, error) { + name := k.name() + if b != nil && b.perHolder { + return PairOutcome{Name: name, Outcome: PairPerHolder, + Why: "a plan names a credential per holder for it (issue 757): there is no one shared value"}, nil + } + rec, held, err := v.Store.Read(name) + if err != nil { + return PairOutcome{}, err + } + sh, recorded, err := v.Store.ReadShared(name) + if err != nil { + return PairOutcome{}, err + } + // The provider's module, when no plan names it any more (its last consumer unbound), is the one recorded. + if b == nil { + b = &binds{providerModules: map[string]bool{}, consumers: map[string]bool{}} + } + if len(b.providerModules) == 0 && recorded && sh.ProviderModule != "" { + for _, m := range strings.Split(sh.ProviderModule, ",") { + b.providerModules[m] = true + } + } + providerModule, consumers := sharedHolders(k, b) + record := func(generation int, adopted string) error { + return v.Store.WriteShared(Shared{Name: name, Provider: k.provider, Provision: k.provision, + ProviderModule: providerModule, Consumers: consumers, Generation: generation, At: v.Now().UTC(), + Adopted: adopted}) + } + switch { + case !held && len(rows) > 0: + stamp, origin, value, madeAt, why := v.oneStamp(rows) + if why != "" { + // A consumer bound since the move, that has no row yet, gets a login of its own; the rows there are + // never touched (issue 771). + if form, formWhy := b.form(k); formWhy == "" && form == FormPair { + have := map[string]bool{} + for _, r := range rows { + have[strings.Join(strings.Split(r.rec.Name, "/")[3:], "/")] = true + } + var missing []string + for _, c := range consumers { + if !have[c] { + missing = append(missing, c) + } + } + if len(missing) > 0 { + if held := heldBack(k, b, rows, seen); held != "" { + return PairOutcome{Name: name, Outcome: PairHeld, Why: held}, nil + } + return v.firstLogins(k, missing) + } + } + return PairOutcome{Name: name, Outcome: PairPerPair, Why: why + "; each consumer keeps a value of its " + + "own for ever (ADR 0048, ADR 0158 §11), and a rotation remakes each one's own, never one shared value"}, nil + } + // Adopted as it is: the holders the rows name hold it, and a bind since then is a remake on the next pass. + changed, err := v.keepWith(name, value, "the vault", "", origin, keepOptions{neverReplace: true, madeAt: madeAt, + movedFrom: &MovedFrom{By: "the move's pair rows", Generation: stamp}}) + value = "" + if err != nil { + return PairOutcome{}, err + } + var fromRows []string + for _, r := range rows { + parts := strings.Split(r.rec.Name, "/") + fromRows = append(fromRows, strings.Join(parts[3:], "/")) + } + consumers = fromRows + if err := record(changed.Generation, stamp); err != nil { + return PairOutcome{}, err + } + return PairOutcome{Name: name, Outcome: PairAdopted, Generation: changed.Generation}, nil + case !held: + if len(consumers) == 0 { + return PairOutcome{Name: name, Outcome: PairUnchanged}, nil + } + if why := heldBack(k, b, rows, seen); why != "" { + return PairOutcome{Name: name, Outcome: PairHeld, Why: why}, nil + } + form, why := b.form(k) + switch { + case why != "": + return PairOutcome{Name: name, Outcome: PairHeld, Why: why}, nil + case form == FormShared: + changed, err := v.mintShared(name) + if err != nil { + return PairOutcome{}, err + } + if err := record(changed.Generation, ""); err != nil { + return PairOutcome{}, err + } + return PairOutcome{Name: name, Outcome: PairMade, Generation: changed.Generation}, nil + case form == FormPair: + return v.firstLogins(k, consumers) + case form == FormIssued: + return PairOutcome{Name: name, Outcome: PairHeld, Why: "its credential is issued by the provider's " + + "software (ADR 0305), which the vault never makes; `secret issued` brings it in"}, nil + } + return PairOutcome{Name: name, Outcome: PairHeld, Why: "its credential is given by a person (ADR 0337), " + + "which the vault never makes; `secret accept` brings it in"}, nil + case rec.Origin == OriginAccepted || rec.Origin == OriginIssued: + // Never remade: who holds it is recorded, so each gets it on request. + if recorded && slices.Equal(sh.Consumers, consumers) && sh.ProviderModule == providerModule { + return PairOutcome{Name: name, Outcome: PairUnchanged, Generation: rec.Generation}, nil + } + if err := record(rec.Generation, sh.Adopted); err != nil { + return PairOutcome{}, err + } + return PairOutcome{Name: name, Outcome: PairRecorded, Generation: rec.Generation, + Why: "a value " + rec.Origin + " is not remade (ADR 0158 §8, ADR 0305 §9); its holders are recorded"}, nil + case !recorded: + // Held but never recorded (made by a rotation before any plan bound it): who binds it now holds it. + if err := record(rec.Generation, ""); err != nil { + return PairOutcome{}, err + } + return PairOutcome{Name: name, Outcome: PairRecorded, Generation: rec.Generation}, nil + case slices.Equal(sh.Consumers, consumers): + return PairOutcome{Name: name, Outcome: PairUnchanged, Generation: rec.Generation}, nil + } + if why := heldBack(k, b, rows, seen); why != "" { + return PairOutcome{Name: name, Outcome: PairHeld, Generation: rec.Generation, Why: why}, nil + } + changed, err := v.mintShared(name) + if err != nil { + return PairOutcome{}, err + } + if err := record(changed.Generation, ""); err != nil { + return PairOutcome{}, err + } + return PairOutcome{Name: name, Outcome: PairRemade, Generation: changed.Generation}, nil +} + +// mintShared makes the provision's one credential anew, for all its holders in one act: a new generation and +// `secret-changed`, which tells every holder to ask again. +func (v *Vault) mintShared(name string) (Changed, error) { + raw := make([]byte, 30) + if _, err := rand.Read(raw); err != nil { + return Changed{}, err + } + return v.keep(name, base64.RawURLEncoding.EncodeToString(raw), "the vault", "", OriginMade) +} + +// firstLogins makes each bound consumer of a per-login provider the vault holds no row of its first login of its +// own (ADR 0048, ADR 0158 §11, issue 768): `pair////[/]`, a value +// apart for each, never one shared value. Once made they are rows, and every later pass keeps them per pair. +func (v *Vault) firstLogins(k pairKey, consumers []string) (PairOutcome, error) { + var failed []error + for _, c := range consumers { + parts := strings.SplitN(c, "/", 3) + local := "" + if len(parts) == 3 { + local = parts[2] + } + row, err := PairSecret(k.provider, k.provision, parts[0], parts[1], local) + if err != nil { + failed = append(failed, err) + continue + } + raw := make([]byte, 30) + if _, err := rand.Read(raw); err != nil { + return PairOutcome{}, err + } + if _, err := v.keepWith(row, base64.RawURLEncoding.EncodeToString(raw), "the vault", "", OriginMade, + keepOptions{neverReplace: true}); err != nil { + failed = append(failed, fmt.Errorf("%s: %w", row, err)) + } + } + if err := errors.Join(failed...); err != nil { + return PairOutcome{}, err + } + return PairOutcome{Name: k.name(), Outcome: PairMade, Why: fmt.Sprintf("a first login of its own for each of "+ + "%d consumers (ADR 0048, ADR 0158 §11)", len(consumers))}, nil +} + +// unknownForm is why the vault makes nothing for a provider and provision it has no row of while no plan names its +// offer's credential form: whether the offer shares one credential (ADR 0158) or keeps a login per consumer (ADR +// 0048) is the offer's, and a plan from a controller before issue 768 does not say which. +func unknownForm(k pairKey) string { + return fmt.Sprintf("the vault holds no row of %s from %s, and no plan says whether its offer shares one credential "+ + "(ADR 0158) or keeps a login per consumer (ADR 0048, §11); it makes nothing until it knows", k.provision, + k.provider) +} + +// rotatePerPair is the rotation of a provider whose consumers each keep a login of their own (ADR 0048, ADR 0158 +// §11): every consumer's own value is remade apart, each a value of its own, and no shared value is made. A value +// accepted or issued is left as it is. Refused whole while any holder would never get its new value. Narrowed to one +// consumer, only that consumer's own logins are remade (issue 767), and a consumer that holds none is refused. +func (v *Vault) rotatePerPair(name string, one Narrow, by string) (Changed, error) { + parts := strings.Split(name, "/") + k := pairKey{parts[1], parts[2]} + rows, err := v.pairRows() + if err != nil { + return Changed{}, err + } + if len(rows[k]) == 0 { + return Changed{}, fmt.Errorf("not rotated: the vault holds no secret named %s, and %s", name, unknownForm(k)) + } + if _, _, _, _, why := v.oneStamp(rows[k]); why == "" { + return Changed{}, fmt.Errorf("not rotated: the rows of %s share one value not adopted yet; the next pass "+ + "adopts it, then rotate it", name) + } + if why := v.rotationHeldBack(name); why != "" { + return Changed{}, fmt.Errorf("not rotated: %s", why) + } + if !slices.ContainsFunc(rows[k], one.holds) { + return Changed{}, fmt.Errorf("not rotated: %s holds no login of %s from %s", one, k.provision, k.provider) + } + out := Changed{Name: name, At: v.Now().UTC(), By: by} + var failed []error + for _, r := range rows[k] { + if !one.holds(r) { + continue + } + if r.rec.Origin == OriginAccepted || r.rec.Origin == OriginIssued { + continue + } + raw := make([]byte, 30) + if _, err := rand.Read(raw); err != nil { + return out, err + } + changed, err := v.keep(r.rec.Name, base64.RawURLEncoding.EncodeToString(raw), by, ControllerCaller, OriginMade) + if err != nil { + failed = append(failed, fmt.Errorf("%s: %w", r.rec.Name, err)) + continue + } + out.Each = append(out.Each, changed) + } + return out, errors.Join(failed...) +} + +// rotationHeldBack says why a rotation of a provision's one credential would leave a holder behind, or "". +func (v *Vault) rotationHeldBack(name string) string { + parts := strings.Split(name, "/") + if len(parts) != 3 || parts[0] != "provision" || v.Assignments.Keys == nil { + return "" + } + k := pairKey{parts[1], parts[2]} + seen, err := readBindings(v.Assignments) + if err != nil { + return err.Error() + } + rows, err := v.pairRows() + if err != nil { + return err.Error() + } + return heldBack(k, seen.bound[k], rows[k], seen) +} + +// sharedRefusal says why caller (`provisioner.`) may not have the provision's one credential name, or "": it +// is handed to the machines the vault made it for, the provider's and each bound consumer's, while the provider's +// assignment there is handed to the provisioner, or a consumer's plan there still names it from that provider. +func sharedRefusal(store *Store, read AssignmentReader, node, provider, provision string) string { + name := "provision/" + provider + "/" + provision + if store == nil { + return "the vault keeps no record of who holds " + name + } + sh, ok, err := store.ReadShared(name) + if err != nil { + return err.Error() + } + if !ok { + return fmt.Sprintf("the vault made %s for nobody yet: it is handed to the machines its provider and its "+ + "consumers run on once their plans bind it (ADR 0349 clause 8)", name) + } + if !slices.Contains(sh.holderNodes(), node) { + return fmt.Sprintf("%s is held by %s, not by a module on %s", name, strings.Join(sh.holderNodes(), ", "), node) + } + if read == nil { + return "the vault cannot read the assignments, so it cannot tell what " + node + "'s plans name" + } + if node == provider && sh.ProviderModule != "" { + for _, m := range strings.Split(sh.ProviderModule, ",") { + if raw, err := read(node + "." + m); err == nil && raw != nil { + var a plannedProvisions + if json.Unmarshal(raw, &a) == nil && a.Node == node && a.Module == m && a.Provisioning != nil { + return "" + } + } + } + } + for _, c := range sh.Consumers { + parts := strings.SplitN(c, "/", 3) + if parts[0] != node { + continue + } + raw, err := read(node + "." + parts[1]) + if err != nil || raw == nil { + continue + } + var a plannedProvisions + if json.Unmarshal(raw, &a) != nil || a.Node != node || a.Module != parts[1] || a.Provisioning == nil { + continue + } + for _, p := range a.Provisioning.Provisions { + if p.Name == "" && p.Provision == provision && p.Provider == provider { + return "" + } + } + } + return fmt.Sprintf("no module on %s that the vault made %s for is handed to the provisioner and bound to it now", + node, name) +} + +// pairPass is one minute's PairPass, saying each outcome when it differs from the last pass's, so a provider held +// back is said once and again when it changes, not every minute. +func (e *Env) pairPass() { + if e.Vault == nil { + return + } + outcomes, err := e.Vault.PairPass() + e.mu.Lock() + defer e.mu.Unlock() + if err != nil && err.Error() != e.lastPairFailure { + say("the pass over the providers' shared credentials did not finish: %v", err) + } + e.lastPairFailure = "" + if err != nil { + e.lastPairFailure = err.Error() + } + if e.pairsSaid == nil { + e.pairsSaid = map[string]string{} + } + for _, o := range outcomes { + said := o.Outcome + ": " + o.Why + if o.Outcome == PairUnchanged || e.pairsSaid[o.Name] == said { + continue + } + e.pairsSaid[o.Name] = said + if o.Why == "" { + say("%s %s (generation %d)", o.Name, o.Outcome, o.Generation) + } else { + say("%s %s: %s", o.Name, o.Outcome, o.Why) + } + } +} + +// pairRowRefusal says why caller's machine node may not have the pair row name (`pair/// +// /[/]`, a consumer's own login, ADR 0048), or "": it is handed to that consumer's machine +// while the consumer's plan there names the provision from that provider under that local name, and to the +// provider's machine while the provider module that plan names is handed to the provisioner there. Another +// consumer, on the same machine or not, is refused: each gets only its own. +func pairRowRefusal(read AssignmentReader, node string, parts []string) string { + provider, provision, consumer, module, local := parts[1], parts[2], parts[3], parts[4], "" + if len(parts) == 6 { + local = parts[5] + } + if node != consumer && node != provider { + return fmt.Sprintf("%s is the login of %s on %s from %s, and %s is neither", strings.Join(parts, "/"), + module, consumer, provider, node) + } + if read == nil { + return "the vault cannot read the assignments, so it cannot tell what " + node + "'s plans name" + } + raw, err := read(consumer + "." + module) + if err != nil || raw == nil { + return fmt.Sprintf("%s is not on %s", module, consumer) + } + var a plannedProvisions + if json.Unmarshal(raw, &a) != nil || a.Node != consumer || a.Module != module { + return fmt.Sprintf("the assignment %s.%s is not the controller's", consumer, module) + } + if a.Provisioning == nil { + return fmt.Sprintf("%s on %s is not handed to the provisioner, so its login reaches it in its declaration "+ + "(ADR 0349 clause 17)", module, consumer) + } + for _, p := range a.Provisioning.Provisions { + // An unnamed entry, or a named one of the same form (the consumer's login file, issue 779): one rule. + if (p.Name != "" && p.CredentialForm != FormPair) || p.Provision != provision || + p.Provider != provider || p.Local != local { + continue + } + if node == consumer { + return "" + } + if p.ProviderModule != "" { + if praw, err := read(provider + "." + p.ProviderModule); err == nil && praw != nil { + var pa plannedProvisions + if json.Unmarshal(praw, &pa) == nil && pa.Node == provider && pa.Provisioning != nil { + return "" + } + } + } + return fmt.Sprintf("the provider of %s on %s is not handed to the provisioner", provision, provider) + } + return fmt.Sprintf("%s's plan on %s does not name %s from %s under that name", module, consumer, provision, provider) +} diff --git a/modules/mesh-vault/cmd/mesh-vault/pair_test.go b/modules/mesh-vault/cmd/mesh-vault/pair_test.go new file mode 100644 index 00000000..4ec4ad0f --- /dev/null +++ b/modules/mesh-vault/cmd/mesh-vault/pair_test.go @@ -0,0 +1,487 @@ +package main + +// ADR 0349's provider test, the vault's part (design 57 V5, issue 677), with a stand-in provider: a consumer binding +// to it remakes its one credential without the provider asking, and a consumer unbinding remakes it for the holders +// left; every holder gets it on request after `secret-changed`; an accepted or issued value is not remade; what the +// move brought in is adopted when its rows share a generation stamp, and otherwise waits for the provider's +// rotation; and no value is made that a holder still filled the old way would never get. + +import ( + "errors" + "sort" + "strings" + "testing" + "time" + + "git.novox.be/novox/mesh-sdk/go/seal" +) + +// keys is every key of a stand-in mirror. +func (a assignments) keys() ([]string, error) { + var out []string + for k := range a { + out = append(out, k) + } + sort.Strings(out) + return out, nil +} + +// handedWith is an assignment handed to the provisioner, its plan naming each provision given as +// provision@provider/provider-module[#local][=name][+credential-form]. +func handedWith(node, module string, provisions ...string) string { + var entries []string + for _, p := range provisions { + form := "" + if i := strings.Index(p, "+"); i >= 0 { + p, form = p[:i], p[i+1:] + } + name := "" + if i := strings.Index(p, "="); i >= 0 { + p, name = p[:i], p[i+1:] + } + local := "" + if i := strings.Index(p, "#"); i >= 0 { + p, local = p[:i], p[i+1:] + } + provision, rest, _ := strings.Cut(p, "@") + provider, providerModule, _ := strings.Cut(rest, "/") + e := `{"provision":"` + provision + `","provider":"` + provider + `","provider-module":"` + providerModule + `"` + if local != "" { + e += `,"local":"` + local + `"` + } + if name != "" { + e += `,"name":"` + name + `","path":"/s/` + name + `"` + } + if form != "" { + e += `,"credential-form":"` + form + `"` + } + entries = append(entries, e+"}") + } + return `{"node":"` + node + `","module":"` + module + `","provisioning":{"templates":{"source":"x","digest":"y"},` + + `"files":[],"provisions":[` + strings.Join(entries, ",") + `]}}` +} + +// unmoved is an assignment the controller has not handed to the provisioner. +func unmoved(node, module string) string { + return `{"node":"` + node + `","module":"` + module + `"}` +} + +const standIn = "provision/anchor/stand-in-api" + +// pairVault is a vault reading mirror for its binds and its rule. +func pairVault(t *testing.T, mirror assignments) testVault { + t.Helper() + v := newVault(t, nil) + v.Askers = Declared{Read: mirror.read, Store: v.Store} + v.Assignments = Mirror{Keys: mirror.keys, Read: mirror.read} + return v +} + +// pass runs one pass and is the outcome for name. +func pass(t *testing.T, v testVault, name string) PairOutcome { + t.Helper() + outcomes, err := v.PairPass() + if err != nil { + t.Fatal(err) + } + for _, o := range outcomes { + if o.Name == name { + return o + } + } + t.Fatalf("the pass said nothing of %s: %+v", name, outcomes) + return PairOutcome{} +} + +// asks is what caller is handed for name, opened, or the refusal. +func asks(v testVault, caller, name string) (string, int, error) { + key, _ := seal.GenerateKeyPair() + a, err := v.Get(caller, name, key.PublicKey) + if err != nil { + return "", 0, err + } + got, err := seal.Open(a.Sealed, key.PrivateKey) + return got, a.Generation, err +} + +// changedSaid is how many times `secret-changed` was said for name, and that none carried a value. +func changedSaid(t *testing.T, v testVault, name, value string) int { + t.Helper() + v.bus.mu.Lock() + defer v.bus.mu.Unlock() + n := 0 + for i, e := range v.bus.events { + if value != "" && strings.Contains(v.bus.raw[i], value) { + t.Fatalf("the bus carried the value: %s", v.bus.raw[i]) + } + if e["event"] == ChangedEvent && e["body"].(map[string]any)["name"] == name { + n++ + } + } + return n +} + +func TestAConsumerBindingOrUnbindingRemakesTheProvidersOneCredentialForEveryHolder(t *testing.T) { + mirror := assignments{ + "anchor.stand-in": handedWith("anchor", "stand-in"), + "g14.app": handedWith("g14", "app", "stand-in-api@anchor/stand-in"), + } + v := pairVault(t, mirror) + movedRow(t, v, "g14", "app", "shared-from-the-move", "stamp-1", OriginMade) + + // The move's shared value is adopted, and said to have changed in the vault. + if o := pass(t, v, standIn); o.Outcome != PairAdopted || o.Generation != 1 { + t.Fatalf("the first pass: %+v", o) + } + first, _, err := asks(v, "provisioner.anchor", standIn) + if err != nil { + t.Fatalf("the provider's machine was refused its credential: %v", err) + } + if got, _, err := asks(v, "provisioner.g14", standIn); err != nil || got != first { + t.Fatalf("the consumer got %q (%v), the provider %q", got, err, first) + } + if first != "shared-from-the-move" || changedSaid(t, v, standIn, "") != 1 { + t.Fatal("the adopted value is not the move's, or secret-changed was not said once for it") + } + if _, _, err := asks(v, "provisioner.ace", standIn); err == nil { + t.Fatal("a machine holding nothing of it was handed the credential") + } + + // Nothing changed: nothing is remade. + if o := pass(t, v, standIn); o.Outcome != PairUnchanged { + t.Fatalf("a pass with no bind or unbind: %+v", o) + } + + // A second consumer binds: the vault remakes it without the provider asking, one new value for all three. + mirror["ace.other"] = handedWith("ace", "other", "stand-in-api@anchor/stand-in") + if o := pass(t, v, standIn); o.Outcome != PairRemade || o.Generation != 2 { + t.Fatalf("a consumer binding: %+v", o) + } + second, gen, _ := asks(v, "provisioner.anchor", standIn) + if second == first || gen != 2 { + t.Fatal("the value was not remade when a consumer bound") + } + for _, c := range []string{"provisioner.g14", "provisioner.ace"} { + if got, _, err := asks(v, c, standIn); err != nil || got != second { + t.Fatalf("%s got %q (%v) after the bind, not the one new value", c, got, err) + } + } + if changedSaid(t, v, standIn, "") != 2 { + t.Fatal("secret-changed was not said for the remade value") + } + + // It unbinds (its plan no longer names it): remade for the holders left, and it is handed nothing more. + mirror["ace.other"] = handedWith("ace", "other") + if o := pass(t, v, standIn); o.Outcome != PairRemade || o.Generation != 3 { + t.Fatalf("a consumer unbinding: %+v", o) + } + third, _, _ := asks(v, "provisioner.anchor", standIn) + if got, _, _ := asks(v, "provisioner.g14", standIn); third == second || got != third { + t.Fatal("the holders left do not share one new value after the unbind") + } + if _, _, err := asks(v, "provisioner.ace", standIn); err == nil { + t.Fatal("a consumer that unbound was still handed the credential") + } + + // Its assignment goes: an unbind too. + delete(mirror, "g14.app") + if o := pass(t, v, standIn); o.Outcome != PairRemade { + t.Fatalf("a consumer taken off its machine: %+v", o) + } + if _, _, err := asks(v, "provisioner.g14", standIn); err == nil { + t.Fatal("a machine whose consumer left was still handed the credential") + } +} + +// Two local names of one provision on one consumer are two binds of the same value; a credential per holder (issue +// 757) has no one shared value. +func TestLocalNamesAreBindsAndAPerHolderCredentialIsNotShared(t *testing.T) { + mirror := assignments{ + "anchor.stand-in": handedWith("anchor", "stand-in"), + "g14.app": handedWith("g14", "app", "stand-in-api@anchor/stand-in#reader"), + "novox.gitea": handedWith("novox", "gitea"), + "g14.claude-code": handedWith("g14", "claude-code", "gitea-read@novox/gitea=gitea-read"), + } + v := pairVault(t, mirror) + movedRow(t, v, "g14", "app", "shared", "stamp-1", OriginMade, "reader") + pass(t, v, standIn) + mirror["g14.app"] = handedWith("g14", "app", "stand-in-api@anchor/stand-in#reader", + "stand-in-api@anchor/stand-in#writer") + if o := pass(t, v, standIn); o.Outcome != PairRemade { + t.Fatalf("a second local name binding: %+v", o) + } + if o := pass(t, v, "provision/novox/gitea-read"); o.Outcome != PairPerHolder { + t.Fatalf("a credential per holder: %+v", o) + } + if _, held, _ := v.Store.Read("provision/novox/gitea-read"); held { + t.Fatal("a shared value was made for a credential issued per holder") + } +} + +// An accepted value is opened by the vault alone and not remade (ADR 0158 §8), nor is one the provider's software +// issued (ADR 0305 §9): a bind records its holder, who gets the value as it is. +func TestAnAcceptedOrIssuedValueIsNotRemadeOnABind(t *testing.T) { + for _, origin := range []string{OriginAccepted, OriginIssued} { + t.Run(origin, func(t *testing.T) { + mirror := assignments{ + "anchor.stand-in": handedWith("anchor", "stand-in"), + "g14.app": handedWith("g14", "app", "stand-in-api@anchor/stand-in"), + } + v := pairVault(t, mirror) + if _, err := v.keep(standIn, "given-by-a-person", "person.jochen", ControllerCaller, origin); err != nil { + t.Fatal(err) + } + if o := pass(t, v, standIn); o.Outcome != PairRecorded || o.Generation != 1 { + t.Fatalf("the first pass: %+v", o) + } + mirror["ace.other"] = handedWith("ace", "other", "stand-in-api@anchor/stand-in") + if o := pass(t, v, standIn); o.Outcome != PairRecorded || o.Generation != 1 { + t.Fatalf("a bind to a value %s: %+v", origin, o) + } + for _, c := range []string{"provisioner.anchor", "provisioner.g14", "provisioner.ace"} { + if got, gen, err := asks(v, c, standIn); err != nil || got != "given-by-a-person" || gen != 1 { + t.Fatalf("%s got %q generation %d (%v)", c, got, gen, err) + } + } + }) + } +} + +// movedRow is a pair row as the move (V3) keeps it: its value, the controller's generation stamp and origin. +func movedRow(t *testing.T, v testVault, consumer, module, value, stamp, origin string, local ...string) { + t.Helper() + name, err := PairSecret("anchor", "stand-in-api", consumer, module, strings.Join(local, "")) + if err != nil { + t.Fatal(err) + } + if _, err := v.keepWith(name, value, "node."+consumer, "", origin, keepOptions{neverReplace: true, + madeAt: time.Date(2026, 9, 1, 0, 0, 0, 0, time.UTC), + movedFrom: &MovedFrom{By: "node." + consumer, Generation: stamp}}); err != nil { + t.Fatal(err) + } +} + +// Rows a provider shares carry one generation stamp (ADR 0158): the vault adopts their value as the provision's one +// credential, keeps its origin and when it was made, and remakes nothing, so the move rotates no credential. +func TestPairRowsSharingAGenerationStampAreAdoptedAndNothingIsRemade(t *testing.T) { + mirror := assignments{ + "anchor.stand-in": handedWith("anchor", "stand-in"), + "g14.app": handedWith("g14", "app", "stand-in-api@anchor/stand-in"), + "ace.other": handedWith("ace", "other", "stand-in-api@anchor/stand-in"), + } + v := pairVault(t, mirror) + movedRow(t, v, "g14", "app", "the-value-they-share", "20260901T000000-1", OriginMade) + movedRow(t, v, "ace", "other", "the-value-they-share", "20260901T000000-1", OriginMade) + + if o := pass(t, v, standIn); o.Outcome != PairAdopted || o.Generation != 1 { + t.Fatalf("stamped rows: %+v", o) + } + for _, c := range []string{"provisioner.anchor", "provisioner.g14", "provisioner.ace"} { + if got, _, err := asks(v, c, standIn); err != nil || got != "the-value-they-share" { + t.Fatalf("%s got %q (%v), not the value it had", c, got, err) + } + } + rec, _, _ := v.Store.Read(standIn) + if rec.Origin != OriginMade || !rec.MadeAt.Equal(time.Date(2026, 9, 1, 0, 0, 0, 0, time.UTC)) || + rec.MovedFrom == nil || rec.MovedFrom.Generation != "20260901T000000-1" { + t.Fatalf("the adopted value does not say where it came from: %+v", rec) + } + if o := pass(t, v, standIn); o.Outcome != PairUnchanged { + t.Fatalf("the pass after the adoption: %+v", o) + } + if got, gen, _ := asks(v, "provisioner.g14", standIn); got != "the-value-they-share" || gen != 1 { + t.Fatal("the adopted value was remade with no bind or unbind") + } +} + +// Rows of a provider with a login per consumer (ADR 0048) carry no stamp: each consumer keeps a value of its own +// for ever (ADR 0158 §11, issue 677's note 12). A bind makes no shared value, and nor does a rotation. +func TestUnstampedPairRowsAreNeverMadeOneSharedValue(t *testing.T) { + mirror := assignments{ + "anchor.stand-in": handedWith("anchor", "stand-in"), + "g14.app": handedWith("g14", "app", "stand-in-api@anchor/stand-in"), + "ace.other": handedWith("ace", "other", "stand-in-api@anchor/stand-in"), + } + v := pairVault(t, mirror) + movedRow(t, v, "g14", "app", "login-of-app", "", OriginMade) + movedRow(t, v, "ace", "other", "login-of-other", "", OriginMade) + + for i := 0; i < 2; i++ { + if o := pass(t, v, standIn); o.Outcome != PairPerPair { + t.Fatalf("unstamped rows, pass %d: %+v", i, o) + } + } + mirror["novox.third"] = handedWith("novox", "third", "stand-in-api@anchor/stand-in") + if o := pass(t, v, standIn); o.Outcome != PairPerPair { + t.Fatalf("a bind to a provider with logins of their own: %+v", o) + } + if _, err := v.Rotate(ControllerCaller, standIn, "person.jochen", "its ordinary rotation"); err != nil { + t.Fatal(err) + } + if _, held, _ := v.Store.Read(standIn); held { + t.Fatal("a shared value was made for a provider whose consumers keep logins of their own") + } + if changedSaid(t, v, standIn, "") != 0 { + t.Fatal("secret-changed was said for a shared value") + } +} + +// Two consumers of postgres-database (a login each, ADR 0048) hold different values after a rotation, each remade +// apart, and each consumer's machine is handed only its own; the provider's machine is handed both. +func TestARotationRemakesEachPostgresConsumersOwnLoginApart(t *testing.T) { + mirror := assignments{ + "novox.postgres": handedWith("novox", "postgres"), + "g14.gitea": handedWith("g14", "gitea", "postgres-database@novox/postgres"), + "ace.umami": handedWith("ace", "umami", "postgres-database@novox/postgres"), + } + v := pairVault(t, mirror) + gitea, umami := "pair/novox/postgres-database/g14/gitea", "pair/novox/postgres-database/ace/umami" + for name, value := range map[string]string{gitea: "login-of-gitea", umami: "login-of-umami"} { + if _, err := v.keepWith(name, value, "node.novox", "", OriginMade, keepOptions{neverReplace: true, + movedFrom: &MovedFrom{By: "node.novox"}}); err != nil { + t.Fatal(err) + } + } + changed, err := v.Rotate(ControllerCaller, "provision/novox/postgres-database", "person.jochen", "yearly") + if err != nil || len(changed.Each) != 2 { + t.Fatalf("the rotation: %+v (%v)", changed, err) + } + ofGitea, _, err := asks(v, "provisioner.g14", gitea) + if err != nil { + t.Fatalf("gitea's machine was refused its own login: %v", err) + } + ofUmami, _, err := asks(v, "provisioner.ace", umami) + if err != nil { + t.Fatalf("umami's machine was refused its own login: %v", err) + } + if ofGitea == ofUmami || ofGitea == "login-of-gitea" || ofUmami == "login-of-umami" { + t.Fatalf("after the rotation gitea holds %q and umami %q: not two new values of their own", ofGitea, ofUmami) + } + if _, _, err := asks(v, "provisioner.g14", umami); err == nil { + t.Fatal("gitea's machine was handed umami's login") + } + if _, _, err := asks(v, "provisioner.ace", gitea); err == nil { + t.Fatal("umami's machine was handed gitea's login") + } + for name, want := range map[string]string{gitea: ofGitea, umami: ofUmami} { + if got, _, err := asks(v, "provisioner.novox", name); err != nil || got != want { + t.Fatalf("the provider's machine got %q for %s (%v)", got, name, err) + } + } + if _, held, _ := v.Store.Read("provision/novox/postgres-database"); held { + t.Fatal("a shared value was made for postgres-database") + } +} + +// A provider and provision the vault holds no row of: whether its offer shares one credential or keeps a login per +// consumer is not said where the vault reads, so it makes nothing rather than guess. +func TestAProviderTheVaultHoldsNoRowOfIsNotGuessed(t *testing.T) { + mirror := assignments{ + "anchor.stand-in": handedWith("anchor", "stand-in"), + "g14.app": handedWith("g14", "app", "stand-in-api@anchor/stand-in"), + } + v := pairVault(t, mirror) + if o := pass(t, v, standIn); o.Outcome != PairHeld || !strings.Contains(o.Why, "no plan says whether") { + t.Fatalf("a provider with no row: %+v", o) + } + if _, err := v.Rotate(ControllerCaller, standIn, "person.jochen", "why not"); err == nil { + t.Fatal("a rotation guessed the form of a provider with no row") + } + if _, held, _ := v.Store.Read(standIn); held { + t.Fatal("a value was made for a provider whose form the vault does not know") + } +} + +// The vault makes no value a holder would never get (the lead's rule on issue 677): a provider not handed to the +// provisioner, or a consumer the move brought in that is still filled the old way, holds every make and remake +// back, and a rotation too. +func TestNoValueIsMadeThatAHolderFilledTheOldWayWouldNeverGet(t *testing.T) { + t.Run("an unmoved provider", func(t *testing.T) { + mirror := assignments{ + "anchor.stand-in": unmoved("anchor", "stand-in"), + "g14.app": handedWith("g14", "app", "stand-in-api@anchor/stand-in"), + } + v := pairVault(t, mirror) + if o := pass(t, v, standIn); o.Outcome != PairHeld || !strings.Contains(o.Why, "stand-in on anchor") { + t.Fatalf("an unmoved provider: %+v", o) + } + if _, held, _ := v.Store.Read(standIn); held { + t.Fatal("a value was made that the provider would never get") + } + if _, err := v.Rotate(ControllerCaller, standIn, "person.jochen", "why not"); err == nil { + t.Fatal("a rotation made a value the provider would never get") + } + }) + t.Run("an unmoved consumer", func(t *testing.T) { + mirror := assignments{ + "anchor.stand-in": handedWith("anchor", "stand-in"), + "g14.app": handedWith("g14", "app", "stand-in-api@anchor/stand-in"), + "ace.old": unmoved("ace", "old"), + } + v := pairVault(t, mirror) + movedRow(t, v, "g14", "app", "shared", "stamp-1", OriginMade) + movedRow(t, v, "ace", "old", "shared", "stamp-1", OriginMade) + if o := pass(t, v, standIn); o.Outcome != PairAdopted { + t.Fatalf("adopting changes no value, so it is not held back: %+v", o) + } + mirror["novox.new"] = handedWith("novox", "new", "stand-in-api@anchor/stand-in") + o := pass(t, v, standIn) + if o.Outcome != PairHeld || !strings.Contains(o.Why, "old on ace") { + t.Fatalf("a bind while a consumer is filled the old way: %+v", o) + } + if got, gen, _ := asks(v, "provisioner.g14", standIn); got != "shared" || gen != 1 { + t.Fatal("the value was remade while a consumer filled the old way holds it") + } + if _, err := v.Rotate(ControllerCaller, standIn, "person.jochen", "why not"); err == nil || + !strings.Contains(err.Error(), "old on ace") { + t.Fatalf("a rotation while a consumer is filled the old way: %v", err) + } + // Once it moves, the remake the bind asked for follows. + mirror["ace.old"] = handedWith("ace", "old", "stand-in-api@anchor/stand-in") + if o := pass(t, v, standIn); o.Outcome != PairRemade { + t.Fatalf("after the consumer moved: %+v", o) + } + }) +} + +// A pass that cannot read every assignment makes nothing: a bind it missed would read as an unbind. +func TestAPassThatCannotReadTheAssignmentsMakesNothing(t *testing.T) { + mirror := assignments{ + "anchor.stand-in": handedWith("anchor", "stand-in"), + "g14.app": handedWith("g14", "app", "stand-in-api@anchor/stand-in"), + } + v := pairVault(t, mirror) + movedRow(t, v, "g14", "app", "shared", "stamp-1", OriginMade) + pass(t, v, standIn) + mirror["g14.broken"] = "not json" + if _, err := v.PairPass(); err == nil { + t.Fatal("a pass over an unreadable assignment said nothing") + } + v.Assignments.Keys = func() ([]string, error) { return nil, errors.New("the bus is away") } + if _, err := v.PairPass(); err == nil { + t.Fatal("a pass that could not list the assignments said nothing") + } + if _, gen, _ := asks(v, "provisioner.g14", standIn); gen != 1 { + t.Fatal("a pass that could not read the assignments remade the value") + } +} + +// A module only pulled in by another has a value of its own in the mirror (issue 765), naming what requires it: its +// plan binds like an assigned module's. +func TestAPulledInConsumerBindsLikeAnAssignedOne(t *testing.T) { + mirror := assignments{ + "anchor.stand-in": handedWith("anchor", "stand-in"), + "g14.app": handedWith("g14", "app", "stand-in-api@anchor/stand-in"), + } + v := pairVault(t, mirror) + movedRow(t, v, "g14", "app", "shared", "stamp-1", OriginMade) + pass(t, v, standIn) + mirror["ace.helper"] = strings.Replace(handedWith("ace", "helper", "stand-in-api@anchor/stand-in"), + `"module":"helper",`, `"module":"helper","required-by":"desk",`, 1) + if o := pass(t, v, standIn); o.Outcome != PairRemade { + t.Fatalf("a pulled-in consumer binding: %+v", o) + } + if _, _, err := asks(v, "provisioner.ace", standIn); err != nil { + t.Fatalf("the pulled-in consumer's machine was refused: %v", err) + } +} diff --git a/modules/mesh-vault/cmd/mesh-vault/perholder.go b/modules/mesh-vault/cmd/mesh-vault/perholder.go new file mode 100644 index 00000000..fc9eb88d --- /dev/null +++ b/modules/mesh-vault/cmd/mesh-vault/perholder.go @@ -0,0 +1,201 @@ +package main + +// A value a provider's software issued for each holder apart (design 57's V2c, issue 749; ADR 0349 clauses 10 and +// 11). The forge mints one read token per agent account (ADR 0365 clause 4): one value per provision, sealed to +// every holder alike, cannot carry that. So an issuing provider's tool may answer one value per holder — a holder +// being a consuming module on one machine — and the vault keeps each as that holder's own and hands it to that +// holder alone. A tool that answers one value keeps the provision's one credential, as before. +// +// Names in the store: `provision////`. + +import ( + "errors" + "fmt" + "strings" +) + +// Holder is who one issued value is for: a consuming module on one machine. +type Holder struct { + Node string `json:"node"` + Module string `json:"module"` +} + +func (h Holder) String() string { return h.Module + " on " + h.Node } + +// HolderSecret is the vault's name for the value a provider's software issued for one holder of a provision. +func HolderSecret(provider, provision string, h Holder) (string, error) { + base, err := ProvisionSecret(provider, provision) + if err != nil { + return "", err + } + if err := h.valid(); err != nil { + return "", err + } + return base + "/" + h.Node + "/" + h.Module, nil +} + +func (h Holder) valid() error { + if !namePart.MatchString(h.Node) { + return fmt.Errorf("the holder's machine %q is not a name the mesh writes", h.Node) + } + if !namePart.MatchString(h.Module) { + return fmt.Errorf("the holder's module %q is not a name the mesh writes", h.Module) + } + return nil +} + +// holderOf is the holder a secret's name is for, when it is one holder's issued value. +func holderOf(name string) (Holder, bool) { + parts := strings.Split(name, "/") + if len(parts) != 5 || parts[0] != "provision" { + return Holder{}, false + } + return Holder{Node: parts[3], Module: parts[4]}, true +} + +// isTheHolder is whether caller, the bus user the runtime proved, is the holder: its machine's provisioner client +// (`provisioner.`, an account of its own, mesh-controller internal/broker) or the module itself under an +// account of its own (`.`, ADR 0259 §8's runs-as). Any other caller is not, whatever else allows it. +func isTheHolder(caller string, h Holder) bool { + return caller == "provisioner."+h.Node || caller == h.Node+"."+h.Module +} + +// notTheHolder says why caller may not have a holder's value, or "". +func notTheHolder(caller, name string) string { + h, per := holderOf(name) + if !per || isTheHolder(caller, h) { + return "" + } + return fmt.Sprintf("%s was issued for %s alone, and is handed only to that holder: provisioner.%s or %s.%s", + name, h, h.Node, h.Node, h.Module) +} + +// checkHolders refuses a list of holders that is not names the mesh writes, each once. +func checkHolders(holders []Holder) error { + seen := map[Holder]bool{} + for _, h := range holders { + if err := h.valid(); err != nil { + return err + } + if seen[h] { + return fmt.Errorf("%s is named twice", h) + } + seen[h] = true + } + return nil +} + +// IssuedHolder is what the vault kept for one holder. Never the value. +type IssuedHolder struct { + Holder + Secret string `json:"secret"` + Generation int `json:"generation"` + Unchanged bool `json:"unchanged,omitempty"` +} + +// sealedFor is one holder's value as the tool answers it: sealed to the call's key. +type sealedFor struct { + Node string `json:"node"` + Module string `json:"module"` + Sealed string `json:"sealed"` +} + +// takePerHolder opens every holder's value the tool sealed to the call's key and keeps each as that holder's own. +// It takes the answer whole or not at all: a holder not asked about, a holder twice, a value not sealed to the call +// or an empty one keeps nothing for anybody. A holder asked about and not answered is said, and keeps nothing. +func (v *Vault) takePerHolder(caller, call, provider, provision string, asked []Holder, answered []sealedFor) (Issued, error) { + c, err := v.takeCall("issued", call) + if err != nil { + return Issued{}, err + } + defer func() { c.private = [32]byte{} }() + if len(asked) == 0 { + return Issued{}, fmt.Errorf("%s answered a value per holder, and no holder was named to it; nothing was taken", c.by) + } + want := map[Holder]bool{} + for _, h := range asked { + want[h] = true + } + type opened struct { + holder Holder + secret string + value string + } + var all []opened + got := map[Holder]bool{} + for _, a := range answered { + h := Holder{Node: a.Node, Module: a.Module} + switch { + case !want[h]: + return Issued{}, fmt.Errorf("%s answered a value for %s, which it was not asked about; nothing was taken", c.by, h) + case got[h]: + return Issued{}, fmt.Errorf("%s answered %s twice; nothing was taken", c.by, h) + } + got[h] = true + raw, err := c.openOne(a.Sealed) + if err != nil { + return Issued{}, fmt.Errorf("%s's value: %w", h, err) + } + value := strings.TrimSpace(string(raw)) + zero(raw) + if value == "" { + return Issued{}, fmt.Errorf("%s answered an empty value for %s; nothing was taken", c.by, h) + } + secret, err := HolderSecret(provider, provision, h) + if err != nil { + return Issued{}, err + } + all = append(all, opened{holder: h, secret: secret, value: value}) + } + base, _ := ProvisionSecret(provider, provision) + out := Issued{Secret: base, Unchanged: true} + var failed []error + for i := range all { + changed, err := v.keep(all[i].secret, all[i].value, c.by, caller, OriginIssued) + all[i].value = "" + if err != nil { + failed = append(failed, fmt.Errorf("%s: %w", all[i].holder, err)) + continue + } + out.Holders = append(out.Holders, IssuedHolder{Holder: all[i].holder, Secret: changed.Name, + Generation: changed.Generation, Unchanged: changed.Unchanged}) + if !changed.Unchanged { + out.Unchanged = false + } + } + for _, h := range asked { + if !got[h] { + out.NotAnswered = append(out.NotAnswered, h) + } + } + if len(failed) > 0 { + return out, fmt.Errorf("not every holder's value was kept: %w", errors.Join(failed...)) + } + return out, nil +} + +// holdersArg reads the holders the controller names, refusing anything else. +func holdersArg(a map[string]any) ([]Holder, error) { + raw, ok := a["holders"] + if !ok || raw == nil { + return nil, nil + } + list, ok := raw.([]any) + if !ok { + return nil, errors.New("holders is a list of {node, module}") + } + out := make([]Holder, 0, len(list)) + for _, item := range list { + m, ok := item.(map[string]any) + if !ok { + return nil, errors.New("holders is a list of {node, module}") + } + node, nok := m["node"].(string) + module, mok := m["module"].(string) + if !nok || !mok || len(m) != 2 { + return nil, errors.New("each holder is {node, module} and nothing else") + } + out = append(out, Holder{Node: strings.TrimSpace(node), Module: strings.TrimSpace(module)}) + } + return out, nil +} diff --git a/modules/mesh-vault/cmd/mesh-vault/perholder_bus_test.go b/modules/mesh-vault/cmd/mesh-vault/perholder_bus_test.go new file mode 100644 index 00000000..36fca76d --- /dev/null +++ b/modules/mesh-vault/cmd/mesh-vault/perholder_bus_test.go @@ -0,0 +1,284 @@ +package main + +// Issue 749's proof on a real bus: a NATS server whose users are the accounts of the mesh's parts (the vault's own, +// the provider's machine's runtime, each machine's provisioner client), each allowed only its own subjects, so the +// caller a tool sees is the one the bus proved — as the mesh's runtime does (subjects ending in the caller's token, +// which only that caller may publish; mesh-controller internal/broker). A listener on every subject keeps every byte +// that crosses. + +import ( + "encoding/base64" + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "git.novox.be/novox/mesh-sdk/go/seal" + "github.com/nats-io/nats-server/v2/server" + "github.com/nats-io/nats.go" +) + +// callerToken is a caller's name as the last token of a subject. +func callerToken(user string) string { return strings.ReplaceAll(user, ".", "_") } + +func untoken(token string) string { return strings.ReplaceAll(token, "_", ".") } + +const busConf = `listen: 127.0.0.1:-1 +authorization { + users = [ + { user: "novox.mesh-vault", password: "pw", permissions: { + publish: { allow: ["mesh.mod.gitea.call.gitea_agent_token.novox.novox_mesh-vault"] } + subscribe: { allow: ["mesh.seat.mesh-vault.call.get.*", "_INBOX.novox.mesh-vault.>"] } + allow_responses: true } } + { user: "novox.node-tools", password: "pw", permissions: { + publish: { deny: [">"] } + subscribe: { allow: ["mesh.mod.gitea.call.gitea_agent_token.novox.*"] } + allow_responses: true } } + { user: "provisioner.anchor", password: "pw", permissions: { + publish: { allow: ["mesh.seat.mesh-vault.call.get.provisioner_anchor"] } + subscribe: { allow: ["_INBOX.provisioner.anchor.>"] } } } + { user: "provisioner.laptop", password: "pw", permissions: { + publish: { allow: ["mesh.seat.mesh-vault.call.get.provisioner_laptop"] } + subscribe: { allow: ["_INBOX.provisioner.laptop.>"] } } } + { user: "tap", password: "pw", permissions: { publish: { deny: [">"] }, subscribe: { allow: [">"] } } } + ] +} +` + +type busAnswer struct { + Result json.RawMessage `json:"result,omitempty"` + Error string `json:"error,omitempty"` +} + +func connectAs(t *testing.T, url, user string) *nats.Conn { + t.Helper() + conn, err := nats.Connect(url, nats.UserInfo(user, "pw"), nats.CustomInboxPrefix("_INBOX."+user), + nats.ErrorHandler(func(*nats.Conn, *nats.Subscription, error) {})) + if err != nil { + t.Fatalf("%s cannot connect: %v", user, err) + } + t.Cleanup(conn.Close) + return conn +} + +// forgeOnABus is the forge's read tokens issued into a vault keeping rule, on a real bus: what a machine's client is +// answered when it asks over the bus with a key of its own (opened), and every byte that crossed. +func forgeOnABus(t *testing.T, rule Askers) (ask func(user, name string) (string, error), heard func() string) { + t.Helper() + conf := filepath.Join(t.TempDir(), "bus.conf") + if err := os.WriteFile(conf, []byte(busConf), 0o600); err != nil { + t.Fatal(err) + } + opts, err := server.ProcessConfigFile(conf) + if err != nil { + t.Fatal(err) + } + opts.NoLog, opts.NoSigs = true, true + s, err := server.NewServer(opts) + if err != nil { + t.Fatal(err) + } + go s.Start() + if !s.ReadyForConnections(10 * time.Second) { + t.Fatal("the bus did not come up") + } + t.Cleanup(s.Shutdown) + url := s.ClientURL() + + // Every byte that crosses. + var mu sync.Mutex + var crossed []string + tap := connectAs(t, url, "tap") + if _, err := tap.Subscribe(">", func(m *nats.Msg) { + mu.Lock() + crossed = append(crossed, m.Subject+" "+string(m.Data)) + mu.Unlock() + }); err != nil { + t.Fatal(err) + } + _ = tap.Flush() + + // The forge's machine's runtime serves the stand-in issuing tool, naming the caller the subject proves. + tool := perHolderProvider(agentTokens, nil) + runtime := connectAs(t, url, "novox.node-tools") + if _, err := runtime.Subscribe("mesh.mod.gitea.call.gitea_agent_token.novox.*", func(m *nats.Msg) { + parts := strings.Split(m.Subject, ".") + var args map[string]any + _ = json.Unmarshal(m.Data, &args) + out, err := tool(untoken(parts[len(parts)-1]), args) + answer := busAnswer{} + if err != nil { + answer.Error = err.Error() + } else { + answer.Result, _ = json.Marshal(out) + } + raw, _ := json.Marshal(answer) + _ = m.Respond(raw) + }); err != nil { + t.Fatal(err) + } + _ = runtime.Flush() + + // The vault, on its own account: it asks the tool over the bus in its own name, and answers `get` to the caller + // the subject proves. Its rule for who may ask would let both clients ask for both values: its rule for a + // holder's value is what keeps each to its own. + v := newVault(t, rule) + vaultConn := connectAs(t, url, "novox.mesh-vault") + v.Ask = func(key string, body any) (json.RawMessage, error) { + call, node, _ := strings.Cut(key, "@") + module, tl, _ := strings.Cut(call, ".") + raw, _ := json.Marshal(body) + msg, err := vaultConn.Request(fmt.Sprintf("mesh.mod.%s.call.%s.%s.%s", module, tl, node, + callerToken("novox.mesh-vault")), raw, 5*time.Second) + if err != nil { + return nil, err + } + var a busAnswer + if err := json.Unmarshal(msg.Data, &a); err != nil { + return nil, err + } + if a.Error != "" { + return nil, fmt.Errorf("%s", a.Error) + } + return a.Result, nil + } + if _, err := vaultConn.Subscribe("mesh.seat.mesh-vault.call.get.*", func(m *nats.Msg) { + parts := strings.Split(m.Subject, ".") + var args struct { + Name string `json:"name"` + PublicKey string `json:"public_key"` + } + _ = json.Unmarshal(m.Data, &args) + answer := busAnswer{} + got, err := v.Get(untoken(parts[len(parts)-1]), args.Name, args.PublicKey) + if err != nil { + answer.Error = err.Error() + } else { + answer.Result, _ = json.Marshal(got) + } + raw, _ := json.Marshal(answer) + _ = m.Respond(raw) + }); err != nil { + t.Fatal(err) + } + _ = vaultConn.Flush() + + // The controller's `secret issued` for the forge's read tokens, naming both holders. + issued, err := v.TakeIssuedFor(ControllerCaller, "gitea-read", "novox", "gitea", "gitea_agent_token", twoAgents) + if err != nil { + t.Fatal(err) + } + if len(issued.Holders) != 2 { + t.Fatalf("kept %+v", issued) + } + + // Each machine's client asks over the bus, with a key of its own. + ask = func(user, name string) (string, error) { + conn := connectAs(t, url, user) + key, err := seal.GenerateKeyPair() + if err != nil { + t.Fatal(err) + } + raw, _ := json.Marshal(map[string]string{"name": name, "public_key": key.PublicKey}) + msg, err := conn.Request("mesh.seat.mesh-vault.call.get."+callerToken(user), raw, 3*time.Second) + if err != nil { + return "", err + } + var a busAnswer + if err := json.Unmarshal(msg.Data, &a); err != nil { + t.Fatal(err) + } + if a.Error != "" { + return "", fmt.Errorf("%s", a.Error) + } + var got Answer + if err := json.Unmarshal(a.Result, &got); err != nil { + t.Fatal(err) + } + return seal.Open(got.Sealed, key.PrivateKey) + } + // No client can ask in another's name: the bus refuses laptop's client anchor's subject. + laptop := connectAs(t, url, "provisioner.laptop") + key, _ := seal.GenerateKeyPair() + raw, _ := json.Marshal(map[string]string{"name": anchorsToken, "public_key": key.PublicKey}) + if _, err := laptop.Request("mesh.seat.mesh-vault.call.get."+callerToken("provisioner.anchor"), raw, + time.Second); err == nil { + t.Error("laptop's client asked in anchor's name and was answered") + } + heard = func() string { + _ = tap.Flush() + time.Sleep(100 * time.Millisecond) + mu.Lock() + defer mu.Unlock() + return strings.Join(crossed, "\n") + } + return ask, heard +} + +// noneInTheClear fails when an issued value, or its base64, crossed the bus. +func noneInTheClear(t *testing.T, all string) { + t.Helper() + if !strings.Contains(all, "mesh.mod.gitea.call.gitea_agent_token.novox.novox_mesh-vault") || + !strings.Contains(all, "mesh.seat.mesh-vault.call.get.provisioner_anchor") { + t.Fatalf("the listener heard nothing of the calls:\n%s", all) + } + for _, token := range agentTokens { + for _, form := range []string{token, base64.StdEncoding.EncodeToString([]byte(token)), + base64.RawURLEncoding.EncodeToString([]byte(token))} { + if strings.Contains(all, form) { + t.Fatalf("an issued value crossed the bus in the clear") + } + } + } +} + +func TestOnARealBusTwoHoldersOnTwoMachinesEachGetOnlyTheirOwnIssuedValueAndNoneCrossesInTheClear(t *testing.T) { + ask, heard := forgeOnABus(t, bothMayAsk) + if got, err := ask("provisioner.anchor", anchorsToken); err != nil || got != agentTokens["anchor/claude-code"] { + t.Errorf("anchor's client got %q, %v", got, err) + } + if got, err := ask("provisioner.laptop", laptopsToken); err != nil || got != agentTokens["laptop/claude-code"] { + t.Errorf("laptop's client got %q, %v", got, err) + } + if got, err := ask("provisioner.anchor", laptopsToken); err == nil || !strings.Contains(err.Error(), "alone") { + t.Errorf("anchor's client was handed laptop's value %q: %v", got, err) + } + if got, err := ask("provisioner.laptop", anchorsToken); err == nil || !strings.Contains(err.Error(), "alone") { + t.Errorf("laptop's client was handed anchor's value %q: %v", got, err) + } + noneInTheClear(t, heard()) +} + +// Issue 757's proof on a real bus, with the rule the running vault keeps: a machine's client is handed the value the +// forge issued for its claude-code because the module's plan there names gitea-read from the forge's machine, and +// is refused another machine's and another module's, sealed to its key and never in the clear. +func TestOnARealBusTheRunningRuleHandsAClientTheIssuedValueItsModulesPlanNamesAndNoOther(t *testing.T) { + plan := `{"templates":{"source":"x","digest":"y"},"files":[],"provisions":[{"name":"gitea-read",` + + `"provision":"gitea-read","provider":"novox","path":"/var/lib/claude-code/gitea-read.secret"}]}` + mirror := assignments{ + "anchor.claude-code": `{"node":"anchor","module":"claude-code","provisioning":` + plan + `}`, + "laptop.claude-code": `{"node":"laptop","module":"claude-code","provisioning":` + plan + `}`, + "anchor.zsh": `{"node":"anchor","module":"zsh","provisioning":{"secrets":[{"name":"token"}]}}`, + } + ask, heard := forgeOnABus(t, Declared{Read: mirror.read}) + if got, err := ask("provisioner.anchor", anchorsToken); err != nil || got != agentTokens["anchor/claude-code"] { + t.Errorf("anchor's client got %q, %v", got, err) + } + if got, err := ask("provisioner.laptop", laptopsToken); err != nil || got != agentTokens["laptop/claude-code"] { + t.Errorf("laptop's client got %q, %v", got, err) + } + for _, r := range []struct{ user, name, why string }{ + {"provisioner.anchor", laptopsToken, "another machine's holder value"}, + {"provisioner.laptop", anchorsToken, "another machine's holder value"}, + {"provisioner.anchor", "provision/novox/gitea-read/anchor/zsh", "another module's, its plan naming it not"}, + } { + if got, err := ask(r.user, r.name); err == nil { + t.Errorf("%s was handed %s, %s: %q", r.user, r.name, r.why, got) + } + } + noneInTheClear(t, heard()) +} diff --git a/modules/mesh-vault/cmd/mesh-vault/perholder_test.go b/modules/mesh-vault/cmd/mesh-vault/perholder_test.go new file mode 100644 index 00000000..440e8752 --- /dev/null +++ b/modules/mesh-vault/cmd/mesh-vault/perholder_test.go @@ -0,0 +1,291 @@ +package main + +// A value a provider's software issued for each holder apart (design 57's V2c, issue 749; ADR 0349 clauses 10 and +// 11; ADR 0365 clause 4, where the forge mints one read token per agent account): the vault asks the provider's tool +// naming the holders, keeps each value the tool answers as that holder's own, and hands it, sealed, to that holder +// alone. A provider that answers one value keeps the provision's one credential, as before. + +import ( + "crypto/rand" + "encoding/base64" + "errors" + "fmt" + "strings" + "testing" + + "git.novox.be/novox/mesh-sdk/go/seal" + "golang.org/x/crypto/nacl/box" +) + +// The two agents' tokens the stand-in forge mints: one per holder, each different. +var agentTokens = map[string]string{ + "anchor/claude-code": "gta_anchor-agent-read-token-0001", + "laptop/claude-code": "gta_laptop-agent-read-token-0002", +} + +// perHolderProvider is a provider's tool that issues one value per holder (the forge's read token per agent +// account): it answers the vault's account alone, each asked holder's value sealed to the call's key. +func perHolderProvider(tokens map[string]string, asked *[]map[string]any) func(string, map[string]any) (map[string]any, error) { + return func(caller string, args map[string]any) (map[string]any, error) { + if asked != nil { + *asked = append(*asked, args) + } + if !strings.HasSuffix(caller, ".mesh-vault") { + return nil, errors.New("this tool answers the vault alone") + } + key, _ := args["seal_to"].(string) + raw, err := base64.StdEncoding.DecodeString(key) + if err != nil || len(raw) != 32 { + return nil, errors.New("seal_to is not a sealing key") + } + var to [32]byte + copy(to[:], raw) + list, _ := args["holders"].([]any) + var out []map[string]any + for _, h := range list { + m, _ := h.(map[string]any) + node, _ := m["node"].(string) + module, _ := m["module"].(string) + value, ok := tokens[node+"/"+module] + if !ok { + continue + } + sealed, err := box.SealAnonymous(nil, []byte(value), &to, rand.Reader) + if err != nil { + return nil, err + } + out = append(out, map[string]any{"node": node, "module": module, + "sealed": base64.StdEncoding.EncodeToString(sealed)}) + } + return map[string]any{"holders": out}, nil + } +} + +var twoAgents = []Holder{{Node: "anchor", Module: "claude-code"}, {Node: "laptop", Module: "claude-code"}} + +// The holders' secrets' names in the vault's store. +const ( + anchorsToken = "provision/novox/gitea-read/anchor/claude-code" + laptopsToken = "provision/novox/gitea-read/laptop/claude-code" +) + +// bothMayAsk is a stand-in rule that would let both machines' clients ask for both values: the vault's own rule +// for a holder's value is what keeps each to its own. +var bothMayAsk = AllowList{ + "provisioner.anchor": {anchorsToken, laptopsToken}, + "provisioner.laptop": {anchorsToken, laptopsToken}, + "laptop.claude-code": {anchorsToken, laptopsToken}, +} + +func forgeMesh(provider func(string, map[string]any) (map[string]any, error)) *aMesh { + return &aMesh{tools: map[string]func(string, map[string]any) (map[string]any, error){ + "gitea.gitea_agent_token@novox": provider}} +} + +// Two holders on two machines: the vault asks the tool naming both, keeps a value for each, and hands each only to +// that holder (its machine's provisioner client, or the module under an account of its own), sealed to its key. +func TestAProviderIssuesAValueForEachHolderAndEachHolderGetsOnlyItsOwn(t *testing.T) { + v := newVault(t, bothMayAsk) + var asked []map[string]any + mesh := forgeMesh(perHolderProvider(agentTokens, &asked)) + v.Ask = mesh.ask + got, err := v.TakeIssuedFor(ControllerCaller, "gitea-read", "novox", "gitea", "gitea_agent_token", twoAgents) + if err != nil { + t.Fatal(err) + } + if len(asked) != 1 || fmt.Sprint(asked[0]["holders"]) != "[map[module:claude-code node:anchor] map[module:claude-code node:laptop]]" { + t.Fatalf("the tool was asked %v", asked) + } + if len(got.Holders) != 2 || got.Holders[0].Secret != anchorsToken || got.Holders[1].Secret != laptopsToken || + got.Holders[0].Generation != 1 || got.Unchanged { + t.Fatalf("answered %+v", got) + } + if _, held, _ := v.Store.Read("provision/novox/gitea-read"); held { + t.Error("a value per holder was also kept as the provision's one credential") + } + // Each holder gets its own, by its machine's client and by itself under an account of its own. + if got := handedTo(t, v, "provisioner.anchor", anchorsToken); got != agentTokens["anchor/claude-code"] { + t.Errorf("anchor got %q", got) + } + if got := handedTo(t, v, "provisioner.laptop", laptopsToken); got != agentTokens["laptop/claude-code"] { + t.Errorf("laptop got %q", got) + } + if got := handedTo(t, v, "laptop.claude-code", laptopsToken); got != agentTokens["laptop/claude-code"] { + t.Errorf("laptop's agent got %q", got) + } + // Neither gets the other's, though the stand-in rule would allow it. + for caller, name := range map[string]string{"provisioner.laptop": anchorsToken, "provisioner.anchor": laptopsToken, + "laptop.claude-code": anchorsToken} { + if _, err := v.Get(caller, name, mustKey(t)); err == nil || !strings.Contains(err.Error(), "alone") { + t.Errorf("%s was handed %s: %v", caller, name, err) + } + } + handed, _ := v.Store.Handed(20) + refused := 0 + for _, h := range handed { + if h.Outcome == "refused" { + refused++ + } + } + if refused != 3 { + t.Errorf("%d refusals recorded, want 3: %+v", refused, handed) + } + for _, token := range agentTokens { + if strings.Contains(mesh.said(), token) || strings.Contains(everything(t, v, got, handed), token) { + t.Fatal("an issued value crossed the bus, or is in something the vault said or wrote, in the clear") + } + } + rec, _, _ := v.Store.Read(anchorsToken) + if rec.Origin != OriginIssued || rec.ChangedVia != ControllerCaller || rec.Operator.Sealed == "" { + t.Errorf("the record: %+v", rec) + } + + // The same values again write nothing; one holder's new value changes that holder alone (ADR 0305 §5). + again, err := v.TakeIssuedFor(ControllerCaller, "gitea-read", "novox", "gitea", "gitea_agent_token", twoAgents) + if err != nil || !again.Unchanged || again.Holders[0].Generation != 1 || again.Holders[1].Generation != 1 { + t.Errorf("again: %+v %v", again, err) + } + renewed := map[string]string{"anchor/claude-code": agentTokens["anchor/claude-code"], + "laptop/claude-code": "gta_laptop-agent-read-token-0003"} + v.Ask = forgeMesh(perHolderProvider(renewed, nil)).ask + moved, err := v.TakeIssuedFor(ControllerCaller, "gitea-read", "novox", "gitea", "gitea_agent_token", twoAgents) + if err != nil || moved.Unchanged || !moved.Holders[0].Unchanged || moved.Holders[1].Unchanged || + moved.Holders[1].Generation != 2 { + t.Errorf("renewed: %+v %v", moved, err) + } + // A holder's value is issued by the software: never rotated, never accepted. + if _, err := v.Rotate(ControllerCaller, laptopsToken, "jochen", "test"); err == nil { + t.Error("a holder's issued value was rotated") + } + if _, err := v.AcceptOpen(ControllerCaller, laptopsToken, "jochen", ""); err == nil { + t.Error("a holder's issued value was opened for accept") + } +} + +// A holder the tool was not asked about, a holder twice, a value not sealed to the call, or an answer of both +// shapes: nothing is kept, for any holder. A holder the tool did not answer is said, not kept. +func TestAPerHolderAnswerIsTakenWholeOrNotAtAll(t *testing.T) { + sealedTo := func(args map[string]any, value string) string { + raw, _ := base64.StdEncoding.DecodeString(args["seal_to"].(string)) + var to [32]byte + copy(to[:], raw) + s, _ := box.SealAnonymous(nil, []byte(value), &to, rand.Reader) + return base64.StdEncoding.EncodeToString(s) + } + var stranger [32]byte + _, _ = rand.Read(stranger[:]) + notToTheCall, _ := box.SealAnonymous(nil, []byte("forged"), &stranger, rand.Reader) + answers := map[string]func(args map[string]any) map[string]any{ + "a holder not asked about": func(args map[string]any) map[string]any { + return map[string]any{"holders": []any{ + map[string]any{"node": "anchor", "module": "claude-code", "sealed": sealedTo(args, "a")}, + map[string]any{"node": "ace", "module": "claude-code", "sealed": sealedTo(args, "b")}}} + }, + "a holder twice": func(args map[string]any) map[string]any { + return map[string]any{"holders": []any{ + map[string]any{"node": "anchor", "module": "claude-code", "sealed": sealedTo(args, "a")}, + map[string]any{"node": "anchor", "module": "claude-code", "sealed": sealedTo(args, "b")}}} + }, + "a value not sealed to the call": func(args map[string]any) map[string]any { + return map[string]any{"holders": []any{ + map[string]any{"node": "anchor", "module": "claude-code", "sealed": sealedTo(args, "a")}, + map[string]any{"node": "laptop", "module": "claude-code", + "sealed": base64.StdEncoding.EncodeToString(notToTheCall)}}} + }, + "both shapes": func(args map[string]any) map[string]any { + return map[string]any{"sealed": sealedTo(args, "one"), "holders": []any{ + map[string]any{"node": "anchor", "module": "claude-code", "sealed": sealedTo(args, "a")}}} + }, + "an empty value": func(args map[string]any) map[string]any { + return map[string]any{"holders": []any{ + map[string]any{"node": "anchor", "module": "claude-code", "sealed": sealedTo(args, " ")}}} + }, + } + for what, answer := range answers { + v := newVault(t, nil) + answer := answer + v.Ask = forgeMesh(func(_ string, args map[string]any) (map[string]any, error) { return answer(args), nil }).ask + if _, err := v.TakeIssuedFor(ControllerCaller, "gitea-read", "novox", "gitea", "gitea_agent_token", twoAgents); err == nil { + t.Errorf("%s was taken", what) + } + if all, _ := v.Store.List(); len(all) != 0 { + t.Errorf("%s kept %d values", what, len(all)) + } + if len(v.pending) != 0 { + t.Errorf("%s left a call's key kept", what) + } + } + // Values per holder when the controller named no holder: refused, nothing kept. + v := newVault(t, nil) + v.Ask = forgeMesh(func(_ string, args map[string]any) (map[string]any, error) { + return map[string]any{"holders": []any{ + map[string]any{"node": "anchor", "module": "claude-code", "sealed": sealedTo(args, "a")}}}, nil + }).ask + if _, err := v.TakeIssuedFor(ControllerCaller, "gitea-read", "novox", "gitea", "gitea_agent_token", nil); err == nil { + t.Error("values per holder were taken for no holder named") + } + // A holder the tool did not answer is said and keeps nothing; the others are kept. + w := newVault(t, nil) + w.Ask = forgeMesh(perHolderProvider(map[string]string{"anchor/claude-code": "only-anchors"}, nil)).ask + got, err := w.TakeIssuedFor(ControllerCaller, "gitea-read", "novox", "gitea", "gitea_agent_token", twoAgents) + if err != nil || len(got.Holders) != 1 || len(got.NotAnswered) != 1 || got.NotAnswered[0] != twoAgents[1] { + t.Errorf("a holder not answered: %+v %v", got, err) + } + if _, held, _ := w.Store.Read(laptopsToken); held { + t.Error("a holder the tool did not answer holds a value") + } + // The holders the controller names are names the mesh writes, each once. + for _, bad := range [][]Holder{{{Node: "anchor", Module: ""}}, {{Node: "../x", Module: "claude-code"}}, + {twoAgents[0], twoAgents[0]}} { + if _, err := w.TakeIssuedFor(ControllerCaller, "gitea-read", "novox", "gitea", "gitea_agent_token", bad); err == nil { + t.Errorf("holders %v were asked for", bad) + } + } +} + +// A provider that answers one value keeps the provision's one credential, though the controller named holders: +// the per-provision path stays as it is. +func TestAProviderAnsweringOneValueKeepsTheProvisionsOneCredential(t *testing.T) { + v := newVault(t, AllowList{"provisioner.anchor": {"provision/anchor/plex-api"}}) + software := "plex-token-from-the-vendor" + v.Ask = (&aMesh{tools: map[string]func(string, map[string]any) (map[string]any, error){ + "plex.plex_issued_credential@anchor": standInProvider(&software)}}).ask + got, err := v.TakeIssuedFor(ControllerCaller, "plex-api", "anchor", "plex", "plex_issued_credential", + []Holder{{Node: "anchor", Module: "kometa"}, {Node: "anchor", Module: "tautulli"}}) + if err != nil || got.Secret != "provision/anchor/plex-api" || got.Generation != 1 || len(got.Holders) != 0 { + t.Fatalf("%+v %v", got, err) + } + if handedTo(t, v, "provisioner.anchor", "provision/anchor/plex-api") != software { + t.Error("the provision's one credential is not what the software issued") + } +} + +// The seat's verb takes the holders as the controller sends them, and refuses a list it cannot read. +func TestTheIssuedVerbTakesTheHolders(t *testing.T) { + v := newVault(t, nil) + v.Ask = forgeMesh(perHolderProvider(agentTokens, nil)).ask + defs := definitions(&Env{Vault: v.Vault}) + issued := toolNamed(t, defs, Seat+".issued") + answer, err := issued.runAs(ControllerCaller, map[string]any{"provision": "gitea-read", "provider": "novox", + "module": "gitea", "tool": "gitea_agent_token", "holders": []any{ + map[string]any{"node": "anchor", "module": "claude-code"}, + map[string]any{"node": "laptop", "module": "claude-code"}}}) + if err != nil || len(answer.(Issued).Holders) != 2 { + t.Fatalf("%+v %v", answer, err) + } + for _, bad := range []any{"anchor", []any{"anchor"}, []any{map[string]any{"node": 1}}} { + if _, err := issued.runAs(ControllerCaller, map[string]any{"provision": "gitea-read", "provider": "novox", + "module": "gitea", "tool": "gitea_agent_token", "holders": bad}); err == nil { + t.Errorf("holders %v were read", bad) + } + } +} + +func mustKey(t *testing.T) string { + t.Helper() + key, err := seal.GenerateKeyPair() + if err != nil { + t.Fatal(err) + } + return key.PublicKey +} diff --git a/modules/mesh-vault/cmd/mesh-vault/providers_test.go b/modules/mesh-vault/cmd/mesh-vault/providers_test.go new file mode 100644 index 00000000..296c13c3 --- /dev/null +++ b/modules/mesh-vault/cmd/mesh-vault/providers_test.go @@ -0,0 +1,21 @@ +package main + +import "testing" + +// A plan names, for each provision its module requires, the provider serving it (issue 760); only an entry with a +// name is a credential the machine's client fills (issue 757). The rule hands the value issued for the holder only +// for such an entry: naming a provider alone widens nothing. +func TestARuleHandsOnlyACredentialThePlanNamesNotAProviderAlone(t *testing.T) { + plan := `{"templates":{"source":"x","digest":"y"},"files":[],"provisions":[` + + `{"provision":"postgres-database","provider":"novox","provider-module":"postgres"},` + + `{"name":"gitea-read","provision":"gitea-read","provider":"novox","provider-module":"gitea",` + + `"path":"/s/gitea-read.secret"}]}` + mirror := assignments{"anchor.app": `{"node":"anchor","module":"app","provisioning":` + plan + `}`} + rule := Declared{Read: mirror.read} + if ok, why := rule.May("provisioner.anchor", "provision/novox/gitea-read/anchor/app"); !ok { + t.Fatalf("the credential the plan names was refused: %s", why) + } + if ok, _ := rule.May("provisioner.anchor", "provision/novox/postgres-database/anchor/app"); ok { + t.Fatal("a provision the plan names only with its provider was handed out") + } +} diff --git a/modules/mesh-vault/cmd/mesh-vault/pulled_in_test.go b/modules/mesh-vault/cmd/mesh-vault/pulled_in_test.go new file mode 100644 index 00000000..dade00d6 --- /dev/null +++ b/modules/mesh-vault/cmd/mesh-vault/pulled_in_test.go @@ -0,0 +1,45 @@ +package main + +import ( + "testing" + + "git.novox.be/novox/mesh-sdk/go/seal" +) + +// A module only pulled in by another on a node has a value of its own in the assignments mirror (mesh-controller +// internal/inventory AssignmentValues, issue 765): its plan, what requires it, and no time it was assigned. The +// running rule hands that node's client what the plan names, as for an assigned module, and nothing else. +func TestTheRunningVaultAnswersAMachinesClientForWhatAPulledInModulesPlanNames(t *testing.T) { + mirror := assignments{ + "g14.app": `{"node":"g14","module":"app","required-by":"desk","provisioning":` + + `{"templates":{"source":"x","digest":"y"},"files":[],"secrets":[{"name":"token"}],` + + `"provisions":[{"name":"gitea-read","provision":"gitea-read","provider":"novox"}]}}`, + "g14.kept": `{"node":"g14","module":"kept","required-by":"desk"}`, + } + v := newVault(t, Declared{Read: mirror.read}) + for _, name := range []string{"own/g14/app/token", "own/g14/app/other", "own/g14/kept/token", + "provision/novox/gitea-read/g14/app"} { + if _, err := v.Put(name, "value-of-"+name, "the test"); err != nil { + t.Fatal(err) + } + } + asker, _ := seal.GenerateKeyPair() + for _, name := range []string{"own/g14/app/token", "provision/novox/gitea-read/g14/app"} { + answer, err := v.Get("provisioner.g14", name, asker.PublicKey) + if err != nil { + t.Fatalf("the machine's client was refused %s, which its pulled-in module's plan names: %v", name, err) + } + if got, err := seal.Open(answer.Sealed, asker.PrivateKey); err != nil || got != "value-of-"+name { + t.Fatalf("the answer opens to %q (%v)", got, err) + } + } + for _, r := range []struct{ caller, name, why string }{ + {"provisioner.g14", "own/g14/app/other", "a secret the pulled-in module's plan does not name"}, + {"provisioner.g14", "own/g14/kept/token", "a pulled-in module not handed to the provisioner"}, + {"provisioner.ace", "own/g14/app/token", "another machine's client"}, + } { + if a, err := v.Get(r.caller, r.name, asker.PublicKey); err == nil { + t.Errorf("get answered %s (%s asking %s): %+v", r.why, r.caller, r.name, a) + } + } +} diff --git a/modules/mesh-vault/cmd/mesh-vault/reseal_health.go b/modules/mesh-vault/cmd/mesh-vault/reseal_health.go new file mode 100644 index 00000000..69dafd7e --- /dev/null +++ b/modules/mesh-vault/cmd/mesh-vault/reseal_health.go @@ -0,0 +1,44 @@ +package main + +// The vault's runtime health (novox/hq ADR 0366 §4): the vault re-seals its operator copies every minute +// (adoptOperatorKey), and a vault that keeps failing to is loud before secrets move into it. The node-engine +// asks vault_health on the vault's own runtime (`runtime-health` in module.json); the controller raises +// module.mesh-vault..unhealthy from it, with these words, and clears it on the first healthy answer. + +import ( + "fmt" + + "git.novox.be/novox/mesh-sdk/go/health" +) + +// FailedPassesUnhealthy is how many failed re-seal passes in a row make the vault unhealthy: one minute's +// pass can fail for a moment's reason and be finished by the next. +const FailedPassesUnhealthy = 3 + +// pass is one minute's re-seal, its outcome kept for vault_health. +func (e *Env) pass() { + err := adoptOperatorKey(e) + e.mu.Lock() + defer e.mu.Unlock() + if err == nil { + e.failedPasses, e.lastFailure = 0, "" + return + } + e.failedPasses++ + e.lastFailure = err.Error() +} + +// health is vault_health's answer. +func (e *Env) health() health.Answer { + e.mu.Lock() + defer e.mu.Unlock() + if e.failedPasses >= FailedPassesUnhealthy { + return health.Unhealthy(fmt.Sprintf("re-sealing the vault's operator copies failed on %d passes in a row "+ + "(one a minute); the last said: %s", e.failedPasses, e.lastFailure)) + } + if e.failedPasses > 0 { + return health.Healthy(fmt.Sprintf("the last %d re-seal pass(es) failed, tried again every minute; "+ + "unhealthy after %d in a row", e.failedPasses, FailedPassesUnhealthy)) + } + return health.Healthy("every re-seal pass succeeds") +} diff --git a/modules/mesh-vault/cmd/mesh-vault/reseal_health_test.go b/modules/mesh-vault/cmd/mesh-vault/reseal_health_test.go new file mode 100644 index 00000000..a93e2c71 --- /dev/null +++ b/modules/mesh-vault/cmd/mesh-vault/reseal_health_test.go @@ -0,0 +1,82 @@ +package main + +import ( + "encoding/json" + "fmt" + "strings" + "testing" + + "git.novox.be/novox/mesh-sdk/go/health" +) + +// novox/hq ADR 0366 §4: the vault's runtime health is a tool that says unhealthy once re-sealing has failed on +// three passes in a row, saying what failed and why, and healthy again after one pass succeeds. +func TestTheVaultIsUnhealthyAfterThreeFailedReSealPassesAndHealthyAfterOne(t *testing.T) { + v := newVault(t, nil) + if _, err := v.Put("pair/postgres/gitea", "x", "the test"); err != nil { + t.Fatal(err) + } + env := &Env{Vault: v.Vault} + answer := func() health.Answer { + t.Helper() + got, err := toolNamed(t, definitions(env), "vault_health").Run(map[string]any{}) + if err != nil { + t.Fatal(err) + } + raw, err := json.Marshal(got) + if err != nil { + t.Fatal(err) + } + a, err := health.Read(raw) + if err != nil { + t.Fatalf("the node-engine cannot read %s: %v", raw, err) + } + return a + } + if a := answer(); !a.Healthy { + t.Fatalf("a vault before any pass says %+v", a) + } + // A new operator key whose copies cannot be written: every pass fails. + pub, _ := operatorKey(t) + v.Store.failWrite = func(Record) error { return fmt.Errorf("disk full") } + _, _ = v.SetOperatorKey(asPEM(t, pub), "the test") + for pass := 1; pass <= 3; pass++ { + env.pass() + a := answer() + if pass < 3 && !a.Healthy { + t.Fatalf("after %d failed pass(es) the vault says %+v; it is unhealthy after three", pass, a) + } + if pass == 3 && (a.Healthy || a.State != health.StateUnhealthy || !strings.Contains(a.Why, "disk full") || + !strings.Contains(a.Why, "3 passes") || !strings.Contains(a.Why, "pair/postgres/gitea")) { + t.Fatalf("after three failed passes the vault says %+v", a) + } + } + // One pass that succeeds makes it healthy again, and the streak starts over. + v.Store.failWrite = nil + env.pass() + if a := answer(); !a.Healthy { + t.Fatalf("after a pass that re-sealed the vault says %+v", a) + } + v.Store.failWrite = func(Record) error { return fmt.Errorf("disk full") } + pub, _ = operatorKey(t) + _, _ = v.SetOperatorKey(asPEM(t, pub), "the test") + env.pass() + if a := answer(); !a.Healthy { + t.Fatalf("one failed pass after a good one says %+v", a) + } +} + +// A vault that could not be built re-seals nothing: its passes fail, saying why. +func TestABrokenVaultsPassesFail(t *testing.T) { + env := &Env{Broken: fmt.Errorf("the vault could not open its store: no such directory")} + for i := 0; i < 3; i++ { + env.pass() + } + got, err := toolNamed(t, definitions(env), "vault_health").Run(map[string]any{}) + if err != nil { + t.Fatal(err) + } + if a := got.(health.Answer); a.Healthy || !strings.Contains(a.Why, "could not open its store") { + t.Errorf("a broken vault says %+v", a) + } +} diff --git a/modules/mesh-vault/cmd/mesh-vault/retirement.go b/modules/mesh-vault/cmd/mesh-vault/retirement.go new file mode 100644 index 00000000..c614a9d1 --- /dev/null +++ b/modules/mesh-vault/cmd/mesh-vault/retirement.go @@ -0,0 +1,466 @@ +package main + +// Retirement of a holder the mesh stops asking for (novox/hq ADR 0230), in the words the controller reads: +// the `provisioner.retirement` event as mesh-controller's link.Retirement, the `provisioner_retirement` +// answer as link.RetirementState, and the acts as the SDK's harness takes them (mesh-sdk +// src/provisioner/index.ts) — the set repeated in `consumers`, a `why`, and the person's `by` and the +// `via` the controller's verbs say. testdata/retirement-*.json are those types marshalled by the controller +// at the commit its README names, and the tests hold this file's words to them. +// +// The vault is mark-only: it holds no access to switch off, so a retired holder keeps what it has until a +// person deletes it, and the mark — when and why — is in the holder's own ledger file, so a restart forgets +// nothing. + +import ( + "encoding/json" + "errors" + "fmt" + "sort" + "strings" + "sync" + "time" +) + +// EventRetirement is the one event a provider says about retirement (ADR 0230 §35). +const EventRetirement = "provisioner.retirement" + +// The rules of ADR 0230 §15 and §18. +const ( + StablePasses = 5 + StableFor = 10 * time.Minute + RetireAtOnce = 3 + RetireFraction = 0.5 + SayAgainEvery = time.Hour +) + +// RetiredConsumer is one holder in a retirement word or answer (link.RetiredConsumer). +type RetiredConsumer struct { + Consumer string `json:"consumer"` + Node string `json:"node,omitempty"` + RetiredAt string `json:"retired_at,omitempty"` + Why string `json:"why,omitempty"` + SizeBytes *int64 `json:"size_bytes,omitempty"` + Kind string `json:"kind,omitempty"` + Access string `json:"access,omitempty"` +} + +// RetirementWord is the event's body (link.Retirement). +type RetirementWord struct { + Provider string `json:"provider"` + ProviderNode string `json:"provider-node"` + Change string `json:"change"` + Consumers []RetiredConsumer `json:"consumers"` + Held int `json:"held"` + Bound string `json:"bound,omitempty"` + Why string `json:"why,omitempty"` + By string `json:"by,omitempty"` + Via string `json:"via,omitempty"` + At time.Time `json:"at"` + Since *time.Time `json:"since,omitempty"` +} + +// RetirementWaiting, RetirementRejected and RetirementState are the answer to provisioner_retirement +// (link.RetirementWaiting, link.RetirementRejected, link.RetirementState). +type RetirementWaiting struct { + Consumers []RetiredConsumer `json:"consumers"` + Since string `json:"since"` + Held int `json:"held"` +} + +type RetirementRejected struct { + Consumers []RetiredConsumer `json:"consumers"` + By string `json:"by"` + Why string `json:"why"` + At string `json:"at"` +} + +type RetirementState struct { + Resource string `json:"resource"` + Node string `json:"node"` + Held []string `json:"held"` + StablePasses int `json:"stable_passes"` + StableForSeconds int `json:"stable_for_seconds,omitempty"` + RetiresBy string `json:"retires_by,omitempty"` + Bound string `json:"bound"` + Waiting *RetirementWaiting `json:"waiting"` + Rejected *RetirementRejected `json:"rejected"` + Retired []RetiredConsumer `json:"retired"` +} + +// Act is a person's act on retirement: why, who (the person the controller proved) and through what. +type Act struct { + Why, By, Via string +} + +// Custody is the pass over the contributions file that keeps the ledger, and what it remembers between +// passes about the holders the mesh no longer asks for. +type Custody struct { + Ledger *Ledger + Receives string + // Resource is the provision the vault provides; Node the machine it runs on. + Resource string + Node string + Emit func(event string, body map[string]any) error + Now func() time.Time + Log func(format string, args ...any) + + mu sync.Mutex + key string + count int + firstSeen time.Time + asked map[string]bool + read bool + waiting *heldSet + rejected *heldSet +} + +type heldSet struct { + set []string + since time.Time + saidAt time.Time + held int + by, why string +} + +func (c *Custody) resource() string { + if c.Resource == "" { + return "secret" + } + return c.Resource +} + +// Pass records every delivered value, re-enables a retired holder asked for again, and counts the holders +// no longer asked for toward their retirement. +func (c *Custody) Pass() { + c.mu.Lock() + defer c.mu.Unlock() + asked, err := Contributions(c.Receives) + if err != nil { + c.Log("THE CONTRIBUTIONS FILE IS NOT USABLE (%s): %v — nothing is recorded or retired on this pass, and "+ + "the count toward retiring anyone starts again", c.Receives, err) + c.key, c.count = "", 0 + return + } + now := c.Now() + keep := map[string]bool{} + for _, a := range asked { + keep[a.As] = true + value, err := delivered(a) + if err != nil { + c.Log("%s: %v", a.As, err) + continue + } + held, outcome, err := c.Ledger.Record(a.As, a.Node, value, now) + if err != nil { + c.Log("%s: %v", a.As, err) + continue + } + if held.Retired != nil { + if err := c.Ledger.Mark(a.As, nil); err != nil { + c.Log("%s: asked for again, and its retirement mark could not be cleared: %v", a.As, err) + } else { + c.Log("%s: asked for again — re-enabled, its retirement mark cleared", a.As) + c.tell(RetirementWord{Change: "reenabled", Consumers: []RetiredConsumer{{Consumer: a.As, Node: a.Node}}}, now) + } + } + if outcome == Unchanged { + continue + } + event := "provisioned" + if outcome == Rotated { + event = "rotated" + } + c.Log("%s: %s (%s…, rotations %d)", held.As, outcome, held.Fingerprint[:19], held.Rotations) + c.say(event, map[string]any{"consumer": held.Consumer, "as": held.As, "fingerprint": held.Fingerprint, "rotations": held.Rotations}) + } + c.asked, c.read = keep, true + c.countUnasked(now) +} + +func (c *Custody) active() []Held { + var out []Held + for _, h := range c.Ledger.List() { + if h.Retired == nil { + out = append(out, h) + } + } + return out +} + +func (c *Custody) countUnasked(now time.Time) { + active := c.active() + var unasked []string + for _, h := range active { + if !c.asked[h.As] { + unasked = append(unasked, h.As) + } + } + key := strings.Join(unasked, ",") + if c.waiting != nil && strings.Join(c.waiting.set, ",") != key { + c.settle(c.waiting, "the set the mesh no longer asks for changed", now) + c.waiting = nil + } + if c.rejected != nil && strings.Join(c.rejected.set, ",") != key { + c.settle(c.rejected, "the set the mesh no longer asks for changed", now) + c.rejected = nil + } + switch { + case len(unasked) == 0: + c.key, c.count = "", 0 + return + case key != c.key: + c.key, c.count, c.firstSeen = key, 1, now + c.Log("no longer asked for: %s; retired once the same set holds for %d passes and %s (novox/hq ADR 0230)", + key, StablePasses, StableFor) + default: + c.count++ + } + if c.rejected != nil || c.count < StablePasses || now.Sub(c.firstSeen) < StableFor { + return + } + if overTheBound(len(unasked), len(active)) { + c.wait(unasked, len(active), now) + return + } + c.retire(unasked, len(active), fmt.Sprintf("the mesh stopped asking for it: the same answer in %d passes over %s, from %s", + c.count, now.Sub(c.firstSeen).Round(time.Second), c.firstSeen.UTC().Format(time.RFC3339)), now) +} + +func overTheBound(n, held int) bool { + return n > RetireAtOnce || (held > 1 && float64(n) > RetireFraction*float64(held)) +} + +func bound(held int) string { + return fmt.Sprintf("more than %d, or more than %d%% of the %d held", RetireAtOnce, int(RetireFraction*100), held) +} + +func (c *Custody) wait(set []string, held int, now time.Time) { + if c.waiting == nil { + c.waiting = &heldSet{set: set, since: now, held: held, saidAt: now} + c.Log("RETIREMENT WAITS FOR A PERSON: the mesh no longer asks for %d of the %d holder(s) (%s), %s. "+ + "Nothing is retired until `retire approve` or `retire reject` (novox/hq ADR 0230)", + len(set), held, strings.Join(set, ", "), bound(held)) + } else if now.Sub(c.waiting.saidAt) < SayAgainEvery { + return + } else { + c.waiting.saidAt = now + } + since := c.waiting.since.UTC() + c.tell(RetirementWord{Change: "waiting", Consumers: c.named(set), Held: c.waiting.held, Since: &since}, now) +} + +func (c *Custody) settle(s *heldSet, why string, now time.Time) { + c.Log("the set waiting on retirement (%s) is settled, nothing retired: %s", strings.Join(s.set, ", "), why) + c.tell(RetirementWord{Change: "settled", Consumers: c.named(s.set), Why: why}, now) +} + +// retire marks each holder in set retired: the ledger keeps it, and its history, until a person deletes it. +func (c *Custody) retire(set []string, held int, why string, now time.Time) []string { + at := now.UTC().Format(isoMillis) + var done []string + for _, as := range set { + if err := c.Ledger.Mark(as, &RetiredMark{At: at, Why: why}); err != nil { + c.Log("%s: could not be marked retired, asked again once its set is stable again: %v", as, err) + continue + } + done = append(done, as) + } + c.key, c.count = "", 0 + if len(done) == 0 { + return done + } + c.Log("%s: RETIRED, MARK ONLY — the vault holds no access to switch off, so each keeps what it holds until a "+ + "person deletes it with `cleanup delete` (novox/hq ADR 0230): %s", strings.Join(done, ", "), why) + c.tell(RetirementWord{Change: "retired", Consumers: c.retiredOnes(done), Held: held, Why: why}, now) + return done +} + +// named is each holder as the controller's words name it. +func (c *Custody) named(set []string) []RetiredConsumer { + out := make([]RetiredConsumer, 0, len(set)) + for _, as := range set { + h, _ := c.Ledger.Get(as) + out = append(out, RetiredConsumer{Consumer: as, Node: h.Consumer}) + } + return out +} + +func (c *Custody) retiredOnes(set []string) []RetiredConsumer { + out := make([]RetiredConsumer, 0, len(set)) + for _, as := range set { + h, _ := c.Ledger.Get(as) + out = append(out, retiredOne(h)) + } + return out +} + +func retiredOne(h Held) RetiredConsumer { + unknown := int64(-1) + r := RetiredConsumer{Consumer: h.As, Node: h.Consumer, SizeBytes: &unknown, Kind: "consumer", Access: "kept"} + if h.Retired != nil { + r.RetiredAt, r.Why = h.Retired.At, h.Retired.Why + } + return r +} + +func sameSet(a, b []string) bool { + x, y := append([]string{}, a...), append([]string{}, b...) + sort.Strings(x) + sort.Strings(y) + return strings.Join(x, ",") == strings.Join(y, ",") +} + +// Approve retires exactly the set waiting (or rejected) for a person, repeated in consumers; any other set +// is refused (ADR 0230 §20). +func (c *Custody) Approve(consumers []string, act Act) ([]string, error) { + if strings.TrimSpace(act.Why) == "" { + return nil, errors.New("approve: why is required") + } + c.mu.Lock() + defer c.mu.Unlock() + s := c.waiting + if s == nil { + s = c.rejected + } + if s == nil || !sameSet(s.set, consumers) { + return nil, fmt.Errorf("approve: nothing waits with exactly that set (%s); waiting: %s; rejected: %s. Look again", + strings.Join(consumers, ", "), setOf(c.waiting), setOf(c.rejected)) + } + now := c.Now() + held := len(c.active()) + c.Log("retirement of %s approved by %s: %s", strings.Join(s.set, ", "), someone(act.By), act.Why) + c.tell(RetirementWord{Change: "approved", Consumers: c.named(s.set), Held: held, Why: act.Why, By: act.By, Via: act.Via}, now) + set := s.set + c.waiting, c.rejected = nil, nil + return c.retire(set, held, "approved by "+someone(act.By)+": "+act.Why, now), nil +} + +// Reject keeps the waiting set active, repeated in consumers, and asks no more about it while it stays the +// same set. +func (c *Custody) Reject(consumers []string, act Act) ([]string, error) { + if strings.TrimSpace(act.Why) == "" { + return nil, errors.New("reject: why is required") + } + c.mu.Lock() + defer c.mu.Unlock() + if c.waiting == nil || !sameSet(c.waiting.set, consumers) { + return nil, fmt.Errorf("reject: nothing waits with exactly that set (%s); waiting: %s. Look again", + strings.Join(consumers, ", "), setOf(c.waiting)) + } + now := c.Now() + c.rejected = &heldSet{set: c.waiting.set, since: now, by: act.By, why: act.Why} + c.waiting = nil + c.Log("retirement of %s rejected by %s: %s; kept active while the mesh says the same", + strings.Join(c.rejected.set, ", "), someone(act.By), act.Why) + c.tell(RetirementWord{Change: "rejected", Consumers: c.named(c.rejected.set), Why: act.Why, By: act.By, Via: act.Via}, now) + return c.rejected.set, nil +} + +// Delete forgets one retired holder on a person's word, its name repeated in confirm. Refused: before a +// pass has read the contributions file, a holder that is not retired, one the contributions file asks for +// now, and any act without a why or a matching confirm. +func (c *Custody) Delete(consumer, confirm string, act Act) error { + if strings.TrimSpace(act.Why) == "" { + return errors.New("delete: why is required") + } + if confirm != consumer { + return errors.New("delete: confirm must repeat the consumer's name") + } + c.mu.Lock() + defer c.mu.Unlock() + if !c.read { + return errors.New("delete: no pass has read the contributions file yet, so whether the mesh asks for it is not known; try again in a minute") + } + h, ok := c.Ledger.Get(consumer) + if !ok { + return fmt.Errorf("delete: %s is not held here", consumer) + } + if h.Retired == nil { + return fmt.Errorf("delete: %s is active, not retired — only a retired holder is deleted", consumer) + } + asked, err := Contributions(c.Receives) + if err != nil { + return fmt.Errorf("delete: the contributions file cannot say whether the mesh asks for %s: %v", consumer, err) + } + for _, a := range asked { + if a.As == consumer { + return fmt.Errorf("delete: the mesh asks for %s again; the next pass re-enables it", consumer) + } + } + if !c.Ledger.Delete(consumer) { + return fmt.Errorf("delete: %s could not be deleted", consumer) + } + c.Log("%s: DELETED from the ledger, asked by %s: %s", consumer, someone(act.By), act.Why) + unknown := int64(-1) + c.tell(RetirementWord{Change: "deleted", Consumers: []RetiredConsumer{{Consumer: consumer, Node: h.Consumer, SizeBytes: &unknown}}, + Why: act.Why, By: act.By, Via: act.Via}, c.Now()) + return nil +} + +func someone(by string) string { + if by == "" { + return "someone" + } + return by +} + +func setOf(s *heldSet) string { + if s == nil || len(s.set) == 0 { + return "none" + } + return strings.Join(s.set, ", ") +} + +// Retirement is the answer to provisioner_retirement. +func (c *Custody) Retirement() RetirementState { + c.mu.Lock() + defer c.mu.Unlock() + st := RetirementState{Resource: c.resource(), Node: c.Node, Held: []string{}, StablePasses: StablePasses, + StableForSeconds: int(StableFor.Seconds()), RetiresBy: "mark-only", Retired: []RetiredConsumer{}} + for _, h := range c.Ledger.List() { + if h.Retired == nil { + st.Held = append(st.Held, h.As) + } else { + st.Retired = append(st.Retired, retiredOne(h)) + } + } + st.Bound = bound(len(st.Held)) + if c.waiting != nil { + st.Waiting = &RetirementWaiting{Consumers: c.named(c.waiting.set), + Since: c.waiting.since.UTC().Format(time.RFC3339), Held: c.waiting.held} + } + if c.rejected != nil { + st.Rejected = &RetirementRejected{Consumers: c.named(c.rejected.set), By: c.rejected.by, Why: c.rejected.why, + At: c.rejected.since.UTC().Format(time.RFC3339)} + } + return st +} + +// tell says one retirement word, with the provider, its machine, the bound and the time filled in. +func (c *Custody) tell(w RetirementWord, now time.Time) { + w.Provider, w.ProviderNode, w.At = c.resource(), c.Node, now.UTC() + if w.Bound == "" { + w.Bound = bound(len(c.active())) + } + if w.Consumers == nil { + w.Consumers = []RetiredConsumer{} + } + c.say(EventRetirement, wordBody(w)) +} + +// wordBody is a word as the map the SDK's Emit takes, through its JSON, so the bus carries exactly the tags. +func wordBody(w RetirementWord) map[string]any { + raw, _ := json.Marshal(w) + var m map[string]any + _ = json.Unmarshal(raw, &m) + return m +} + +func (c *Custody) say(event string, body map[string]any) { + if c.Emit == nil { + return + } + if err := c.Emit(event, body); err != nil { + c.Log("%s could not be said: %v", event, err) + } +} diff --git a/modules/mesh-vault/cmd/mesh-vault/retirement_test.go b/modules/mesh-vault/cmd/mesh-vault/retirement_test.go new file mode 100644 index 00000000..83c23abf --- /dev/null +++ b/modules/mesh-vault/cmd/mesh-vault/retirement_test.go @@ -0,0 +1,537 @@ +package main + +// ADR 0230 in the vault's custody ledger, and its words held to the controller's: testdata/retirement-*.json +// are mesh-controller's link.Retirement and link.RetirementState marshalled at dc36536 (feat/57, with #317). + +import ( + "bytes" + "encoding/json" + "fmt" + "os" + "path/filepath" + "sort" + "strings" + "testing" + "time" +) + +// custodyAt is a custody pass over a contributions file in dir, on a clock the test moves. +type custodyAt struct { + *Custody + bus *published + receives string + dir string + clock *time.Time +} + +func newCustody(t *testing.T) custodyAt { + dir := t.TempDir() + l, _ := NewLedger(filepath.Join(dir, "ledger")) + bus := &published{} + clock := time.Date(2026, 10, 14, 9, 0, 0, 0, time.UTC) + c := custodyAt{bus: bus, receives: filepath.Join(dir, "mesh.json"), dir: dir, clock: &clock} + c.Custody = &Custody{Ledger: l, Receives: c.receives, Resource: "secret", Node: "novox", Emit: bus.emit, + Now: func() time.Time { return *c.clock }, Log: t.Logf} + return c +} + +// ask writes a contributions file asking for these logins, each with its value. +func (c custodyAt) ask(values map[string]string) { + given := []map[string]string{} + for as, v := range values { + f := filepath.Join(c.dir, as+".secret") + _ = os.WriteFile(f, []byte(v), 0o600) + given = append(given, map[string]string{"node": "anchor", "as": as, "secret": f}) + } + doc, _ := json.Marshal(map[string]any{"requirement": "secret", "given": given}) + _ = os.WriteFile(c.receives, doc, 0o600) +} + +// apps is n holders named a-app, b-app, … +func apps(n int) map[string]string { + out := map[string]string{} + for i := 0; i < n; i++ { + out[fmt.Sprintf("%c-app", 'a'+i)] = fmt.Sprint(i) + } + return out +} + +func keepOnly(all map[string]string, n int) map[string]string { + names := make([]string, 0, len(all)) + for k := range all { + names = append(names, k) + } + sort.Strings(names) + out := map[string]string{} + for _, k := range names[:n] { + out[k] = all[k] + } + return out +} + +// passes runs n passes a minute apart. +func (c custodyAt) passes(n int) { + for i := 0; i < n; i++ { + c.Pass() + *c.clock = c.clock.Add(time.Minute) + } +} + +// stableFor runs StablePasses passes over an hour: past both of ADR 0230's conditions. +func (c custodyAt) stable() { + for i := 0; i < StablePasses; i++ { + c.Pass() + *c.clock = c.clock.Add(15 * time.Minute) + } +} + +func (c custodyAt) changes() []string { + var out []string + for _, e := range c.bus.events { + if e["event"] == EventRetirement { + out = append(out, e["body"].(map[string]any)["change"].(string)) + } else { + out = append(out, e["event"].(string)) + } + } + return out +} + +func (c custodyAt) retired(as string) bool { + h, _ := c.Ledger.Get(as) + return h.Retired != nil +} + +func waitingNames(st RetirementState) []string { + if st.Waiting == nil { + return nil + } + var out []string + for _, w := range st.Waiting.Consumers { + out = append(out, w.Consumer) + } + return out +} + +func act(why string) Act { return Act{Why: why, By: "person.jochen", Via: "mesh-controller"} } + +// ---- the controller's words ---------------------------------------------------------------------------- + +// paths is every key path in a JSON document, arrays read through their first element. +func paths(t *testing.T, raw []byte) map[string]bool { + t.Helper() + var v any + if err := json.Unmarshal(raw, &v); err != nil { + t.Fatal(err) + } + out := map[string]bool{} + var walk func(prefix string, v any) + walk = func(prefix string, v any) { + switch x := v.(type) { + case map[string]any: + for k, e := range x { + out[prefix+k] = true + walk(prefix+k+".", e) + } + case []any: + if len(x) > 0 { + walk(prefix, x[0]) + } + } + } + walk("", v) + return out +} + +// sameWords says every key the vault says is a key the controller's type has, and every key the controller +// needs is said. +func sameWords(t *testing.T, what string, ours []byte, golden string, needed ...string) { + t.Helper() + theirs, err := os.ReadFile(filepath.Join("testdata", golden)) + if err != nil { + t.Fatal(err) + } + want := paths(t, theirs) + got := paths(t, ours) + for p := range got { + if !want[p] { + t.Errorf("%s says %q, which the controller's type has no field for", what, p) + } + } + for _, p := range needed { + if !got[p] { + t.Errorf("%s does not say %q, which the controller reads", what, p) + } + } + // And decoding strictly into the shape of the controller's golden document's own Go type would succeed: + // the same kinds at the same keys. + var g, o any + _ = json.Unmarshal(theirs, &g) + _ = json.Unmarshal(ours, &o) + sameKinds(t, what, "", g, o) +} + +func sameKinds(t *testing.T, what, at string, want, got any) { + switch w := want.(type) { + case map[string]any: + g, ok := got.(map[string]any) + if !ok { + if got != nil { + t.Errorf("%s: %q is %T, the controller's is an object", what, at, got) + } + return + } + for k, v := range g { + if wv, known := w[k]; known { + sameKinds(t, what, at+"."+k, wv, v) + } + } + case []any: + g, ok := got.([]any) + if !ok { + if got != nil { + t.Errorf("%s: %q is %T, the controller's is a list", what, at, got) + } + return + } + if len(w) > 0 { + for _, e := range g { + sameKinds(t, what, at+"[]", w[0], e) + } + } + default: + if got != nil && fmt.Sprintf("%T", want) != fmt.Sprintf("%T", got) { + t.Errorf("%s: %q is %T, the controller's is %T", what, at, got, want) + } + } +} + +// Every retirement word the vault says is one the controller's link.ReadRetirement reads: its machine +// named, its change known, its consumers objects. +func TestTheRetirementWordsAreTheControllers(t *testing.T) { + c := newCustody(t) + all := apps(5) + c.ask(all) + c.passes(1) + c.ask(keepOnly(all, 1)) + c.stable() // waiting: four of five + st := c.Retirement() + if _, err := c.Reject(waitingNames(st), act("kept for now")); err != nil { + t.Fatal(err) + } + if _, err := c.Approve(waitingNames(st), act("they are gone")); err != nil { + t.Fatal(err) + } + c.passes(1) + if err := c.Delete("b-app", "b-app", act("gone for good")); err != nil { + t.Fatal(err) + } + c.ask(all) + c.passes(1) // re-enabled + seen := map[string]bool{} + for i, e := range c.bus.events { + if e["event"] != EventRetirement { + continue + } + raw := []byte(c.bus.raw[i]) + var outer struct { + Body json.RawMessage `json:"body"` + } + _ = json.Unmarshal(raw, &outer) + var w struct { + Change string `json:"change"` + ProviderNode string `json:"provider-node"` + Consumers []struct { + Consumer string `json:"consumer"` + } `json:"consumers"` + } + dec := json.NewDecoder(bytes.NewReader(outer.Body)) + if err := dec.Decode(&w); err != nil { + t.Fatalf("a retirement word the controller cannot read: %v: %s", err, outer.Body) + } + if w.ProviderNode != "novox" || len(w.Consumers) == 0 || w.Consumers[0].Consumer == "" { + t.Errorf("a %s word names no machine or no consumer: %s", w.Change, outer.Body) + } + seen[w.Change] = true + sameWords(t, "the "+w.Change+" word", outer.Body, "retirement-word.json", + "provider", "provider-node", "change", "consumers", "consumers.consumer", "at") + } + for _, change := range []string{"waiting", "rejected", "approved", "retired", "deleted", "reenabled"} { + if !seen[change] { + t.Errorf("no %s word was said", change) + } + } +} + +// The answer to provisioner_retirement is the controller's RetirementState, so `retire approve`, `retire +// reject` and `cleanup list` read the vault. +func TestTheRetirementAnswerIsTheControllers(t *testing.T) { + c := newCustody(t) + all := apps(5) + c.ask(all) + c.passes(1) + c.ask(keepOnly(all, 1)) + c.stable() + raw, _ := json.Marshal(c.Retirement()) + sameWords(t, "the waiting answer", raw, "retirement-state.json", "resource", "node", "held", "stable_passes", + "retires_by", "bound", "waiting", "waiting.consumers", "waiting.consumers.consumer", "waiting.since", "retired") + if _, err := c.Approve(waitingNames(c.Retirement()), act("gone")); err != nil { + t.Fatal(err) + } + raw, _ = json.Marshal(c.Retirement()) + sameWords(t, "the retired answer", raw, "retirement-state.json", "retired.consumer", "retired.retired_at", + "retired.why", "retired.access") + st := c.Retirement() + if st.RetiresBy != "mark-only" || st.Resource != "secret" || st.Node != "novox" || len(st.Retired) != 4 || + st.Retired[0].Access != "kept" { + t.Errorf("the answer is %+v", st) + } +} + +// ---- the rules, one at a time --------------------------------------------------------------------------- + +func TestTheCustodyPassRecordsAndRotates(t *testing.T) { + c := newCustody(t) + c.ask(map[string]string{"anchor-redis": "v1"}) + c.passes(1) + c.ask(map[string]string{"anchor-redis": "v2"}) + c.passes(1) + if got := strings.Join(c.changes(), ","); got != "provisioned,rotated" { + t.Errorf("announced %s", got) + } + for _, raw := range c.bus.raw { + if strings.Contains(raw, `"v1"`) || strings.Contains(raw, `"v2"`) { + t.Errorf("announced a value: %s", raw) + } + } +} + +// A contributions file that does not say who is asked for is no answer: nobody is retired. +func TestAFileThatSaysNothingRetiresNobody(t *testing.T) { + for _, doc := range []string{"{not json", `{"requirement":"secret"}`, `{"requirement":"secret","given":null}`, + `{"requirement":"redis-cache","given":[]}`} { + c := newCustody(t) + c.ask(map[string]string{"anchor-redis": "v1"}) + c.passes(1) + _ = os.WriteFile(c.receives, []byte(doc), 0o600) + c.passes(30) + if h, ok := c.Ledger.Get("anchor-redis"); !ok || h.Retired != nil { + t.Errorf("%s: the holder was retired or lost: %+v %v", doc, h, ok) + } + if _, err := Contributions(c.receives); err == nil { + t.Errorf("%s was read as an answer", doc) + } + } + if _, err := Contributions(filepath.Join(t.TempDir(), "missing.json")); err == nil { + t.Error("a missing file was read as an answer") + } +} + +// Five passes, not four, however much time has passed. +func TestRetiringTakesFivePasses(t *testing.T) { + c := newCustody(t) + c.ask(apps(3)) + c.passes(1) + c.ask(keepOnly(apps(3), 2)) + for i := 0; i < StablePasses-1; i++ { + c.Pass() + *c.clock = c.clock.Add(time.Hour) + } + if c.retired("c-app") { + t.Fatal("retired after four passes") + } + c.Pass() + if !c.retired("c-app") { + t.Fatal("not retired after five passes and hours") + } +} + +// Ten minutes, not less, however many passes. +func TestRetiringTakesTenMinutes(t *testing.T) { + c := newCustody(t) + c.ask(apps(3)) + c.passes(1) + c.ask(keepOnly(apps(3), 2)) + for i := 0; i < 20; i++ { + c.Pass() + *c.clock = c.clock.Add(25 * time.Second) + } + if c.retired("c-app") { + t.Fatal("retired before ten minutes") + } + *c.clock = c.clock.Add(5 * time.Minute) + c.Pass() + if !c.retired("c-app") { + t.Fatal("not retired after twenty passes and ten minutes") + } +} + +// More than three at once waits, though under half: 4 of 10. +func TestMoreThanThreeWaits(t *testing.T) { + c := newCustody(t) + c.ask(apps(10)) + c.passes(1) + c.ask(keepOnly(apps(10), 6)) + c.stable() + if got := waitingNames(c.Retirement()); len(got) != 4 { + t.Fatalf("four of ten did not wait: %v", got) + } + if c.retired("j-app") { + t.Fatal("retired while waiting") + } +} + +// More than half waits, though three or fewer: 2 of 3. +func TestMoreThanHalfWaits(t *testing.T) { + c := newCustody(t) + c.ask(apps(3)) + c.passes(1) + c.ask(keepOnly(apps(3), 1)) + c.stable() + if got := waitingNames(c.Retirement()); len(got) != 2 { + t.Fatalf("two of three did not wait: %v", got) + } +} + +// Within both bounds it retires without a person: 3 of 7. +func TestWithinTheBoundRetires(t *testing.T) { + c := newCustody(t) + c.ask(apps(7)) + c.passes(1) + c.ask(keepOnly(apps(7), 4)) + c.stable() + if c.Retirement().Waiting != nil || !c.retired("g-app") { + t.Fatal("three of seven did not retire") + } +} + +// A holder is retired, never deleted by a pass, keeps its history, and is re-enabled when asked again. +func TestAHolderNoLongerAskedForIsRetiredNotDeleted(t *testing.T) { + c := newCustody(t) + c.ask(apps(3)) + c.passes(1) + c.ask(keepOnly(apps(3), 2)) + c.stable() + h, ok := c.Ledger.Get("c-app") + if !ok || h.Retired == nil || h.Retired.Why == "" || h.Fingerprint != Fingerprint("2") { + t.Fatalf("%+v %v", h, ok) + } + c.ask(apps(3)) + c.Pass() + if c.retired("c-app") { + t.Error("asked for again, the holder stayed retired") + } + if got := strings.Join(c.changes(), ","); got != "provisioned,provisioned,provisioned,retired,reenabled" { + t.Errorf("announced %s", got) + } +} + +// A file that says nothing starts the count again. +func TestTheCountStartsAgain(t *testing.T) { + c := newCustody(t) + c.ask(apps(3)) + c.passes(1) + c.ask(keepOnly(apps(3), 2)) + c.passes(4) + _ = os.WriteFile(c.receives, []byte(`{"given":null}`), 0o600) + c.passes(1) + c.ask(keepOnly(apps(3), 2)) + c.passes(4) + *c.clock = c.clock.Add(time.Hour) + if c.retired("c-app") { + t.Fatal("retired though the count was broken") + } + c.passes(1) + if !c.retired("c-app") { + t.Fatal("not retired after five unbroken passes and an hour") + } +} + +// Approve and reject take exactly the set waiting, and refuse any other (ADR 0230 §20). +func TestApproveAndRejectTakeExactlyTheSetWaiting(t *testing.T) { + for _, decide := range []string{"approve", "reject"} { + c := newCustody(t) + c.ask(apps(5)) + c.passes(1) + c.ask(keepOnly(apps(5), 1)) + c.stable() + set := waitingNames(c.Retirement()) + for _, wrong := range [][]string{set[:3], append(append([]string{}, set...), "a-app"), {}} { + if _, err := c.Approve(wrong, act("x")); err == nil { + t.Errorf("approve took the set %v", wrong) + } + if _, err := c.Reject(wrong, act("x")); err == nil { + t.Errorf("reject took the set %v", wrong) + } + } + if _, err := c.Approve(set, Act{}); err == nil { + t.Error("an approval without a why was taken") + } + reversed := append([]string{}, set...) + sort.Sort(sort.Reverse(sort.StringSlice(reversed))) + if decide == "approve" { + done, err := c.Approve(reversed, act("the four apps are gone")) + if err != nil || len(done) != 4 || !c.retired("b-app") || c.retired("a-app") { + t.Fatalf("%v %v", done, err) + } + } else { + if _, err := c.Reject(reversed, act("they come back")); err != nil { + t.Fatal(err) + } + c.passes(10) + if c.retired("b-app") || c.Retirement().Waiting != nil { + t.Error("a rejected set was retired or asked about again") + } + } + } +} + +// retiredC is a custody with c-app retired, d-app active. +func retiredC(t *testing.T) custodyAt { + c := newCustody(t) + c.ask(apps(4)) + c.passes(1) + c.ask(keepOnly(apps(4), 3)) + c.stable() + if !c.retired("d-app") { + t.Fatal("d-app was not retired") + } + return c +} + +// Each refusal of a delete, one at a time. +func TestEachDeleteRefusal(t *testing.T) { + fresh := newCustody(t) + fresh.ask(apps(1)) + if err := fresh.Delete("a-app", "a-app", act("x")); err == nil || !strings.Contains(err.Error(), "no pass") { + t.Errorf("a delete before any pass read the file: %v", err) + } + c := retiredC(t) + if err := c.Delete("a-app", "a-app", act("x")); err == nil || !strings.Contains(err.Error(), "active") { + t.Errorf("an active holder: %v", err) + } + if err := c.Delete("d-app", "d-app", Act{By: "person.jochen"}); err == nil || !strings.Contains(err.Error(), "why") { + t.Errorf("no why: %v", err) + } + if err := c.Delete("d-app", "c-app", act("x")); err == nil || !strings.Contains(err.Error(), "confirm") { + t.Errorf("a confirm naming another: %v", err) + } + if err := c.Delete("nobody", "nobody", act("x")); err == nil { + t.Error("a holder not held") + } + c.ask(apps(4)) // asked for again, no pass yet + if err := c.Delete("d-app", "d-app", act("x")); err == nil || !strings.Contains(err.Error(), "asks for") { + t.Errorf("a holder the file asks for again: %v", err) + } + c.ask(keepOnly(apps(4), 3)) + if err := c.Delete("d-app", "d-app", act("gone for good")); err != nil { + t.Fatal(err) + } + if _, ok := c.Ledger.Get("d-app"); ok { + t.Error("deleted, but still in the ledger") + } + last := c.bus.events[len(c.bus.events)-1]["body"].(map[string]any) + if last["change"] != "deleted" || last["by"] != "person.jochen" || last["via"] != "mesh-controller" { + t.Errorf("the deletion said %v", last) + } +} diff --git a/modules/mesh-vault/cmd/mesh-vault/rotate_one_test.go b/modules/mesh-vault/cmd/mesh-vault/rotate_one_test.go new file mode 100644 index 00000000..71b9a85d --- /dev/null +++ b/modules/mesh-vault/cmd/mesh-vault/rotate_one_test.go @@ -0,0 +1,119 @@ +package main + +// The controller's `rotate [--consumer ] [--module ]` reaches the vault (issue 767): a +// provider's one shared value is rotated for every holder together, and it is never rotated for one consumer alone; +// a per-login provider (ADR 0158 §11) rotates the named consumer's own login and no other. + +import ( + "strings" + "testing" +) + +// postgresLogins is a vault holding a login each for gitea on g14 and umami on ace, from postgres on novox. +func postgresLogins(t *testing.T) (testVault, string, string) { + t.Helper() + v := pairVault(t, assignments{ + "novox.postgres": handedWith("novox", "postgres"), + "g14.gitea": handedWith("g14", "gitea", "postgres-database@novox/postgres"), + "ace.umami": handedWith("ace", "umami", "postgres-database@novox/postgres"), + }) + gitea, umami := "pair/novox/postgres-database/g14/gitea", "pair/novox/postgres-database/ace/umami" + for name, value := range map[string]string{gitea: "login-of-gitea", umami: "login-of-umami"} { + if _, err := v.keepWith(name, value, "node.novox", "", OriginMade, keepOptions{neverReplace: true, + movedFrom: &MovedFrom{By: "node.novox"}}); err != nil { + t.Fatal(err) + } + } + return v, gitea, umami +} + +func TestARotationNamingOneConsumerRemakesThatConsumersLoginAlone(t *testing.T) { + for _, one := range []Narrow{{Consumer: "g14"}, {ConsumerModule: "gitea"}, {Consumer: "g14", ConsumerModule: "gitea"}} { + v, gitea, umami := postgresLogins(t) + before := changedSaid(t, v, umami, "") + changed, err := v.RotateFor(ControllerCaller, "provision/novox/postgres-database", one, "person.jochen", + "suspected on g14") + if err != nil || len(changed.Each) != 1 || changed.Each[0].Name != gitea { + t.Fatalf("%+v: the rotation %+v (%v)", one, changed, err) + } + if got, _, err := asks(v, "provisioner.g14", gitea); err != nil || got == "login-of-gitea" { + t.Fatalf("%+v: gitea's login was not remade: %q (%v)", one, got, err) + } + if got, gen, err := asks(v, "provisioner.ace", umami); err != nil || got != "login-of-umami" || gen != 1 { + t.Fatalf("%+v: umami's login was touched: %q generation %d (%v)", one, got, gen, err) + } + if changedSaid(t, v, umami, "") != before { + t.Fatalf("%+v: secret-changed was said for umami's login", one) + } + } +} + +func TestARotationNamingAConsumerThatHoldsNoLoginChangesNothing(t *testing.T) { + v, gitea, umami := postgresLogins(t) + _, err := v.RotateFor(ControllerCaller, "provision/novox/postgres-database", Narrow{Consumer: "g14", + ConsumerModule: "umami"}, "person.jochen", "suspected") + if err == nil || !strings.Contains(err.Error(), "umami on g14") { + t.Fatalf("a consumer holding no login was not refused by name: %v", err) + } + for name, want := range map[string]string{gitea: "login-of-gitea", umami: "login-of-umami"} { + if got, _, _ := asks(v, "provisioner.novox", name); got != want { + t.Fatalf("%s changed on a refused rotation", name) + } + } +} + +// A shared value is one for every holder (ADR 0158): rotated, every holder gets the new one after secret-changed; it +// is never rotated for one consumer alone, which would change the value for all of them. +func TestASharedValueIsRotatedForEveryHolderAndNeverForOneConsumer(t *testing.T) { + v := pairVault(t, assignments{ + "anchor.stand-in": handedWith("anchor", "stand-in"), + "g14.app": handedWith("g14", "app", "stand-in-api@anchor/stand-in"), + "ace.other": handedWith("ace", "other", "stand-in-api@anchor/stand-in"), + }) + movedRow(t, v, "g14", "app", "the-value-they-share", "20260901T000000-1", OriginMade) + movedRow(t, v, "ace", "other", "the-value-they-share", "20260901T000000-1", OriginMade) + if o := pass(t, v, standIn); o.Outcome != PairAdopted { + t.Fatalf("stamped rows: %+v", o) + } + + if _, err := v.RotateFor(ControllerCaller, standIn, Narrow{Consumer: "g14"}, "person.jochen", "suspected"); err == nil || + !strings.Contains(err.Error(), "every holder") { + t.Fatalf("a shared value was rotated for one consumer: %v", err) + } + if got, gen, _ := asks(v, "provisioner.g14", standIn); got != "the-value-they-share" || gen != 1 { + t.Fatal("a refused rotation changed the shared value") + } + + changed, err := v.RotateFor(ControllerCaller, standIn, Narrow{}, "person.jochen", "yearly") + if err != nil || changed.Generation != 2 { + t.Fatalf("the rotation: %+v (%v)", changed, err) + } + var seen string + for _, c := range []string{"provisioner.anchor", "provisioner.g14", "provisioner.ace"} { + got, gen, err := asks(v, c, standIn) + if err != nil || got == "the-value-they-share" || gen != 2 || (seen != "" && got != seen) { + t.Fatalf("%s got %q generation %d (%v): not the one new value", c, got, gen, err) + } + seen = got + } + if changedSaid(t, v, standIn, seen) == 0 { + t.Fatal("secret-changed was not said for the shared value") + } +} + +func TestARotationOfAModulesOwnSecretNamesNoConsumer(t *testing.T) { + v := newVault(t, nil) + if _, err := v.RotateFor(ControllerCaller, "own/g14/app/key", Narrow{Consumer: "g14"}, "person.jochen", "x"); err == nil { + t.Fatal("a module's own secret was rotated with a consumer named") + } +} + +func TestTheRotateToolTakesTheConsumerAndItsModule(t *testing.T) { + n, err := narrowNamed(map[string]any{"consumer": "g14", "consumer_module": "gitea"}) + if err != nil || n != (Narrow{Consumer: "g14", ConsumerModule: "gitea"}) { + t.Fatalf("%+v (%v)", n, err) + } + if _, err := narrowNamed(map[string]any{"consumer": "g14/x"}); err == nil { + t.Fatal("a consumer that is not a name was taken") + } +} diff --git a/modules/mesh-vault/cmd/mesh-vault/store.go b/modules/mesh-vault/cmd/mesh-vault/store.go new file mode 100644 index 00000000..dd98a506 --- /dev/null +++ b/modules/mesh-vault/cmd/mesh-vault/store.go @@ -0,0 +1,443 @@ +package main + +// The vault's own store (novox/hq ADR 0349 §7, ADR 0113 §22): one file per secret on the vault's own disk, +// every value in it sealed — once to the vault's own key, the copy the vault opens to answer `get`, and once +// to the operator key, the copy the operator opens when the vault is lost (ADR 0113 §21). No database and no +// bucket: the vault is installed before any other module (ADR 0113 §14), a database login would be a +// secret only the vault makes, and a bucket holds no secret, sealed or not (ADR 0201 §13). +// +// Each file is written whole and renamed into place, so a reader never sees half a record. + +import ( + "encoding/json" + "errors" + "fmt" + "net/url" + "os" + "path/filepath" + "regexp" + "sort" + "strings" + "sync" + "time" + + "git.novox.be/novox/mesh-sdk/go/seal" +) + +// OperatorBox names the box an operator copy is sealed in: the NaCl anonymous box the operator key is made +// for today (`operator key make`) and `secret recover --from-export` opens. +const OperatorBox = "nacl-anonymous-box" + +// Record is one secret as the store keeps it: never its value. +type Record struct { + Name string `json:"name"` + Generation int `json:"generation"` + // AtRest is the value sealed to the vault's own key (the ADR 0206 box). + AtRest seal.Box `json:"at-rest"` + // Operator is the value sealed to the operator key. + Operator OperatorCopy `json:"operator"` + MadeAt time.Time `json:"made-at"` + ChangedAt time.Time `json:"changed-at"` + ChangedBy string `json:"changed-by"` + // ChangedVia is the caller the runtime proved, when it is not who ChangedBy names: the controller + // carrying a person's act (design 57's V2). Empty when they are the same. + ChangedVia string `json:"changed-via,omitempty"` + // Origin is where the value came from: made by the vault, accepted at the controller's terminal, typed at + // a desk, or issued by a provider's software (ADR 0349 clauses 8 to 10). Empty is made, as V1 wrote it. + Origin string `json:"origin,omitempty"` + // MovedFrom is, for a value the move took from the controller's store (design 57's V3), the node-engine that + // handed it in and the controller's generation for it, kept as it was. + MovedFrom *MovedFrom `json:"moved-from,omitempty"` +} + +// OperatorCopy is a value sealed to the operator key named in Key. +type OperatorCopy struct { + Box string `json:"box"` + Key string `json:"key"` + Sealed string `json:"sealed"` +} + +// Store is the vault's directory of secrets. +type Store struct { + dir string + // failWrite, when a test sets it, fails the write of a record: how a failure partway is proven. + failWrite func(Record) error + // HandedMax is the size at which the record of hand-outs is rolled (0: HandedMaxDefault). + HandedMax int64 + + handedMu sync.Mutex + rolled int + // asksMu serialises the record of asks, so two asks for one secret at once do not both pass its bounds. + asksMu sync.Mutex +} + +// The record of hand-outs is bounded: rolled at HandedMaxDefault into handed.jsonl.1 … .HandedKept, the +// oldest dropped, so a flood of asks costs at most (HandedKept+1) × HandedMaxDefault on the vault's disk. +const ( + HandedMaxDefault = 1 << 20 + HandedKept = 3 +) + +// OpenStore opens (and makes) the store at dir. +func OpenStore(dir string) (*Store, error) { + if dir == "" { + return nil, errors.New("the vault's store has no directory: MESH_VAULT_STORE is not set") + } + if err := os.MkdirAll(filepath.Join(dir, "secrets"), 0o700); err != nil { + return nil, err + } + return &Store{dir: dir}, nil +} + +var secretName = regexp.MustCompile(`^[a-z0-9][a-z0-9._-]*(/[a-z0-9][a-z0-9._-]*)*$`) + +// validName says why name is not a secret's name, or nil. A name is words joined by `/`, never a path. +func validName(name string) error { + if len(name) == 0 || len(name) > 200 || !secretName.MatchString(name) { + return fmt.Errorf("%q is not a secret's name: lower-case words joined by /, at most 200 characters", name) + } + return nil +} + +func (s *Store) pathOf(name string) string { + return filepath.Join(s.dir, "secrets", url.PathEscape(name)+".json") +} + +// Read is the record of one secret, and whether there is one. +func (s *Store) Read(name string) (Record, bool, error) { + if err := validName(name); err != nil { + return Record{}, false, err + } + raw, err := os.ReadFile(s.pathOf(name)) + if errors.Is(err, os.ErrNotExist) { + return Record{}, false, nil + } + if err != nil { + return Record{}, false, err + } + var r Record + if err := json.Unmarshal(raw, &r); err != nil { + return Record{}, false, fmt.Errorf("the record of %s is not readable: %w", name, err) + } + return r, true, nil +} + +// Write keeps one record, replacing any earlier one. +func (s *Store) Write(r Record) error { + if err := validName(r.Name); err != nil { + return err + } + if s.failWrite != nil { + if err := s.failWrite(r); err != nil { + return err + } + } + raw, err := json.MarshalIndent(r, "", " ") + if err != nil { + return err + } + return writeWhole(s.pathOf(r.Name), append(raw, '\n')) +} + +// Empty says whether the store holds no secret at all. +func (s *Store) Empty() (bool, error) { + all, err := s.List() + return len(all) == 0, err +} + +// Handing is one answer to `get`, or one refusal, as the store records it (as ADR 0277 §25 records asks): +// who asked, for what, when, and what it got — never the value. +type Handing struct { + At time.Time `json:"at"` + To string `json:"to"` + Name string `json:"name"` + Generation int `json:"generation,omitempty"` + Outcome string `json:"outcome"` + Why string `json:"why,omitempty"` +} + +// RecordHanding appends one handing to the store's record, synced before it returns. It says whether the +// record was rolled to make room. +func (s *Store) RecordHanding(h Handing) (rolled bool, err error) { + raw, err := json.Marshal(h) + if err != nil { + return false, err + } + s.handedMu.Lock() + defer s.handedMu.Unlock() + path := filepath.Join(s.dir, "handed.jsonl") + max := s.HandedBound() + if info, err := os.Stat(path); err == nil && info.Mode().IsRegular() && info.Size()+int64(len(raw))+1 > max { + if err := s.roll(path); err != nil { + return false, err + } + s.rolled++ + rolled = true + } + return rolled, appendSynced(path, append(raw, '\n')) +} + +// roll moves handed.jsonl to .1, .1 to .2 and so on, dropping what falls past HandedKept. +func (s *Store) roll(path string) error { + _ = os.Remove(fmt.Sprintf("%s.%d", path, HandedKept)) + for i := HandedKept - 1; i >= 1; i-- { + if err := os.Rename(fmt.Sprintf("%s.%d", path, i), fmt.Sprintf("%s.%d", path, i+1)); err != nil && !errors.Is(err, os.ErrNotExist) { + return err + } + } + if err := os.Rename(path, path+".1"); err != nil { + return err + } + return syncDir(s.dir) +} + +// Rolled is how many times the record of hand-outs was rolled since the vault started. +func (s *Store) Rolled() int { + s.handedMu.Lock() + defer s.handedMu.Unlock() + return s.rolled +} + +func appendSynced(path string, raw []byte) error { + f, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0o600) + if err != nil { + return err + } + if _, err := f.Write(raw); err != nil { + f.Close() + return err + } + if err := f.Sync(); err != nil { + f.Close() + return err + } + return f.Close() +} + +// Handed is the latest handings, newest first, at most limit: the current record, then the rolled ones. +func (s *Store) Handed(limit int) ([]Handing, error) { + s.handedMu.Lock() + defer s.handedMu.Unlock() + path := filepath.Join(s.dir, "handed.jsonl") + out := []Handing{} + for i := 0; i <= HandedKept && len(out) < limit; i++ { + file := path + if i > 0 { + file = fmt.Sprintf("%s.%d", path, i) + } + raw, err := os.ReadFile(file) + if errors.Is(err, os.ErrNotExist) { + continue + } + if err != nil { + return nil, err + } + lines := strings.Split(strings.TrimSpace(string(raw)), "\n") + for j := len(lines) - 1; j >= 0 && len(out) < limit; j-- { + var h Handing + if json.Unmarshal([]byte(lines[j]), &h) == nil { + out = append(out, h) + } + } + } + return out, nil +} + +// HandedBound is the size at which the record of hand-outs is rolled. +func (s *Store) HandedBound() int64 { + if s.HandedMax > 0 { + return s.HandedMax + } + return HandedMaxDefault +} + +// List is every record, by name. +func (s *Store) List() ([]Record, error) { + entries, err := os.ReadDir(filepath.Join(s.dir, "secrets")) + if err != nil { + return nil, err + } + var out []Record + for _, e := range entries { + if !strings.HasSuffix(e.Name(), ".json") { + continue + } + name, err := url.PathUnescape(strings.TrimSuffix(e.Name(), ".json")) + if err != nil { + continue + } + r, ok, err := s.Read(name) + if err != nil { + return nil, err + } + if ok { + out = append(out, r) + } + } + sort.Slice(out, func(i, j int) bool { return out[i].Name < out[j].Name }) + return out, nil +} + +type operatorFile struct { + Key string `json:"key"` + Since time.Time `json:"since"` + By string `json:"by"` +} + +// OperatorKey is the operator's public key the store seals to, or "" when it has none. +func (s *Store) OperatorKey() (string, error) { + raw, err := os.ReadFile(filepath.Join(s.dir, "operator.json")) + if errors.Is(err, os.ErrNotExist) { + return "", nil + } + if err != nil { + return "", err + } + var f operatorFile + if err := json.Unmarshal(raw, &f); err != nil { + return "", err + } + return f.Key, nil +} + +func (s *Store) setOperatorKey(key, by string, at time.Time) error { + raw, _ := json.MarshalIndent(operatorFile{Key: key, Since: at, By: by}, "", " ") + return writeWhole(filepath.Join(s.dir, "operator.json"), append(raw, '\n')) +} + +// writeWhole writes a file readable by the vault alone, synced, then renames it into place and syncs the +// directory, so a crash leaves the old file or the new one, never neither. +func writeWhole(path string, raw []byte) error { + tmp := fmt.Sprintf("%s.%d.tmp", path, os.Getpid()) + if err := writeSynced(tmp, raw); err != nil { + return err + } + if err := os.Rename(tmp, path); err != nil { + _ = os.Remove(tmp) + return err + } + return syncDir(filepath.Dir(path)) +} + +// writeSynced writes a new file at 0600 and syncs it to disk. +func writeSynced(path string, raw []byte) error { + f, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o600) + if err != nil { + return err + } + if _, err := f.Write(raw); err != nil { + f.Close() + return err + } + if err := f.Sync(); err != nil { + f.Close() + return err + } + return f.Close() +} + +// syncDir syncs a directory, so a rename or a link in it is on disk. +func syncDir(dir string) error { + d, err := os.Open(dir) + if err != nil { + return err + } + defer d.Close() + return d.Sync() +} + +// The two lines an ask leaves in the record: when it opened, and how it ended. +const ( + AskOpened = "opened" + AskEnded = "ended" +) + +// AskRecord is one line of the record of asks for a module's own secret at a desk (ADR 0277 §25, in the vault's +// store since ADR 0349 clause 9): who asked, as the bus named them, for what, at which desk, and how it ended. +// Never a value. +type AskRecord struct { + ID string `json:"id"` + At time.Time `json:"at"` + Event string `json:"event"` + Secret string `json:"secret"` + Node string `json:"node,omitempty"` + Module string `json:"module,omitempty"` + Name string `json:"name,omitempty"` + Desk string `json:"desk,omitempty"` + AskedBy string `json:"asked_by,omitempty"` + Caller string `json:"caller,omitempty"` + Outcome string `json:"outcome,omitempty"` +} + +// RecordAsk appends one line to the record of asks, synced before it returns; rolled as the record of +// hand-outs is, and it says whether it was. +func (s *Store) RecordAsk(r AskRecord) (rolled bool, err error) { + raw, err := json.Marshal(r) + if err != nil { + return false, err + } + path := filepath.Join(s.dir, "asks.jsonl") + if info, err := os.Stat(path); err == nil && info.Mode().IsRegular() && info.Size()+int64(len(raw))+1 > s.HandedBound() { + if err := s.roll(path); err != nil { + return false, err + } + rolled = true + } + return rolled, appendSynced(path, append(raw, '\n')) +} + +// asks is every line of the record of asks, oldest first, the rolled files before the current one. +func (s *Store) asks() ([]AskRecord, error) { + path := filepath.Join(s.dir, "asks.jsonl") + var out []AskRecord + for i := HandedKept; i >= 0; i-- { + file := path + if i > 0 { + file = fmt.Sprintf("%s.%d", path, i) + } + raw, err := os.ReadFile(file) + if errors.Is(err, os.ErrNotExist) { + continue + } + if err != nil { + return nil, err + } + for _, line := range strings.Split(strings.TrimSpace(string(raw)), "\n") { + var r AskRecord + if line != "" && json.Unmarshal([]byte(line), &r) == nil { + out = append(out, r) + } + } + } + return out, nil +} + +// AsksSince is every line of the record of asks at or after since, oldest first. +func (s *Store) AsksSince(since time.Time) ([]AskRecord, error) { + all, err := s.asks() + if err != nil { + return nil, err + } + var out []AskRecord + for _, r := range all { + if !r.At.Before(since) { + out = append(out, r) + } + } + return out, nil +} + +// Asks is the latest lines of the record of asks, newest first, at most limit. +func (s *Store) Asks(limit int) ([]AskRecord, error) { + s.asksMu.Lock() + defer s.asksMu.Unlock() + all, err := s.asks() + if err != nil { + return nil, err + } + out := []AskRecord{} + for i := len(all) - 1; i >= 0 && len(out) < limit; i-- { + out = append(out, all[i]) + } + return out, nil +} diff --git a/modules/mesh-vault/cmd/mesh-vault/testdata/retirement-state.json b/modules/mesh-vault/cmd/mesh-vault/testdata/retirement-state.json new file mode 100644 index 00000000..600bf79f --- /dev/null +++ b/modules/mesh-vault/cmd/mesh-vault/testdata/retirement-state.json @@ -0,0 +1,56 @@ +{ + "resource": "secret", + "node": "novox", + "held": [ + "a" + ], + "held_sizes": { + "a": 1 + }, + "stable_passes": 5, + "stable_for_seconds": 600, + "retires_by": "mark-only", + "bound": "b", + "waiting": { + "consumers": [ + { + "consumer": "c", + "node": "n", + "retired_at": "t", + "why": "w", + "size_bytes": -1, + "kind": "consumer", + "access": "kept" + } + ], + "since": "t", + "held": 1 + }, + "rejected": { + "consumers": [ + { + "consumer": "c", + "node": "n", + "retired_at": "t", + "why": "w", + "size_bytes": -1, + "kind": "consumer", + "access": "kept" + } + ], + "by": "b", + "why": "w", + "at": "t" + }, + "retired": [ + { + "consumer": "c", + "node": "n", + "retired_at": "t", + "why": "w", + "size_bytes": -1, + "kind": "consumer", + "access": "kept" + } + ] +} diff --git a/modules/mesh-vault/cmd/mesh-vault/testdata/retirement-word.json b/modules/mesh-vault/cmd/mesh-vault/testdata/retirement-word.json new file mode 100644 index 00000000..57792616 --- /dev/null +++ b/modules/mesh-vault/cmd/mesh-vault/testdata/retirement-word.json @@ -0,0 +1,23 @@ +{ + "provider": "secret", + "provider-node": "novox", + "change": "waiting", + "consumers": [ + { + "consumer": "c", + "node": "n", + "retired_at": "t", + "why": "w", + "size_bytes": -1, + "kind": "consumer", + "access": "kept" + } + ], + "held": 1, + "bound": "b", + "why": "w", + "by": "person.x", + "via": "mesh-controller", + "at": "2026-10-14T09:00:00Z", + "since": "2026-10-14T09:00:00Z" +} diff --git a/modules/mesh-vault/cmd/mesh-vault/tools.go b/modules/mesh-vault/cmd/mesh-vault/tools.go new file mode 100644 index 00000000..a737cfdf --- /dev/null +++ b/modules/mesh-vault/cmd/mesh-vault/tools.go @@ -0,0 +1,663 @@ +package main + +// What the vault serves: the seat's verb `get`, and the three tools the TypeScript vault offered — +// secret_export, secret_holders and secret_verify — answering as they did; beside them secret_handed and +// secret_vault_export over the vault's own store, and ADR 0230's four retirement tools over the custody +// ledger, and vault_status. None returns a value: `get` returns one sealed to the asker's key, the others +// fingerprints and ciphertext only. + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "strings" + "sync" + "time" + + stdio "git.novox.be/novox/mesh-sdk/go" +) + +// Env is what the tools work on. A part the module's words did not give is nil, and a tool that needs it +// says so. +type Env struct { + Vault *Vault + Ledger *Ledger + Receives string + Root string + Custody *Custody + // refused is why the operator key in the mesh's export was not taken, or empty. + mu sync.Mutex + refused string + // Broken is why the vault could not be built, said by `get`. + Broken error + // failedPasses is how many re-seal passes in a row failed, and lastFailure why the last one did + // (vault_health, novox/hq ADR 0366 §4). + failedPasses int + lastFailure string + // pairsSaid is what the last pass over the shared credentials said of each, and lastPairFailure why it last + // did not finish (pair.go): each said once, and again when it changes. + pairsSaid map[string]string + lastPairFailure string +} + +func (e *Env) keyRefused() string { + e.mu.Lock() + defer e.mu.Unlock() + return e.refused +} + +func (e *Env) refuseKey(why string) { + e.mu.Lock() + defer e.mu.Unlock() + e.refused = why +} + +// toolDef is a tool with its caller given apart from its arguments, so a test can be the runtime. +type toolDef struct { + Name string + Description string + Input map[string]any + run func(caller string, a map[string]any) (any, error) +} + +// Run answers a call the runtime made: the caller is the one it proved. +func (t toolDef) Run(a map[string]any) (any, error) { return t.run(stdio.CallerOf(a), a) } + +func (t toolDef) runAs(caller string, a map[string]any) (any, error) { return t.run(caller, a) } + +func str(description string) map[string]any { + return map[string]any{"type": "string", "description": description} +} + +func definitions(env *Env) []toolDef { + return []toolDef{ + {Name: Seat + ".get", + Description: "One secret, sealed to the public key the asker sends (the sealed box of ADR 0206), with its " + + "generation; answered only to an asker the vault allows, as the runtime proved it (ADR 0349 §11). " + + "Never the value in the clear, and never to a private key.", + Input: map[string]any{"name": str("the secret's name"), + "public_key": str("the asker's X25519 public key as SPKI PEM — never a private key")}, + run: func(caller string, a map[string]any) (any, error) { + if env.Vault == nil { + if env.Broken != nil { + return nil, env.Broken + } + return nil, errors.New("the vault is not ready") + } + name, _ := a["name"].(string) + key, _ := a["public_key"].(string) + return env.Vault.Get(caller, strings.TrimSpace(name), key) + }}, + {Name: Seat + ".accept", + Description: "Take a value given at the controller's terminal (ADR 0349 clause 8), in two steps from the " + + "controller alone: name the secret (node, module and name for a module's own; provider and provision " + + "for a provision's one credential) and get a call and a key made for it; then send the call and the " + + "value sealed to that key. The vault alone opens it, and never remakes an accepted credential. With " + + "origin issued it keeps a covered bus account the controller minted, replacing it in place.", + Input: intakeInput(map[string]string{"node": "the machine, for a module's own secret", + "module": "the module, for a module's own secret", "name": "the secret's name in the module", + "provider": "the provider's machine, for a provision's credential", "provision": "the provision", + "by": "who gave it at the terminal, as the controller names them", + "origin": "issued for a bus account the controller minted for a core or moved module and hands at " + + "once (ADR 0374, issue 787), kept as that module's own secret and replaced in place on a " + + "re-issue; left out for a value given at the terminal"}), + run: func(caller string, a map[string]any) (any, error) { + v, err := env.ready() + if err != nil { + return nil, err + } + if call := arg(a, "call"); call != "" { + return v.AcceptTake(caller, call, arg(a, "sealed")) + } + secret, err := secretNamed(a) + if err != nil { + return nil, err + } + return v.AcceptOpen(caller, secret, arg(a, "by"), arg(a, "origin")) + }}, + {Name: Seat + ".secret-ask", + Description: "Ask the operator for a module's own secret at a desk (ADR 0349 clause 9, ADR 0277 §22–§27), " + + "asked by the controller alone: node, module, name, desk and asked_by. The ask is recorded before any " + + "prompt opens — one open ask per secret and three an hour — and the vault asks the desk's prompt itself, " + + "in its own name, with a key made for the call; what is typed opens in the vault alone. Answers how the " + + "prompt ended (given, dismissed, timed-out, empty, refused or failed) and who asked as recorded; never " + + "the value.", + Input: map[string]any{"type": "object", "properties": map[string]any{ + "node": str("the machine the secret is for"), "module": str("the module"), + "name": str("the secret's name in the module"), "desk": str("the machine whose prompt asks"), + "asked_by": str("who asked the controller, as the bus named its caller")}}, + run: func(caller string, a map[string]any) (any, error) { + v, err := env.ready() + if err != nil { + return nil, err + } + return v.AskAtDesk(caller, AskOf{Node: arg(a, "node"), Module: arg(a, "module"), Name: arg(a, "name"), + Desk: arg(a, "desk"), AskedBy: arg(a, "asked_by")}) + }}, + {Name: Seat + ".issued", + Description: "Take the credential a provider's software issued for a provision (ADR 0349 clause 10, ADR 0305), " + + "asked by the controller alone: provision, provider (its machine), module and tool. The vault asks that " + + "tool itself, in its own name, with a key made for the call, and keeps what it sealed as the provision's " + + "one credential, or, when the tool answers a value per holder named in holders, each as that holder's " + + "own, handed to it alone; the same value again writes nothing. Never answers the value.", + Input: map[string]any{"type": "object", "properties": map[string]any{ + "provision": str("the provision"), "provider": str("the provider's machine"), + "module": str("the provider's module, whose software issued it"), + "tool": str("the provider's tool that answers it, as its offer names it"), + "holders": map[string]any{"type": "array", "description": "the provision's holders from that " + + "provider, each {node, module}: the tool may answer a value for each, kept as that holder's own " + + "and handed to it alone", + "items": map[string]any{"type": "object", "properties": map[string]any{ + "node": str("the holder's machine"), "module": str("the consuming module")}}}}}, + run: func(caller string, a map[string]any) (any, error) { + v, err := env.ready() + if err != nil { + return nil, err + } + holders, err := holdersArg(a) + if err != nil { + return nil, err + } + return v.TakeIssuedFor(caller, arg(a, "provision"), arg(a, "provider"), arg(a, "module"), arg(a, "tool"), + holders) + }}, + {Name: Seat + ".rotate", + Description: "Make a secret anew (ADR 0114, ADR 0228): node, module and name for a module's own, or provider " + + "and provision for a provision's credential. Refused for a value a provider's software issued, which " + + "the software rotates, and for an accepted credential, which the vault does not remake. Never answers " + + "the value. From the controller alone, and with a why.", + Input: map[string]any{"type": "object", "properties": map[string]any{ + "node": str("the machine, for a module's own secret"), "module": str("the module"), + "name": str("the secret's name in the module"), "provider": str("the provider's machine"), + "provision": str("the provision"), + "consumer": str("a provision's credential of a provider whose consumers keep a login each: only " + + "this consumer's machine's own"), + "consumer_module": str("with it or alone: only this consuming module's own login"), + "by": str("who asked, as the controller names them when it carries the act"), + "why": str("why it is rotated: required")}}, + run: func(caller string, a map[string]any) (any, error) { + v, err := env.ready() + if err != nil { + return nil, err + } + secret, err := secretNamed(a) + if err != nil { + return nil, err + } + one, err := narrowNamed(a) + if err != nil { + return nil, err + } + return v.RotateFor(caller, secret, one, arg(a, "by"), arg(a, "why")) + }}, + {Name: Seat + ".move-in", + Description: "Take one value of the controller's store into the vault, as it is (design 57's V3): from the " + + "node-engine of a machine it was sealed to, and no other caller, in two steps — name the row (node, " + + "module and name for a module's own; provider, provision, consumer, consumer_module and local for a " + + "pair credential), its origin and the controller's generation, and get a call and a key made for it; " + + "then send the call, the value sealed to that key and read_back_to, a key of your own. Never replaces " + + "a value the vault holds; answers what it kept, read back from its store sealed to read_back_to.", + Input: intakeInput(map[string]string{"node": "a module's own secret: its machine", "module": "its module", + "name": "its name in the module", "provider": "a pair credential: the provider's machine", + "provision": "the provision", "consumer": "the consumer's machine", + "consumer_module": "the consuming module", "local": "the credential's local name, when it has one", + "origin": "where the value came from in the controller's store: made, accepted or issued", + "generation": "the controller's generation for the value, kept as it is", + "made_at": "when the controller made or took it (RFC 3339)", + "read_back_to": "the second step: a raw X25519 public key in base64 the read-back is sealed to", + "sealing_key": "the first step, optional: this machine's sealing key, a raw X25519 public key in " + + "base64, which the core's credentials are sealed to from then on (ADR 0349 §16)"}), + run: func(caller string, a map[string]any) (any, error) { + v, err := env.ready() + if err != nil { + return nil, err + } + if call := arg(a, "call"); call != "" { + return v.MoveTake(caller, call, arg(a, "sealed"), arg(a, "read_back_to")) + } + m := MoveOf{Node: arg(a, "node"), Module: arg(a, "module"), Name: arg(a, "name"), + Provider: arg(a, "provider"), Provision: arg(a, "provision"), Consumer: arg(a, "consumer"), + ConsumerModule: arg(a, "consumer_module"), Local: arg(a, "local"), Origin: arg(a, "origin"), + Generation: arg(a, "generation"), SealingKey: arg(a, "sealing_key")} + if at := arg(a, "made_at"); at != "" { + if m.MadeAt, err = time.Parse(time.RFC3339Nano, at); err != nil { + return nil, fmt.Errorf("made_at %q is not a time (RFC 3339)", at) + } + } + return v.MoveOpen(caller, m) + }}, + {Name: Seat + ".node-key", + Description: "State this machine's sealing key to the vault, in its own name (ADR 0372, issue 786): from the " + + "node-engine of the machine alone, at its start and whenever the key changes; the machine is the caller's, " + + "as the bus proved it. A first or changed key is recorded and `secret-changed` is raised for the machine's " + + "core secrets, so the controller asks again; the same key again changes nothing.", + Input: map[string]any{"type": "object", "properties": map[string]any{ + "key": str("this machine's sealing key, a raw X25519 public key in base64, as move-in's sealing_key carries it")}}, + run: func(caller string, a map[string]any) (any, error) { + v, err := env.ready() + if err != nil { + return nil, err + } + if err := v.StateNodeKey(caller, arg(a, "key")); err != nil { + return nil, err + } + return map[string]any{"recorded": true}, nil + }}, + {Name: Seat + ".move-asks", + Description: "Copy rows of the controller's record of asks at a desk into the vault's, as they are and once " + + "(design 57's V3): from the controller alone, at most 500 at a time. No value is in any of them.", + Input: map[string]any{"type": "object", "properties": map[string]any{ + "asks": map[string]any{"type": "array", "description": "the controller's rows: id, node, module, name, " + + "desk, asked_by, opened_at, and ended_at and outcome when the ask ended", + "items": map[string]any{"type": "object"}}}}, + run: func(caller string, a map[string]any) (any, error) { + v, err := env.ready() + if err != nil { + return nil, err + } + raw, err := json.Marshal(a["asks"]) + if err != nil { + return nil, err + } + var asks []MovedAsk + if err := json.Unmarshal(raw, &asks); err != nil { + return nil, fmt.Errorf("asks is not the controller's rows: %w", err) + } + return v.MoveAsks(caller, asks) + }}, + {Name: "secret_asks", + Description: "The record of asks for a module's own secret at a desk, newest first: who asked as the bus named " + + "them, for what, at which desk, when, and how each ended. Never a value.", + Input: map[string]any{"limit": map[string]any{"type": "number", "description": "how many lines (default 50)"}}, + run: func(_ string, a map[string]any) (any, error) { + v, err := env.ready() + if err != nil { + return nil, err + } + return v.Store.Asks(limitOf(a)) + }}, + {Name: "secret_export", + Description: "The mesh's root secrets as this vault keeps them: every secret a module holds for itself, " + + "sealed to the operator's key (novox/hq ADR 0085, amended). Ciphertext — nothing here can open a line " + + "of it; the operator, holding the private key off the mesh, recovers one with " + + "`nox secret recover --from-export` on the control-node. Also lists what is NOT recoverable: secrets " + + "made before the mesh had an operator key.", + Input: map[string]any{"sealed": map[string]any{"type": "boolean", + "description": "include the sealed blobs (default true); false lists holders and the key only"}}, + run: func(_ string, a map[string]any) (any, error) { + return exportAnswer(env.Root, a["sealed"] != false), nil + }}, + {Name: "secret_holders", + Description: "Who holds a secret from this vault: each consumer's login, node and module, when it was " + + "granted, how many times it has been rotated and when, and the fingerprint of the current value. " + + "Fingerprints only — the value is never returned.", + run: func(string, map[string]any) (any, error) { + if env.Ledger == nil { + return map[string]any{"holders": []any{}, "count": 0, "error": "the vault has no ledger (MESH_VAULT_LEDGER is not set)"}, nil + } + var asked []Contribution + if env.Receives != "" { + asked, _ = Contributions(env.Receives) + } + holders := []map[string]any{} + for _, h := range env.Ledger.List() { + row := map[string]any{} + raw, _ := json.Marshal(h) + _ = json.Unmarshal(raw, &row) + row["module"], row["asked"] = nil, false + for _, c := range asked { + if c.As == h.As { + row["module"], row["asked"] = c.From, true + } + } + holders = append(holders, row) + } + return map[string]any{"holders": holders, "count": len(holders)}, nil + }}, + {Name: "secret_verify", + Description: "Check one holder's secret without seeing it: the fingerprint the vault recorded against the " + + "fingerprint of the value the mesh currently delivers here, and optionally against a fingerprint " + + "computed on the holder's own machine (sha256 of the file, as `sha256:`). Two ends agreeing " + + "proves they agree, not that either works — the login itself is the test.", + Input: map[string]any{"as": str("the holder's login, e.g. anchor-redis"), + "fingerprint": str("optional: sha256: of the value as the holder reads it, computed there — never the value")}, + run: func(_ string, a map[string]any) (any, error) { + as, _ := a["as"].(string) + given, _ := a["fingerprint"].(string) + return verifyAnswer(env, as, given), nil + }}, + {Name: "secret_handed", + Description: "Who was handed which secret by `get`, and when, and who was refused, newest first, as the " + + "vault's own store records it. Never a value.", + Input: map[string]any{"limit": map[string]any{"type": "number", "description": "how many (default 50)"}}, + run: func(_ string, a map[string]any) (any, error) { + if env.Vault == nil { + return nil, errors.New("the vault is not ready") + } + return env.Vault.Store.Handed(limitOf(a)) + }}, + {Name: "secret_vault_export", + Description: "Every secret the vault itself holds, as its copy sealed to the operator key, in the shape of " + + "`secret export`: saved to a file, `nox secret recover vault mesh-vault --key " + + "--from-export --out ` opens one with the operator's private key. Ciphertext only.", + run: func(string, map[string]any) (any, error) { + if env.Vault == nil { + return nil, errors.New("the vault is not ready") + } + return vaultExport(env.Vault) + }}, + {Name: "provisioner_retirement", + Description: "This provider's retirement (novox/hq ADR 0230): the holders it keeps active, a retirement waiting " + + "for a person, one a person rejected, and every holder retired with when and why. Mark-only: a retired " + + "holder keeps what it has until a person deletes it.", + run: func(string, map[string]any) (any, error) { + if env.Custody == nil { + return nil, errors.New("the vault has no custody ledger") + } + return env.Custody.Retirement(), nil + }}, + {Name: "provisioner_retire_approve", + Description: "Retire exactly the set waiting for a person (or rejected earlier). Repeat the set in consumers; why is required.", + Input: actInput(true, false), + run: func(caller string, a map[string]any) (any, error) { + if env.Custody == nil { + return nil, errors.New("the vault has no custody ledger") + } + done, err := env.Custody.Approve(stringsOf(a["consumers"]), actOf(caller, a)) + if err != nil { + return nil, err + } + return map[string]any{"retired": done}, nil + }}, + {Name: "provisioner_retire_reject", + Description: "Keep the set waiting for a person active, though the mesh no longer asks for it. Repeat the set in consumers; why is required.", + Input: actInput(true, false), + run: func(caller string, a map[string]any) (any, error) { + if env.Custody == nil { + return nil, errors.New("the vault has no custody ledger") + } + kept, err := env.Custody.Reject(stringsOf(a["consumers"]), actOf(caller, a)) + if err != nil { + return nil, err + } + return map[string]any{"kept": kept}, nil + }}, + {Name: "provisioner_delete", + Description: "Delete one RETIRED holder from the custody ledger, its rotation history with it — never an active one. " + + "Repeat its name in confirm; why is required.", + Input: actInput(false, true), + run: func(caller string, a map[string]any) (any, error) { + if env.Custody == nil { + return nil, errors.New("the vault has no custody ledger") + } + consumer, _ := a["consumer"].(string) + confirm, _ := a["confirm"].(string) + if err := env.Custody.Delete(consumer, confirm, actOf(caller, a)); err != nil { + return nil, err + } + return map[string]any{"deleted": consumer, "freed_bytes": -1}, nil + }}, + {Name: "vault_health", + Description: "The vault's runtime health, as the node-engine asks it (novox/hq ADR 0366): unhealthy once " + + "re-sealing the operator copies has failed on three passes in a row, saying what failed and why; " + + "healthy again after one pass succeeds.", + run: func(string, map[string]any) (any, error) { return env.health(), nil }}, + {Name: "vault_status", + Description: "What needs a person's eye in the vault: operator copies still sealed to an earlier key, the operator " + + "key the mesh's export holds if the vault refused it, and how often the record of hand-outs was rolled.", + run: func(string, map[string]any) (any, error) { + if env.Vault == nil { + return map[string]any{"ready": false, "error": fmt.Sprint(env.Broken)}, nil + } + left, err := env.Vault.StillSealedToAnother() + if err != nil { + return nil, err + } + key, _ := env.Vault.Store.OperatorKey() + return map[string]any{"ready": true, "operator_key": key, "copies_on_an_earlier_key": left, + "operator_key_refused": env.keyRefused(), "handed_rolled": env.Vault.Store.Rolled(), + "ok": left == 0 && env.keyRefused() == "" && key != ""}, nil + }}, + } +} + +// actInput is the arguments of an act on retirement, as the SDK's harness takes them. +func actInput(set, one bool) map[string]any { + text := map[string]any{"type": "string"} + in := map[string]any{"why": text, "by": text, "via": text} + if set { + in["consumers"] = map[string]any{"type": "array", "items": text} + } + if one { + in["consumer"], in["confirm"] = text, text + } + return in +} + +// actOf is an act's why, by and via. `by` is the person the controller names; with none, the caller the +// runtime proved. `by` is an argument, so it is not proven: when it names somebody other than the caller the +// runtime proved, the proven caller is kept beside it in via, so the record says both. +func actOf(caller string, a map[string]any) Act { + act := Act{} + act.Why, _ = a["why"].(string) + act.By, _ = a["by"].(string) + act.Via, _ = a["via"].(string) + if act.By == "" { + act.By = caller + } + if act.By != caller { + proven := "called by " + caller + if act.Via == "" { + act.Via = proven + } else { + act.Via += ", " + proven + } + } + return act +} + +// ready is the vault, or why there is none. +func (e *Env) ready() (*Vault, error) { + if e.Vault != nil { + return e.Vault, nil + } + if e.Broken != nil { + return nil, e.Broken + } + return nil, errors.New("the vault is not ready") +} + +func arg(a map[string]any, key string) string { + s, _ := a[key].(string) + return strings.TrimSpace(s) +} + +// secretNamed is the secret a call names: a module's own (node, module, name), or a provision's one credential +// (provider, provision) — one or the other. +func secretNamed(a map[string]any) (string, error) { + own := arg(a, "node") != "" || arg(a, "module") != "" || arg(a, "name") != "" + pair := arg(a, "provider") != "" || arg(a, "provision") != "" + switch { + case own && !pair: + return OwnSecret(arg(a, "node"), arg(a, "module"), arg(a, "name")) + case pair && !own: + return ProvisionSecret(arg(a, "provider"), arg(a, "provision")) + } + return "", errors.New("name a module's own secret (node, module, name) or a provision's credential " + + "(provider, provision), one of the two") +} + +// narrowNamed is the one consumer a rotation names, if any (issue 767). +func narrowNamed(a map[string]any) (Narrow, error) { + n := Narrow{Consumer: arg(a, "consumer"), ConsumerModule: arg(a, "consumer_module")} + for what, v := range map[string]string{"the consumer's machine": n.Consumer, "the consuming module": n.ConsumerModule} { + if v != "" && !namePart.MatchString(v) { + return Narrow{}, fmt.Errorf("%s %q is not a name the mesh writes", what, v) + } + } + return n, nil +} + +// intakeInput is a two-step verb's arguments: what its first step names, then the call and what was sealed. +func intakeInput(first map[string]string) map[string]any { + props := map[string]any{} + for k, d := range first { + props[k] = str(d) + } + props["call"] = str("the second step: the call the first step answered") + props["sealed"] = str("the second step: the value sealed to the call's key (NaCl anonymous box, base64) — never the value") + return map[string]any{"type": "object", "properties": props} +} + +func stringsOf(v any) []string { + list, _ := v.([]any) + out := []string{} + for _, x := range list { + if s, ok := x.(string); ok { + out = append(out, s) + } + } + return out +} + +func limitOf(a map[string]any) int { + if v, ok := a["limit"].(float64); ok && v > 0 { + return int(v) + } + return 50 +} + +// keyFingerprint is a key's fingerprint as the controller's secrets.Fingerprint says it: sha256 of the key, +// its first eight bytes in hex. +func keyFingerprint(key string) string { + if key == "" { + return "" + } + sum := sha256.Sum256([]byte(strings.TrimSpace(key))) + return "sha256:" + hex.EncodeToString(sum[:8]) +} + +// vaultExport is the vault's operator copies in the shape `secret export` writes and `secret recover +// --from-export` reads: each as node "vault", module "mesh-vault" and the secret's name. +func vaultExport(v *Vault) (map[string]any, error) { + key, err := v.Store.OperatorKey() + if err != nil { + return nil, err + } + all, err := v.Store.List() + if err != nil { + return nil, err + } + kept, earlier := []map[string]any{}, []map[string]any{} + for _, rec := range all { + k := map[string]any{"node": "vault", "module": Seat, "name": rec.Name, "kind": "own", "origin": "made", + "sealed": rec.Operator.Sealed, "key": rec.Operator.Key, "made-at": rec.MadeAt} + if rec.Operator.Key == key { + kept = append(kept, k) + } else { + earlier = append(earlier, k) + } + } + return map[string]any{"export": 1, "operator-key": key, "fingerprint": keyFingerprint(key), "kept": kept, + "sealed-to-earlier-key": earlier}, nil +} + +func exportAnswer(root string, withBlobs bool) map[string]any { + if root == "" { + return map[string]any{"available": false, "error": "this vault keeps no root secrets (MESH_VAULT_ROOT is not set)"} + } + raw, err := os.ReadFile(filepath.Join(root, "export.json")) + if err != nil { + return map[string]any{"available": false, "error": "the mesh has not written an export here yet: " + err.Error()} + } + var doc struct { + Export int `json:"export"` + OperatorKey string `json:"operator-key"` + Fingerprint string `json:"fingerprint"` + Kept []map[string]any `json:"kept"` + Unrecoverable []map[string]any `json:"unrecoverable"` + } + if err := json.Unmarshal(raw, &doc); err != nil { + return map[string]any{"available": false, "error": "the export is not readable: " + err.Error()} + } + kept := []map[string]any{} + for _, k := range doc.Kept { + if !withBlobs { + k = map[string]any{"node": k["node"], "module": k["module"], "name": k["name"], "origin": k["origin"], "made-at": k["made-at"]} + } + kept = append(kept, k) + } + if doc.Unrecoverable == nil { + doc.Unrecoverable = []map[string]any{} + } + return map[string]any{"available": true, "export": doc.Export, "operator-key": doc.OperatorKey, + "fingerprint": doc.Fingerprint, "count": len(kept), "kept": kept, "unrecoverable": doc.Unrecoverable} +} + +// exportedOperatorKey is the operator key in the export the mesh writes to the vault's root, or "". +func exportedOperatorKey(root string) string { + raw, err := os.ReadFile(filepath.Join(root, "export.json")) + if err != nil { + return "" + } + var doc struct { + OperatorKey string `json:"operator-key"` + } + _ = json.Unmarshal(raw, &doc) + return doc.OperatorKey +} + +func verifyAnswer(env *Env, as, given string) map[string]any { + if env.Ledger == nil { + return map[string]any{"as": as, "known": false, "error": "the vault has no ledger (MESH_VAULT_LEDGER is not set)"} + } + recorded, ok := env.Ledger.Get(as) + if !ok { + return map[string]any{"as": as, "known": false, "error": "this vault holds nothing for " + as} + } + var delivered, deliveredErr any + var deliveredMatches any + var asked *Contribution + if env.Receives != "" { + all, _ := Contributions(env.Receives) + for i := range all { + if all[i].As == as { + asked = &all[i] + } + } + } + if asked == nil { + deliveredErr = "the mesh does not currently ask this vault to serve that login" + } else if fp, _, err := DeliveredFingerprint(*asked); err != nil { + deliveredErr = err.Error() + } else { + delivered, deliveredMatches = fp, fp == recorded.Fingerprint + } + var givenAny, givenMatches, earlier any + if given != "" { + givenAny, givenMatches = given, given == recorded.Fingerprint + was := false + for _, h := range recorded.History { + was = was || h.Fingerprint == given + } + earlier = was + } + return map[string]any{"as": recorded.As, "known": true, "recorded": recorded.Fingerprint, + "rotations": recorded.Rotations, "changed": recorded.Changed, + "delivered": delivered, "deliveredError": deliveredErr, "deliveredMatchesRecorded": deliveredMatches, + "given": givenAny, "givenMatchesRecorded": givenMatches, "givenIsAnEarlierValue": earlier, + "ok": deliveredMatches != false && givenMatches != false} +} + +// tools are the definitions as the SDK serves them. +func tools(env *Env) []stdio.Tool { + var out []stdio.Tool + for _, d := range definitions(env) { + out = append(out, stdio.Tool{Name: d.Name, Description: d.Description, Input: d.Input, Run: d.Run}) + } + return out +} diff --git a/modules/mesh-vault/cmd/mesh-vault/tools_test.go b/modules/mesh-vault/cmd/mesh-vault/tools_test.go new file mode 100644 index 00000000..dee060cb --- /dev/null +++ b/modules/mesh-vault/cmd/mesh-vault/tools_test.go @@ -0,0 +1,152 @@ +package main + +// The tools the TypeScript vault offered, answering as they did: secret_export, secret_holders and +// secret_verify — none returns a value. And the seat's verb `get`, reached the way the runtime calls it. + +import ( + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "git.novox.be/novox/mesh-sdk/go/seal" +) + +func toolNamed(t *testing.T, tools []toolDef, name string) toolDef { + t.Helper() + for _, tl := range tools { + if tl.Name == name { + return tl + } + } + t.Fatalf("no tool %s", name) + return toolDef{} +} + +func TestTheManifestNamesWhatTheBundleServes(t *testing.T) { + raw, _ := os.ReadFile("../../module.json") + var m struct { + Tools []string `json:"tools"` + Claims []struct { + Name string `json:"name"` + Serves []string `json:"serves"` + } `json:"claims"` + Emits []string `json:"emits"` + } + if err := json.Unmarshal(raw, &m); err != nil { + t.Fatal(err) + } + served := map[string]bool{} + for _, tl := range definitions(&Env{}) { + served[tl.Name] = true + } + for _, name := range m.Tools { + if !served[name] { + t.Errorf("the manifest names the tool %s, which the bundle does not serve", name) + } + } + for _, c := range m.Claims { + for _, verb := range c.Serves { + if !served[c.Name+"."+verb] { + t.Errorf("the claim of %s serves %s, which the bundle does not", c.Name, verb) + } + } + } + for name := range served { + if !strings.HasPrefix(name, Seat+".") && !contains(m.Tools, name) { + t.Errorf("the bundle serves %s, which the manifest does not name", name) + } + } + if !contains(m.Emits, ChangedEvent) { + t.Errorf("the manifest does not say the vault emits %s", ChangedEvent) + } +} + +func TestSecretExportAnswersTheMeshsExport(t *testing.T) { + root := t.TempDir() + env := &Env{Root: root} + export := toolNamed(t, definitions(env), "secret_export") + got, _ := export.Run(map[string]any{}) + if got.(map[string]any)["available"] != false { + t.Errorf("no export said %v", got) + } + doc := `{"export":1,"operator-key":"b3A=","fingerprint":"sha256:f","kept":[{"node":"g14","module":"dunst","name":"token","kind":"own","origin":"made","sealed":"c2VhbGVk","key":"b3A=","made-at":"2026-10-01T00:00:00Z"}],"unrecoverable":[{"node":"ace","module":"x","name":"y"}]}` + _ = os.WriteFile(filepath.Join(root, "export.json"), []byte(doc), 0o600) + got, _ = export.Run(map[string]any{}) + a := got.(map[string]any) + if a["available"] != true || a["count"] != 1 || a["operator-key"] != "b3A=" { + t.Errorf("the export said %v", a) + } + got, _ = export.Run(map[string]any{"sealed": false}) + kept := got.(map[string]any)["kept"].([]map[string]any) + if _, ok := kept[0]["sealed"]; ok { + t.Error("sealed=false still returned the blobs") + } +} + +func TestSecretHoldersAndVerifySayFingerprintsOnly(t *testing.T) { + dir := t.TempDir() + l, _ := NewLedger(filepath.Join(dir, "ledger")) + secret := filepath.Join(dir, "s") + _ = os.WriteFile(secret, []byte("the-value"), 0o600) + receives := filepath.Join(dir, "mesh.json") + doc, _ := json.Marshal(map[string]any{"given": []map[string]string{{"from": "redis", "node": "anchor", "as": "anchor-redis", "secret": secret}}}) + _ = os.WriteFile(receives, doc, 0o600) + _, _, _ = l.Record("anchor-redis", "anchor", "the-value", time.Now()) + env := &Env{Ledger: l, Receives: receives} + + holders, _ := toolNamed(t, definitions(env), "secret_holders").Run(map[string]any{}) + verify, _ := toolNamed(t, definitions(env), "secret_verify").Run(map[string]any{"as": "anchor-redis", "fingerprint": Fingerprint("the-value")}) + unknown, _ := toolNamed(t, definitions(env), "secret_verify").Run(map[string]any{"as": "nobody"}) + for _, answer := range []any{holders, verify, unknown} { + raw, _ := json.Marshal(answer) + if strings.Contains(string(raw), "the-value") { + t.Errorf("a tool returned the value: %s", raw) + } + } + v := verify.(map[string]any) + if v["ok"] != true || v["deliveredMatchesRecorded"] != true || v["givenMatchesRecorded"] != true { + t.Errorf("verify said %v", v) + } + h := holders.(map[string]any) + if h["count"] != 1 { + t.Errorf("holders said %v", h) + } + if unknown.(map[string]any)["known"] != false { + t.Errorf("an unknown login said %v", unknown) + } +} + +// The seat's verb, called as the runtime calls it: the caller is the one the runtime proved, never an +// argument. +func TestTheGetVerbTakesItsCallerFromTheRuntime(t *testing.T) { + v := newVault(t, AllowList{"g14.node-provisioner": {"pair/postgres/gitea"}}) + if _, err := v.Put("pair/postgres/gitea", "x", "the test"); err != nil { + t.Fatal(err) + } + asker, _ := seal.GenerateKeyPair() + get := toolNamed(t, definitions(&Env{Vault: v.Vault}), Seat+".get") + // No runtime proved a caller here, so the call is unproven whatever the arguments say. + if _, err := get.Run(map[string]any{"name": "pair/postgres/gitea", "public_key": asker.PublicKey, "caller": "g14.node-provisioner"}); err == nil { + t.Error("get answered a caller named in its arguments") + } + answer, err := get.runAs("g14.node-provisioner", map[string]any{"name": "pair/postgres/gitea", "public_key": asker.PublicKey}) + if err != nil { + t.Fatal(err) + } + a := answer.(Answer) + if got, err := seal.Open(a.Sealed, asker.PrivateKey); err != nil || got != "x" { + t.Errorf("%q %v", got, err) + } +} + +func contains(list []string, s string) bool { + for _, x := range list { + if x == s { + return true + } + } + return false +} diff --git a/modules/mesh-vault/cmd/mesh-vault/vault.go b/modules/mesh-vault/cmd/mesh-vault/vault.go new file mode 100644 index 00000000..817bb3b3 --- /dev/null +++ b/modules/mesh-vault/cmd/mesh-vault/vault.go @@ -0,0 +1,361 @@ +package main + +// The vault's rules (novox/hq ADR 0349 §7 and §11; ADR 0113 §1, §2, §22): it holds every secret sealed at +// rest, with a copy sealed to the operator key; it hands one out only on request and reply, sealed to the +// asker's key, and only to an asker it allows; it says a secret changed — who, when, never a value. + +import ( + "crypto/rand" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "regexp" + "sync" + "time" + + stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/seal" +) + +// Seat is the role this module holds. +const Seat = "mesh-vault" + +// ChangedEvent is what the vault raises when a secret it holds changes: who, when, which, its generation. +const ChangedEvent = "secret-changed" + +// Vault holds the secrets. +type Vault struct { + Store *Store + // Own is the vault's own key; the private half never leaves its state directory. + Own seal.KeyPair + // Askers decides who may ask `get` for what. Nil allows nobody. + Askers Askers + // Assignments is the controller's assignments mirror, where the vault sees a consumer bind a provider and + // unbind (pair.go). Empty sees nothing, and makes no provision's credential. + Assignments Mirror + Emit func(event string, body map[string]any) error + // Ask calls a tool on the mesh as the vault, through the runtime that launched it, under the vault's own bus + // account (ADR 0259 §8's runs-as, issue 723): the desks' secret prompt and each issuing provider's tool, which + // the bus lets the vault's account alone call in its own name. Nil asks nobody. + Ask func(key string, body any) (json.RawMessage, error) + Now func() time.Time + Log func(format string, args ...any) + + mu sync.Mutex + // callsMu guards pending: the keys of the calls open now (intake.go), never written anywhere. + callsMu sync.Mutex + pending map[string]*callKey +} + +// Changed is what one change to a secret was. Never its value. +type Changed struct { + Name string `json:"name"` + Generation int `json:"generation"` + At time.Time `json:"at"` + By string `json:"by"` + Unchanged bool `json:"unchanged,omitempty"` + // Each is, for a rotation of a provider whose consumers keep logins of their own, each one's own value remade. + Each []Changed `json:"each,omitempty"` +} + +// Answer is `get`'s reply: the secret sealed to the asker's key, and which generation it is. +type Answer struct { + Name string `json:"name"` + Generation int `json:"generation"` + ChangedAt time.Time `json:"changed-at"` + Sealed seal.Box `json:"sealed"` +} + +func (v *Vault) log(format string, args ...any) { + if v.Log != nil { + v.Log(format, args...) + } +} + +// Put keeps value as the secret name: sealed to the vault's key and to the operator key, a new generation +// when it differs from the one held, and `secret-changed` said. The same value again changes nothing. +// Refused: a private key (it stays where it was made), and any value while the vault has no operator key — +// a secret only the vault could open is what ADR 0113 §22 rules out. +func (v *Vault) Put(name, value, by string) (Changed, error) { + return v.keep(name, value, by, "", OriginMade) +} + +// keep is Put with where the value came from (origin) and the caller the runtime proved when it is not who +// by names (via). +func (v *Vault) keep(name, value, by, via, origin string) (Changed, error) { + return v.keepWith(name, value, by, via, origin, keepOptions{}) +} + +// keepOptions are what the move keeps beside a value (move.go): that it never replaces another, when the value +// was first made, and where it was moved from. +type keepOptions struct { + neverReplace bool + madeAt time.Time + movedFrom *MovedFrom +} + +// keepWith is keep with the move's options. +func (v *Vault) keepWith(name, value, by, via, origin string, opt keepOptions) (Changed, error) { + if err := validName(name); err != nil { + return Changed{}, err + } + if value == "" { + return Changed{}, fmt.Errorf("%s: a secret is not empty", name) + } + if privateKeyIn(value) { + return Changed{}, fmt.Errorf("%s: that is a private key, and a private key stays where it was made and never reaches the vault (ADR 0113 §2)", name) + } + if by == "" { + by = "the vault" + } + v.mu.Lock() + defer v.mu.Unlock() + operator, err := v.Store.OperatorKey() + if err != nil { + return Changed{}, err + } + if operator == "" { + return Changed{}, fmt.Errorf("%s is not kept: the vault has no operator key to seal its copy to (ADR 0113 §22) — the mesh writes it with the export once `operator key set` has been run", name) + } + before, held, err := v.Store.Read(name) + if err != nil { + return Changed{}, err + } + if held { + if old, err := seal.Open(before.AtRest, v.Own.PrivateKey); err == nil && old == value { + return Changed{Name: name, Generation: before.Generation, At: before.ChangedAt, By: before.ChangedBy, Unchanged: true}, nil + } + if opt.neverReplace { + return Changed{}, fmt.Errorf("the vault already holds another value for %s (generation %d, changed by %s "+ + "at %s), and the move never replaces a value the vault holds: nothing was taken", name, + before.Generation, before.ChangedBy, before.ChangedAt.Format(time.RFC3339)) + } + } + atRest, err := seal.Seal(value, v.Own.PublicKey) + if err != nil { + return Changed{}, err + } + forOperator, err := operatorSeal(operator, []byte(value)) + if err != nil { + return Changed{}, err + } + now := v.Now().UTC() + if via == by { + via = "" + } + rec := Record{Name: name, Generation: 1, AtRest: atRest, + Operator: OperatorCopy{Box: OperatorBox, Key: operator, Sealed: forOperator}, + MadeAt: now, ChangedAt: now, ChangedBy: by, ChangedVia: via, Origin: origin} + if held { + rec.Generation, rec.MadeAt = before.Generation+1, before.MadeAt + } else if !opt.madeAt.IsZero() { + rec.MadeAt = opt.madeAt.UTC() + } + if opt.movedFrom != nil { + moved := *opt.movedFrom + moved.At = now + rec.MovedFrom = &moved + } + if err := v.Store.Write(rec); err != nil { + return Changed{}, err + } + c := Changed{Name: name, Generation: rec.Generation, At: now, By: by} + v.announce(c) + return c, nil +} + +// Mint makes a fresh value for name and keeps it: 30 random bytes, base64url without padding — 40 +// characters, the controller's shape, which fits the tightest backend a minted password reaches. +func (v *Vault) Mint(name, by string) (Changed, error) { + raw := make([]byte, 30) + if _, err := rand.Read(raw); err != nil { + return Changed{}, err + } + return v.Put(name, base64.RawURLEncoding.EncodeToString(raw), by) +} + +// announce raises `secret-changed`: who, when, which and its generation — never a value or a sealed copy. +func (v *Vault) announce(c Changed) { + if v.Emit == nil { + return + } + body := map[string]any{"name": c.Name, "by": c.By, "at": c.At.Format(time.RFC3339Nano), "generation": c.Generation} + if err := v.Emit(ChangedEvent, body); err != nil { + v.log("%s is kept at generation %d, but %s could not be said: %v", c.Name, c.Generation, ChangedEvent, err) + } +} + +// recordHanding keeps who asked for what, and what they got, in the vault's own store, and says loudly +// when the record had to be rolled to make room. +func (v *Vault) recordHanding(h Handing) error { + h.At = v.Now().UTC() + rolled, err := v.Store.RecordHanding(h) + if err != nil { + v.log("the record of %s asking for %s could not be written: %v", h.To, h.Name, err) + return err + } + if rolled { + v.log("THE RECORD OF HAND-OUTS WAS ROLLED (%d times since the vault started): it reached its bound of %d bytes; "+ + "the oldest of %d kept files is dropped. Many asks in little time is worth a look: secret_handed and vault_status", + v.Store.Rolled(), v.Store.HandedBound(), HandedKept) + } + return nil +} + +// busUser is the shape of a caller the runtime proves (the SDK's): the one shape a refusal is recorded for. +var busUser = regexp.MustCompile(`^[A-Za-z0-9_-]+(\.[A-Za-z0-9_-]+)*$`) + +// Get answers caller with the secret name sealed to publicKey, if caller may ask for it. The caller is the +// bus user the runtime proved, never an argument; an unproven call is refused. +func (v *Vault) Get(caller, name, publicKey string) (Answer, error) { + if caller == "" || caller == stdio.Unproven { + return Answer{}, errors.New("the vault answers only a caller the runtime proved") + } + askers := v.Askers + if askers == nil { + askers = NoAsker{} + } + if !busUser.MatchString(caller) || len(caller) > 200 { + return Answer{}, errors.New("the vault answers only a caller the runtime proved") + } + // A name is judged before anything is recorded, so an arbitrary argument fills no record. + if err := validName(name); err != nil { + return Answer{}, err + } + // A value issued for one holder is that holder's alone, whatever else would allow it (issue 749). + if why := notTheHolder(caller, name); why != "" { + v.log("refused %s to %s: %s", name, caller, why) + _ = v.recordHanding(Handing{To: caller, Name: name, Outcome: "refused", Why: why}) + return Answer{}, fmt.Errorf("%s may not ask for %s: %s", caller, name, why) + } + // The core's own credentials, to the controller alone, sealed only to their machine's key (ADR 0349 §16). + if caller == ControllerCaller { + if why := v.coreRefusal(name, publicKey); why != "" { + v.log("refused %s to %s: %s", name, caller, why) + _ = v.recordHanding(Handing{To: caller, Name: name, Outcome: "refused", Why: why}) + return Answer{}, fmt.Errorf("%s may not ask for %s: %s", caller, name, why) + } + } else if ok, why := askers.May(caller, name); !ok { + v.log("refused %s to %s: %s", name, caller, why) + _ = v.recordHanding(Handing{To: caller, Name: name, Outcome: "refused", Why: why}) + return Answer{}, fmt.Errorf("%s may not ask for %s: %s", caller, name, why) + } + key, err := askerKey(publicKey) + if err != nil { + return Answer{}, err + } + rec, held, err := v.Store.Read(name) + if err != nil { + return Answer{}, err + } + if !held { + return Answer{}, fmt.Errorf("the vault holds no secret named %s", name) + } + value, err := seal.Open(rec.AtRest, v.Own.PrivateKey) + if err != nil { + return Answer{}, fmt.Errorf("%s does not open with the vault's key: %w", name, err) + } + sealed, err := seal.Seal(value, key) + if err != nil { + return Answer{}, err + } + if err := v.recordHanding(Handing{To: caller, Name: name, Generation: rec.Generation, Outcome: "handed"}); err != nil { + return Answer{}, fmt.Errorf("%s is not handed out: who was handed it could not be recorded: %w", name, err) + } + v.log("handed %s generation %d to %s, sealed to its key", name, rec.Generation, caller) + return Answer{Name: name, Generation: rec.Generation, ChangedAt: rec.ChangedAt, Sealed: sealed}, nil +} + +// SetOperatorKey makes key (SPKI PEM) the operator key, then seals again to it every operator copy that is +// not yet — see Reseal. It returns how many copies it sealed again. +// +// The key is recorded first and each record says which key its copy is sealed to, so that record is the +// progress: a failure partway leaves the rest for the next call, and the same key given again (every minute +// from the mesh's export) finishes them. Nothing returns early while a copy is sealed to another key. +func (v *Vault) SetOperatorKey(key, by string) (int, error) { + raw, err := checkOperatorKey(key) + if err != nil { + return 0, err + } + v.mu.Lock() + defer v.mu.Unlock() + current, err := v.Store.OperatorKey() + if err != nil { + return 0, err + } + if current != raw { + if err := v.Store.setOperatorKey(raw, by, v.Now().UTC()); err != nil { + return 0, err + } + v.log("the operator key is now %s (by %s); every operator copy is sealed again to it", raw, by) + } + return v.reseal(raw) +} + +// Reseal seals again to the current operator key every copy sealed to another one, and says how many it +// did. Called every minute, so a copy a failure left behind is not left for ever. +func (v *Vault) Reseal() (int, error) { + v.mu.Lock() + defer v.mu.Unlock() + key, err := v.Store.OperatorKey() + if err != nil || key == "" { + return 0, err + } + return v.reseal(key) +} + +func (v *Vault) reseal(key string) (int, error) { + all, err := v.Store.List() + if err != nil { + return 0, err + } + n := 0 + var failed []error + for _, rec := range all { + if rec.Operator.Key == key && rec.Operator.Box == OperatorBox { + continue + } + value, err := seal.Open(rec.AtRest, v.Own.PrivateKey) + if err != nil { + failed = append(failed, fmt.Errorf("%s does not open with the vault's key: %w", rec.Name, err)) + continue + } + sealed, err := operatorSeal(key, []byte(value)) + if err == nil { + rec.Operator = OperatorCopy{Box: OperatorBox, Key: key, Sealed: sealed} + err = v.Store.Write(rec) + } + if err != nil { + failed = append(failed, fmt.Errorf("%s: %w", rec.Name, err)) + continue + } + n++ + } + if len(failed) > 0 { + err := fmt.Errorf("%d operator copies are still sealed to an earlier key, tried again on the next pass: %w", + len(failed), errors.Join(failed...)) + v.log("%v", err) + return n, err + } + return n, nil +} + +// StillSealedToAnother is how many operator copies are not yet sealed to the current operator key. +func (v *Vault) StillSealedToAnother() (int, error) { + key, err := v.Store.OperatorKey() + if err != nil { + return 0, err + } + all, err := v.Store.List() + if err != nil { + return 0, err + } + n := 0 + for _, rec := range all { + if rec.Operator.Key != key { + n++ + } + } + return n, nil +} diff --git a/modules/mesh-vault/cmd/mesh-vault/vault_test.go b/modules/mesh-vault/cmd/mesh-vault/vault_test.go new file mode 100644 index 00000000..db21a44a --- /dev/null +++ b/modules/mesh-vault/cmd/mesh-vault/vault_test.go @@ -0,0 +1,864 @@ +package main + +// ADR 0349's vault test list, the part this package (V1, issue 674) answers: no bucket holds a secret; no +// private key is ever sent to the vault; every secret has a copy sealed to the operator key; `get` answers +// only an allowed asker, sealed to its key; `secret-changed` carries no value. The rest of that list (a +// consumer binding remakes the provider's credential, an accepted value is not remade, issued and typed +// values) is V2's and V5's. + +import ( + "crypto/ecdh" + "crypto/rand" + "crypto/x509" + "encoding/base64" + "encoding/json" + "encoding/pem" + "os" + "path/filepath" + "regexp" + "strings" + "sync" + "testing" + "time" + + stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/seal" + "golang.org/x/crypto/nacl/box" +) + +// published is everything the vault said on the bus during a test, as it said it. +type published struct { + mu sync.Mutex + events []map[string]any + raw []string +} + +func (p *published) emit(event string, body map[string]any) error { + p.mu.Lock() + defer p.mu.Unlock() + b, _ := json.Marshal(map[string]any{"event": event, "body": body}) + p.raw = append(p.raw, string(b)) + p.events = append(p.events, map[string]any{"event": event, "body": body}) + return nil +} + +// operatorKey is an operator key the way the controller's `operator key make` makes one: NaCl, raw X25519, +// base64 — the private half kept by the test, as the operator keeps it off the mesh. +func operatorKey(t *testing.T) (public, private string) { + t.Helper() + pub, priv, err := box.GenerateKey(rand.Reader) + if err != nil { + t.Fatal(err) + } + return base64.StdEncoding.EncodeToString(pub[:]), base64.StdEncoding.EncodeToString(priv[:]) +} + +// asPEM is a raw operator key as the SPKI PEM the vault takes. +func asPEM(t *testing.T, raw string) string { + t.Helper() + p, err := publicPEM(raw) + if err != nil { + t.Fatal(err) + } + return p +} + +// openOperator opens an operator copy the way `secret recover --from-export` does. +func openOperator(t *testing.T, public, private, sealed string) string { + t.Helper() + var pub, priv [32]byte + p, _ := base64.StdEncoding.DecodeString(public) + q, _ := base64.StdEncoding.DecodeString(private) + copy(pub[:], p) + copy(priv[:], q) + blob, err := base64.StdEncoding.DecodeString(sealed) + if err != nil { + t.Fatal(err) + } + out, ok := box.OpenAnonymous(nil, blob, &pub, &priv) + if !ok { + t.Fatal("the operator copy does not open with the operator's key") + } + return string(out) +} + +type testVault struct { + *Vault + dir string + bus *published + operatorPublic string + operatorPrivate string +} + +func newVault(t *testing.T, askers Askers) testVault { + t.Helper() + dir := t.TempDir() + store, err := OpenStore(filepath.Join(dir, "store")) + if err != nil { + t.Fatal(err) + } + own, err := OwnKey(filepath.Join(dir, "state"), store) + if err != nil { + t.Fatal(err) + } + bus := &published{} + clock := time.Date(2026, 10, 14, 9, 0, 0, 0, time.UTC) + v := &Vault{Store: store, Own: own, Askers: askers, Emit: bus.emit, + Now: func() time.Time { clock = clock.Add(time.Second); return clock }, Log: t.Logf} + pub, priv := operatorKey(t) + if _, err := v.SetOperatorKey(asPEM(t, pub), "the test"); err != nil { + t.Fatal(err) + } + return testVault{Vault: v, dir: dir, bus: bus, operatorPublic: pub, operatorPrivate: priv} +} + +// filesUnder is every file's content under dir. +func filesUnder(t *testing.T, dir string) map[string]string { + t.Helper() + out := map[string]string{} + _ = filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { + if err == nil && !info.IsDir() { + raw, _ := os.ReadFile(path) + out[path] = string(raw) + } + return nil + }) + return out +} + +// ---- no bucket holds a secret --------------------------------------------------------------------------- + +// The manifest declares no bucket, and the code opens none: the vault's only place for a secret is its own +// store on its own disk (ADR 0349 §7, ADR 0201 §13). It reads one bucket of another's, and only reads it: the +// controller's assignments, for what each node's plan names (askers.go, design 57 V4). +func TestNoBucketHoldsASecret(t *testing.T) { + raw, err := os.ReadFile("../../module.json") + if err != nil { + t.Fatal(err) + } + var manifest map[string]any + if err := json.Unmarshal(raw, &manifest); err != nil { + t.Fatal(err) + } + if s, ok := manifest["state"]; ok { + t.Errorf("the manifest declares buckets %v: a secret is in no bucket, sealed or not (ADR 0201 §13)", s) + } + if r, _ := json.Marshal(manifest["reads"]); string(r) != `["mesh-controller.assignments"]` { + t.Errorf("the manifest reads %s: the vault reads the controller's assignments alone", r) + } + sources, _ := filepath.Glob("*.go") + bucket := regexp.MustCompile(`stdio\.State\(|nats-io|jetstream|KeyValue`) + for _, f := range sources { + if strings.HasSuffix(f, "_test.go") { + continue + } + code, _ := os.ReadFile(f) + // The reads it makes, and nothing that writes: Get and Keys alone, on the assignments (Keys for the + // binds it remakes a provider's credential on, pair.go). + code = []byte(strings.Replace(string(code), "stdio.State(AssignmentsRead).Get(", "", 1)) + code = []byte(strings.Replace(string(code), "stdio.State(AssignmentsRead).Keys(", "", 1)) + if m := bucket.Find(code); m != nil { + t.Errorf("%s opens a bucket (%s): the vault keeps every secret in its own store only", f, m) + } + } + + // And what the vault publishes while it holds, changes and hands out secrets carries none of them, in + // the clear or sealed. + v := newVault(t, AllowList{"g14.node-provisioner": {"pair/postgres/gitea"}}) + value := "the-password-nobody-may-see" + if _, err := v.Put("pair/postgres/gitea", value, "the test"); err != nil { + t.Fatal(err) + } + asker, _ := seal.GenerateKeyPair() + answer, err := v.Get("g14.node-provisioner", "pair/postgres/gitea", asker.PublicKey) + if err != nil { + t.Fatal(err) + } + rec, _, _ := v.Store.Read("pair/postgres/gitea") + for _, said := range v.bus.raw { + for _, secret := range []string{value, answer.Sealed.Ct, rec.AtRest.Ct, rec.Operator.Sealed} { + if strings.Contains(said, secret) { + t.Errorf("the vault published %s, which carries a secret or a sealed copy of one", said) + } + } + } +} + +// Every secret is encrypted at rest: no file the vault writes holds a value in the clear, and each opens +// only with the vault's own key. +func TestEverySecretIsEncryptedAtRest(t *testing.T) { + v := newVault(t, nil) + values := map[string]string{"own/g14/dunst/token": "tok-1-plain", "pair/postgres/gitea": "pw-2-plain"} + for name, value := range values { + if _, err := v.Put(name, value, "the test"); err != nil { + t.Fatal(err) + } + } + for path, content := range filesUnder(t, filepath.Join(v.dir, "store")) { + for _, value := range values { + if strings.Contains(content, value) { + t.Errorf("%s holds a value in the clear", path) + } + } + if info, _ := os.Stat(path); info.Mode().Perm()&0o077 != 0 { + t.Errorf("%s is readable by others: %v", path, info.Mode()) + } + } + for name, value := range values { + rec, ok, err := v.Store.Read(name) + if err != nil || !ok { + t.Fatalf("%s: %v %v", name, ok, err) + } + got, err := seal.Open(rec.AtRest, v.Own.PrivateKey) + if err != nil || got != value { + t.Errorf("%s at rest opens to %q (%v), not its value", name, got, err) + } + } +} + +// ---- no private key is accepted ------------------------------------------------------------------------- + +func TestNoPrivateKeyIsAccepted(t *testing.T) { + v := newVault(t, AllowList{"g14.node-provisioner": {"pair/postgres/gitea"}}) + keys, _ := seal.GenerateKeyPair() + ssh := "-----BEGIN OPENSSH PRIVATE KEY-----\nb3BlbnNzaC1rZXktdjEAAAAA\n-----END OPENSSH PRIVATE KEY-----\n" + rsa := "-----BEGIN RSA PRIVATE KEY-----\nMIIBOgIBAAJBAK\n-----END RSA PRIVATE KEY-----\n" + + // Not as a secret to hold: a private key stays where it was made (ADR 0113 §2). + for _, value := range []string{keys.PrivateKey, ssh, rsa, "prefix " + keys.PrivateKey} { + if _, err := v.Put("own/g14/sshd/host", value, "the test"); err == nil { + t.Errorf("the vault took a private key as a secret: %.40q", value) + } + } + if _, ok, _ := v.Store.Read("own/g14/sshd/host"); ok { + t.Error("a refused private key was kept all the same") + } + + // Not as the key to seal an answer to. + if _, err := v.Put("pair/postgres/gitea", "pw", "the test"); err != nil { + t.Fatal(err) + } + raw32 := base64.StdEncoding.EncodeToString(make([]byte, 32)) + for _, key := range []string{keys.PrivateKey, ssh, raw32, "", "not a key"} { + if _, err := v.Get("g14.node-provisioner", "pair/postgres/gitea", key); err == nil { + t.Errorf("get sealed an answer to %.40q, which is not a PEM public key", key) + } + } + + // Not as the operator key: neither a PEM private key nor a raw key, which looks the same as a raw + // private one. + rawPub, rawPriv := operatorKey(t) + for _, key := range []string{keys.PrivateKey, rawPriv, rawPub} { + if _, err := v.SetOperatorKey(key, "the test"); err == nil { + t.Errorf("the vault took %.20q as the operator key", key) + } + } + if k, _ := v.Store.OperatorKey(); k != v.operatorPublic { + t.Error("a refused operator key replaced the operator key") + } + + // And nothing it publishes or writes names one that it refused. + for _, said := range v.bus.raw { + if strings.Contains(said, "PRIVATE KEY") { + t.Errorf("the vault published a private key: %s", said) + } + } + for path, content := range filesUnder(t, filepath.Join(v.dir, "store")) { + if strings.Contains(content, "PRIVATE KEY") { + t.Errorf("%s holds a private key", path) + } + } +} + +// ---- every secret has an operator-sealed copy ----------------------------------------------------------- + +func TestEverySecretHasAnOperatorSealedCopy(t *testing.T) { + v := newVault(t, nil) + values := map[string]string{"own/g14/dunst/token": "a", "pair/postgres/gitea": "b"} + for name, value := range values { + if _, err := v.Put(name, value, "the test"); err != nil { + t.Fatal(err) + } + } + if _, err := v.Mint("pair/postgres/keycloak", "the test"); err != nil { + t.Fatal(err) + } + all, err := v.Store.List() + if err != nil { + t.Fatal(err) + } + if len(all) != 3 { + t.Fatalf("the store lists %d secrets, not 3", len(all)) + } + for _, rec := range all { + if rec.Operator.Key != v.operatorPublic || rec.Operator.Box != OperatorBox { + t.Errorf("%s is sealed to %q in %q, not to the operator key", rec.Name, rec.Operator.Key, rec.Operator.Box) + } + got := openOperator(t, v.operatorPublic, v.operatorPrivate, rec.Operator.Sealed) + want, _ := seal.Open(rec.AtRest, v.Own.PrivateKey) + if got != want || got == "" { + t.Errorf("%s: the operator copy opens to %q, the vault's to %q", rec.Name, got, want) + } + } + + // A new operator key: every copy is sealed again to it, and none stays sealed to the old one. + pub, priv := operatorKey(t) + n, err := v.SetOperatorKey(asPEM(t, pub), "the test") + if err != nil || n != 3 { + t.Fatalf("a new operator key sealed %d copies again (%v), not 3", n, err) + } + all, _ = v.Store.List() + for _, rec := range all { + if rec.Operator.Key != pub { + t.Errorf("%s is still sealed to the earlier operator key", rec.Name) + } + openOperator(t, pub, priv, rec.Operator.Sealed) + } +} + +// Without an operator key the vault keeps nothing: a secret only it could open is what ADR 0113 §22 rules +// out, so the refusal is loud rather than a copy quietly missing. +func TestNoOperatorKeyKeepsNothing(t *testing.T) { + dir := t.TempDir() + store, _ := OpenStore(filepath.Join(dir, "store")) + own, _ := OwnKey(filepath.Join(dir, "state"), store) + v := &Vault{Store: store, Own: own, Emit: (&published{}).emit, Now: time.Now, Log: t.Logf} + if _, err := v.Put("own/g14/dunst/token", "x", "the test"); err == nil || !strings.Contains(err.Error(), "operator key") { + t.Fatalf("a secret was kept with no operator key to seal its copy to: %v", err) + } + if all, _ := store.List(); len(all) != 0 { + t.Fatalf("the store holds %d secrets", len(all)) + } +} + +// ---- get answers only an allowed asker, sealed to its key ----------------------------------------------- + +func TestGetAnswersOnlyAnAllowedAskerSealedToItsKey(t *testing.T) { + v := newVault(t, AllowList{"g14.node-provisioner": {"pair/postgres/gitea"}}) + if _, err := v.Put("pair/postgres/gitea", "the-value", "the test"); err != nil { + t.Fatal(err) + } + if _, err := v.Put("own/ace/dunst/token", "another", "the test"); err != nil { + t.Fatal(err) + } + asker, _ := seal.GenerateKeyPair() + + answer, err := v.Get("g14.node-provisioner", "pair/postgres/gitea", asker.PublicKey) + if err != nil { + t.Fatal(err) + } + if got, err := seal.Open(answer.Sealed, asker.PrivateKey); err != nil || got != "the-value" { + t.Fatalf("the answer opens to %q (%v)", got, err) + } + other, _ := seal.GenerateKeyPair() + if _, err := seal.Open(answer.Sealed, other.PrivateKey); err == nil { + t.Error("the answer opens with a key it was not sealed to") + } + if _, err := seal.Open(answer.Sealed, v.Own.PrivateKey); err == nil { + t.Error("the answer opens with the vault's own key") + } + if answer.Generation != 1 || answer.Name != "pair/postgres/gitea" { + t.Errorf("the answer says %+v", answer) + } + + refusals := []struct{ caller, name, why string }{ + {"ace.node-provisioner", "pair/postgres/gitea", "an asker not on the list"}, + {"g14.node-provisioner", "own/ace/dunst/token", "a secret the asker is not allowed"}, + {stdio.Unproven, "pair/postgres/gitea", "a call whose caller the runtime did not prove"}, + {"", "pair/postgres/gitea", "no caller at all"}, + {"g14.node-provisioner", "pair/postgres/nothing", "a secret the vault does not hold"}, + } + for _, r := range refusals { + if a, err := v.Get(r.caller, r.name, asker.PublicKey); err == nil { + t.Errorf("get answered %s: %+v", r.why, a) + } + } +} + +// A vault given no rule answers nobody. +func TestAVaultWithNoRuleAnswersNobody(t *testing.T) { + v := newVault(t, nil) + if _, err := v.Put("pair/postgres/gitea", "x", "the test"); err != nil { + t.Fatal(err) + } + asker, _ := seal.GenerateKeyPair() + for _, caller := range []string{"provisioner.g14", "novox.mesh-provisioner", "controller", "person.jochen", "novox.node-tools"} { + if _, err := v.Get(caller, "pair/postgres/gitea", asker.PublicKey); err == nil { + t.Errorf("a vault with no rule answered %s", caller) + } + } +} + +// assignments stands in for the controller's assignments mirror as the vault reads it. +type assignments map[string]string + +func (a assignments) read(key string) ([]byte, error) { + if v, ok := a[key]; ok { + return []byte(v), nil + } + return nil, nil +} + +// The running vault's rule (ADR 0349 §11, design 57 V4): a node's provisioner client, under its own account, for a +// secret of a module on its node that the module's plan there names, and nobody else for anything else. Each +// refusal is recorded with why. +func TestTheRunningVaultAnswersAMachinesClientForWhatItsPlanNames(t *testing.T) { + mirror := assignments{ + "g14.dunst": `{"node":"g14","module":"dunst","assigned":"2026-10-14T20:00:00Z","provisioning":` + + `{"templates":{"source":"x","digest":"y"},"files":[],"secrets":[{"name":"token"}]}}`, + "g14.picom": `{"node":"g14","module":"picom","assigned":"2026-10-14T20:00:00Z"}`, + "ace.dunst": `{"node":"ace","module":"dunst","assigned":"2026-10-14T20:00:00Z","provisioning":{"secrets":[{"name":"token"}]}}`, + "g14.liar": `{"node":"ace","module":"liar","provisioning":{"secrets":[{"name":"token"}]}}`, + "g14.unparsed": `not json`, + } + v := newVault(t, Declared{Read: mirror.read}) + for _, name := range []string{"own/g14/dunst/token", "own/g14/dunst/other", "own/g14/picom/token", + "own/ace/dunst/token", "own/g14/liar/token", "own/g14/unparsed/token", "provision/novox/postgres", + "own/g14/absent/token"} { + if _, err := v.Put(name, "value-of-"+name, "the test"); err != nil { + t.Fatal(err) + } + } + asker, _ := seal.GenerateKeyPair() + + answer, err := v.Get("provisioner.g14", "own/g14/dunst/token", asker.PublicKey) + if err != nil { + t.Fatalf("the machine's client was refused the secret its module's plan names: %v", err) + } + if got, err := seal.Open(answer.Sealed, asker.PrivateKey); err != nil || got != "value-of-own/g14/dunst/token" { + t.Fatalf("the answer opens to %q (%v)", got, err) + } + + refusals := []struct{ caller, name, why string }{ + {"provisioner.g14", "own/g14/dunst/other", "a secret the plan does not name"}, + {"provisioner.g14", "own/g14/picom/token", "a module not handed to the provisioner"}, + {"provisioner.g14", "own/ace/dunst/token", "another machine's secret"}, + {"provisioner.ace", "own/g14/dunst/token", "another machine's client"}, + {"provisioner.g14", "own/g14/absent/token", "a module not assigned there"}, + {"provisioner.g14", "own/g14/liar/token", "an assignment whose value names another machine"}, + {"provisioner.g14", "own/g14/unparsed/token", "an assignment that is not the controller's"}, + {"provisioner.g14", "provision/novox/postgres", "a provision's credential the vault made for nobody there"}, + {"g14.node-tools", "own/g14/dunst/token", "the tool runner's account, which an agent shares"}, + {"g14.dunst", "own/g14/dunst/token", "a module's own account, until issue 569"}, + {"controller", "own/g14/dunst/token", "the controller"}, + {"provisioner.", "own/g14/dunst/token", "a client naming no machine"}, + } + for _, r := range refusals { + if a, err := v.Get(r.caller, r.name, asker.PublicKey); err == nil { + t.Errorf("get answered %s (%s asking %s): %+v", r.why, r.caller, r.name, a) + } + } + handed, err := v.Store.Handed(100) + if err != nil { + t.Fatal(err) + } + refused := 0 + for _, h := range handed { + if h.Outcome == "refused" { + if h.Why == "" { + t.Errorf("a refusal of %s to %s is recorded without why", h.Name, h.To) + } + refused++ + } + } + // Every refusal but the last, whose caller is not a bus user's shape and so fills no record (vault.go). + if refused != len(refusals)-1 { + t.Errorf("%d refusals recorded, want %d", refused, len(refusals)-1) + } +} + +// A value a provider's software issued for one holder (issue 757): the running rule hands +// provision//// to that node's client when the module's plan there names the +// provision from that provider, and refuses another node's, another module's, another provider's and the provision's +// one credential. +func TestTheRunningVaultAnswersAMachinesClientForAnIssuedValueItsPlanNames(t *testing.T) { + plan := `{"templates":{"source":"x","digest":"y"},"files":[],` + + `"provisions":[{"name":"gitea-read","provision":"gitea-read","provider":"novox","path":"/s/gitea-read.secret"}]}` + mirror := assignments{ + "anchor.claude-code": `{"node":"anchor","module":"claude-code","provisioning":` + plan + `}`, + "laptop.claude-code": `{"node":"laptop","module":"claude-code","provisioning":` + plan + `}`, + "anchor.zsh": `{"node":"anchor","module":"zsh","provisioning":{"secrets":[{"name":"token"}]}}`, + "anchor.picom": `{"node":"anchor","module":"picom"}`, + } + v := newVault(t, Declared{Read: mirror.read}) + for _, name := range []string{anchorsToken, laptopsToken, "provision/novox/gitea-read/anchor/zsh", + "provision/novox/gitea-read/anchor/picom", "provision/ace/gitea-read/anchor/claude-code", + "provision/novox/other/anchor/claude-code", "provision/novox/gitea-read", + "provision/novox/gitea-read/anchor/absent"} { + if _, err := v.Put(name, "value-of-"+name, "the test"); err != nil { + t.Fatal(err) + } + } + asker, _ := seal.GenerateKeyPair() + answer, err := v.Get("provisioner.anchor", anchorsToken, asker.PublicKey) + if err != nil { + t.Fatalf("the machine's client was refused the value issued for its module, which its plan names: %v", err) + } + if got, err := seal.Open(answer.Sealed, asker.PrivateKey); err != nil || got != "value-of-"+anchorsToken { + t.Fatalf("the answer opens to %q (%v)", got, err) + } + refusals := []struct{ caller, name, why string }{ + {"provisioner.anchor", laptopsToken, "another machine's holder value"}, + {"provisioner.laptop", anchorsToken, "another machine's holder value, the other way"}, + {"provisioner.anchor", "provision/novox/gitea-read/anchor/zsh", "another module's, whose plan names it not"}, + {"provisioner.anchor", "provision/novox/gitea-read/anchor/picom", "a module not handed to the provisioner"}, + {"provisioner.anchor", "provision/novox/gitea-read/anchor/absent", "a module not assigned there"}, + {"provisioner.anchor", "provision/ace/gitea-read/anchor/claude-code", "another provider's"}, + {"provisioner.anchor", "provision/novox/other/anchor/claude-code", "a provision the plan names not"}, + {"provisioner.anchor", "provision/novox/gitea-read", "the provision's one credential"}, + {"anchor.node-tools", anchorsToken, "the tool runner's account, which an agent shares"}, + {"anchor.claude-code", anchorsToken, "the module's own account, until issue 569"}, + } + for _, r := range refusals { + if a, err := v.Get(r.caller, r.name, asker.PublicKey); err == nil { + t.Errorf("get answered %s (%s asking %s): %+v", r.why, r.caller, r.name, a) + } + } +} + +// The node-engine's sealing key (mesh-host internal/identity/sealing.go: raw X25519, base64, the NaCl +// anonymous box) is the same curve as the ADR 0206 box. A node's client asks with that key's public half as +// SPKI PEM, and opens the answer with its private half as PKCS #8 — no second key on the node, and no NaCl +// in the vault's answers. This is the path V4 builds on. +func TestANodesSealingKeyOpensTheAnswer(t *testing.T) { + v := newVault(t, AllowList{"g14.node-provisioner": {"pair/postgres/gitea"}}) + if _, err := v.Put("pair/postgres/gitea", "for-the-node", "the test"); err != nil { + t.Fatal(err) + } + // As mesh-host's GenerateSealingKey makes it. + priv, _ := ecdh.X25519().GenerateKey(rand.Reader) + nodePublic := base64.StdEncoding.EncodeToString(priv.PublicKey().Bytes()) + nodePrivate := base64.StdEncoding.EncodeToString(priv.Bytes()) + + rawPub, _ := base64.StdEncoding.DecodeString(nodePublic) + pubKey, _ := ecdh.X25519().NewPublicKey(rawPub) + spki, _ := x509.MarshalPKIXPublicKey(pubKey) + askWith := string(pem.EncodeToMemory(&pem.Block{Type: "PUBLIC KEY", Bytes: spki})) + + answer, err := v.Get("g14.node-provisioner", "pair/postgres/gitea", askWith) + if err != nil { + t.Fatal(err) + } + rawPriv, _ := base64.StdEncoding.DecodeString(nodePrivate) + privKey, _ := ecdh.X25519().NewPrivateKey(rawPriv) + pkcs8, _ := x509.MarshalPKCS8PrivateKey(privKey) + openWith := string(pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: pkcs8})) + if got, err := seal.Open(answer.Sealed, openWith); err != nil || got != "for-the-node" { + t.Fatalf("the node's own sealing key does not open the answer: %q %v", got, err) + } +} + +// ---- secret-changed carries no value -------------------------------------------------------------------- + +func TestSecretChangedCarriesNoValue(t *testing.T) { + v := newVault(t, nil) + if _, err := v.Put("pair/postgres/gitea", "first-value", "g14.mesh-vault-test"); err != nil { + t.Fatal(err) + } + if _, err := v.Put("pair/postgres/gitea", "second-value", "person.jochen"); err != nil { + t.Fatal(err) + } + if _, err := v.Mint("own/g14/dunst/token", "the vault"); err != nil { + t.Fatal(err) + } + var changed []map[string]any + for _, e := range v.bus.events { + if e["event"] == ChangedEvent { + changed = append(changed, e["body"].(map[string]any)) + } + } + if len(changed) != 3 { + t.Fatalf("%d secret-changed events, not 3: %v", len(changed), v.bus.events) + } + for i, body := range changed { + keys := make([]string, 0, len(body)) + for k := range body { + keys = append(keys, k) + } + for _, k := range keys { + switch k { + case "name", "by", "at", "generation": + default: + t.Errorf("secret-changed carries %q, beyond who, when, which and its generation", k) + } + } + if body["by"] == "" || body["at"] == "" { + t.Errorf("secret-changed %d says not who or when: %v", i, body) + } + } + if changed[1]["by"] != "person.jochen" || changed[1]["generation"] != 2 { + t.Errorf("the second change says %v", changed[1]) + } + minted, _, _ := v.Store.Read("own/g14/dunst/token") + mintedValue, _ := seal.Open(minted.AtRest, v.Own.PrivateKey) + for _, said := range v.bus.raw { + for _, value := range []string{"first-value", "second-value", mintedValue} { + if strings.Contains(said, value) { + t.Errorf("an event carries a value: %s", said) + } + } + } +} + +// The same value again is no change: no new generation, no event. +func TestTheSameValueAgainIsNoChange(t *testing.T) { + v := newVault(t, nil) + for i := 0; i < 2; i++ { + if _, err := v.Put("pair/postgres/gitea", "same", "the test"); err != nil { + t.Fatal(err) + } + } + rec, _, _ := v.Store.Read("pair/postgres/gitea") + if rec.Generation != 1 { + t.Errorf("the same value twice is generation %d", rec.Generation) + } + n := 0 + for _, e := range v.bus.events { + if e["event"] == ChangedEvent { + n++ + } + } + if n != 1 { + t.Errorf("%d secret-changed events for one value", n) + } +} + +// A secret's name is a name, not a path. +func TestASecretsNameIsNotAPath(t *testing.T) { + v := newVault(t, nil) + for _, name := range []string{"../etc/passwd", "/abs", "a//b", "A/b", "a/../b", "", strings.Repeat("a", 300)} { + if _, err := v.Put(name, "x", "the test"); err == nil { + t.Errorf("the vault kept a secret named %q", name) + } + } +} + +// The vault's own key is made once, kept beside it, and the same on the next start. +func TestTheVaultsOwnKeyIsMadeOnce(t *testing.T) { + dir := t.TempDir() + a, err := OwnKey(dir, nil) + if err != nil { + t.Fatal(err) + } + b, err := OwnKey(dir, nil) + if err != nil { + t.Fatal(err) + } + if a != b { + t.Error("the vault's key changed between two starts") + } + info, _ := os.Stat(filepath.Join(dir, OwnKeyFile)) + if info.Mode().Perm() != 0o600 { + t.Errorf("the vault's key is %v", info.Mode()) + } +} + +// The vault's key gone while its store holds secrets: no new key is made, and the vault does not start. +func TestAMissingKeyBesideSecretsIsRefused(t *testing.T) { + v := newVault(t, nil) + if _, err := v.Put("pair/postgres/gitea", "x", "the test"); err != nil { + t.Fatal(err) + } + if err := os.Remove(filepath.Join(v.dir, "state", OwnKeyFile)); err != nil { + t.Fatal(err) + } + if _, err := OwnKey(filepath.Join(v.dir, "state"), v.Store); err == nil || !strings.Contains(err.Error(), "MISSING") { + t.Fatalf("a new key was made beside secrets sealed to the lost one: %v", err) + } + if _, err := os.Stat(filepath.Join(v.dir, "state", OwnKeyFile)); err == nil { + t.Error("a key file was written all the same") + } +} + +// A new operator key that fails partway leaves no copy sealed to the earlier key for good: the next call +// with the same key — every minute, from the mesh's export — finishes the rest. +func TestANewOperatorKeyFailingPartwayIsFinished(t *testing.T) { + v := newVault(t, nil) + for _, name := range []string{"a/one", "b/two", "c/three"} { + if _, err := v.Mint(name, "the test"); err != nil { + t.Fatal(err) + } + } + pub, priv := operatorKey(t) + v.Store.failWrite = func(r Record) error { + if r.Name == "b/two" { + return os.ErrPermission + } + return nil + } + n, err := v.SetOperatorKey(asPEM(t, pub), "the test") + if err == nil || n != 2 { + t.Fatalf("a failure partway said %d sealed again, err %v", n, err) + } + if left, _ := v.StillSealedToAnother(); left != 1 { + t.Fatalf("%d copies left on the earlier key, not 1", left) + } + v.Store.failWrite = nil + // The path the minute takes while the export holds the key: the same key again. + if n, err := v.SetOperatorKey(asPEM(t, pub), "the test"); err != nil || n != 1 { + t.Fatalf("the same key again sealed %d again (%v), not the 1 left", n, err) + } + if n, err := v.SetOperatorKey(asPEM(t, pub), "the test"); err != nil || n != 0 { + t.Fatalf("the same key a third time sealed %d (%v)", n, err) + } + all, _ := v.Store.List() + for _, rec := range all { + if rec.Operator.Key != pub { + t.Errorf("%s is still sealed to an earlier key", rec.Name) + } + openOperator(t, pub, priv, rec.Operator.Sealed) + } +} + +// Who was handed which secret, and when, and who was refused, is in the vault's own store. +func TestWhoWasHandedWhatIsRecordedInTheStore(t *testing.T) { + v := newVault(t, AllowList{"g14.node-provisioner": {"pair/postgres/gitea"}}) + if _, err := v.Put("pair/postgres/gitea", "the-value", "the test"); err != nil { + t.Fatal(err) + } + asker, _ := seal.GenerateKeyPair() + if _, err := v.Get("g14.node-provisioner", "pair/postgres/gitea", asker.PublicKey); err != nil { + t.Fatal(err) + } + _, _ = v.Get("ace.node-provisioner", "pair/postgres/gitea", asker.PublicKey) + handed, err := v.Store.Handed(10) + if err != nil || len(handed) != 2 { + t.Fatalf("%v %v", handed, err) + } + if handed[1].To != "g14.node-provisioner" || handed[1].Outcome != "handed" || handed[1].Generation != 1 || handed[1].At.IsZero() { + t.Errorf("the hand-out is recorded as %+v", handed[1]) + } + if handed[0].To != "ace.node-provisioner" || handed[0].Outcome != "refused" || handed[0].Why == "" { + t.Errorf("the refusal is recorded as %+v", handed[0]) + } + raw, _ := os.ReadFile(filepath.Join(v.dir, "store", "handed.jsonl")) + if strings.Contains(string(raw), "the-value") { + t.Error("the record of hand-outs holds a value") + } +} + +// The vault's operator copies reach the operator's recovery: secret_vault_export is a document +// `secret recover --from-export` reads, found by node "vault", module "mesh-vault" and the secret's name, +// and opened with the operator's private key as that command opens it (mesh-controller +// cmd/mesh-controller/secret.go keptFromExport, internal/secrets Open). +func TestTheOperatorRecoversAVaultSecretFromItsExport(t *testing.T) { + v := newVault(t, nil) + if _, err := v.Put("pair/postgres/gitea", "recover-me", "the test"); err != nil { + t.Fatal(err) + } + answer, err := toolNamed(t, definitions(&Env{Vault: v.Vault}), "secret_vault_export").Run(map[string]any{}) + if err != nil { + t.Fatal(err) + } + file := filepath.Join(t.TempDir(), "vault-export.json") + raw, _ := json.Marshal(answer) + _ = os.WriteFile(file, raw, 0o600) + + // What `secret recover vault mesh-vault pair/postgres/gitea --from-export ` reads. + var e struct { + Kept []struct { + Node, Module, Name, Sealed, Key string + } `json:"kept"` + } + read, _ := os.ReadFile(file) + if err := json.Unmarshal(read, &e); err != nil { + t.Fatal(err) + } + var sealed string + for _, k := range e.Kept { + if k.Node == "vault" && k.Module == "mesh-vault" && k.Name == "pair/postgres/gitea" { + sealed = k.Sealed + if k.Key != v.operatorPublic { + t.Errorf("the export names key %s, not the operator's", k.Key) + } + } + } + if sealed == "" { + t.Fatalf("the export holds no copy of the secret: %s", read) + } + // secrets.Open: the public half derived from the private key, then the anonymous box. + privRaw, _ := base64.StdEncoding.DecodeString(v.operatorPrivate) + key, _ := ecdh.X25519().NewPrivateKey(privRaw) + if got := openOperator(t, base64.StdEncoding.EncodeToString(key.PublicKey().Bytes()), v.operatorPrivate, sealed); got != "recover-me" { + t.Fatalf("recovered %q", got) + } + if strings.Contains(string(read), "recover-me") { + t.Error("the export holds the value in the clear") + } +} + +// With no key in the export the minute calls Reseal, which finishes what is pending too. +func TestResealFinishesWhatIsPending(t *testing.T) { + v := newVault(t, nil) + if _, err := v.Mint("a/one", "the test"); err != nil { + t.Fatal(err) + } + pub, _ := operatorKey(t) + v.Store.failWrite = func(Record) error { return os.ErrPermission } + if _, err := v.SetOperatorKey(asPEM(t, pub), "the test"); err == nil { + t.Fatal("a failing write was not said") + } + v.Store.failWrite = nil + if n, err := v.Reseal(); err != nil || n != 1 { + t.Fatalf("%d %v", n, err) + } +} + +// A secret is not handed out when who was handed it cannot be recorded. +func TestNoHandOutWithoutItsRecord(t *testing.T) { + v := newVault(t, AllowList{"g14.node-provisioner": {"pair/postgres/gitea"}}) + if _, err := v.Put("pair/postgres/gitea", "x", "the test"); err != nil { + t.Fatal(err) + } + if err := os.Mkdir(filepath.Join(v.dir, "store", "handed.jsonl"), 0o700); err != nil { + t.Fatal(err) + } + asker, _ := seal.GenerateKeyPair() + if a, err := v.Get("g14.node-provisioner", "pair/postgres/gitea", asker.PublicKey); err == nil { + t.Fatalf("handed out with no record of it: %+v", a) + } +} + +// What an asker cannot shape fills no record: a name that is no secret's name, a caller that is no bus user. +func TestAnArbitraryAskFillsNoRecord(t *testing.T) { + v := newVault(t, nil) + asker, _ := seal.GenerateKeyPair() + for _, ask := range [][2]string{{"g14.x", strings.Repeat("junk ", 1000)}, {"g14.x", "../../etc"}, + {"not a bus user!", "pair/a"}, {strings.Repeat("a", 300), "pair/a"}} { + _, _ = v.Get(ask[0], ask[1], asker.PublicKey) + } + if h, _ := v.Store.Handed(10); len(h) != 0 { + t.Errorf("recorded %d arbitrary asks: %+v", len(h), h) + } +} + +// The record of hand-outs is bounded: rolled at its size into a fixed number of files, the oldest dropped. +func TestTheRecordOfHandOutsIsBounded(t *testing.T) { + v := newVault(t, nil) // refuses everyone, as the running vault does + v.Store.HandedMax = 2000 + asker, _ := seal.GenerateKeyPair() + for i := 0; i < 500; i++ { + _, _ = v.Get("g14.flood", "pair/postgres/gitea", asker.PublicKey) + } + var total int64 + files, _ := filepath.Glob(filepath.Join(v.dir, "store", "handed.jsonl*")) + if len(files) != HandedKept+1 { + t.Errorf("%d files of the record, not %d: %v", len(files), HandedKept+1, files) + } + for _, f := range files { + info, _ := os.Stat(f) + if info.Size() > 2000 { + t.Errorf("%s is %d bytes, over its bound", f, info.Size()) + } + total += info.Size() + } + if total > int64(HandedKept+1)*2000 { + t.Errorf("the record holds %d bytes", total) + } + if v.Store.Rolled() == 0 { + t.Error("rolling was not counted") + } + if h, _ := v.Store.Handed(5); len(h) != 5 || h[0].To != "g14.flood" { + t.Errorf("the newest are not read back: %+v", h) + } +} diff --git a/modules/mesh-vault/go.mod b/modules/mesh-vault/go.mod new file mode 100644 index 00000000..b025e17a --- /dev/null +++ b/modules/mesh-vault/go.mod @@ -0,0 +1,22 @@ +module mesh-vault + +go 1.25.0 + +require ( + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261013235212-c8b07242a009 + github.com/nats-io/nats-server/v2 v2.11.17 + github.com/nats-io/nats.go v1.51.0 + golang.org/x/crypto v0.50.0 +) + +require ( + github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op // indirect + github.com/google/go-tpm v0.9.8 // indirect + github.com/klauspost/compress v1.18.5 // indirect + github.com/minio/highwayhash v1.0.4 // indirect + github.com/nats-io/jwt/v2 v2.8.1 // indirect + github.com/nats-io/nkeys v0.4.15 // indirect + github.com/nats-io/nuid v1.0.1 // indirect + golang.org/x/sys v0.43.0 // indirect + golang.org/x/time v0.15.0 // indirect +) diff --git a/modules/mesh-vault/go.sum b/modules/mesh-vault/go.sum new file mode 100644 index 00000000..da3d1f5d --- /dev/null +++ b/modules/mesh-vault/go.sum @@ -0,0 +1,27 @@ +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261013235212-c8b07242a009 h1:I9rm8HhYRJ0+mzQQz+VAH0kgv2YJWkvctEc67Yd0PX8= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261013235212-c8b07242a009/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY= +github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= +github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= +github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= +github.com/google/go-tpm v0.9.8/go.mod h1:h9jEsEECg7gtLis0upRBQU+GhYVH6jMjrFxI8u6bVUY= +github.com/klauspost/compress v1.18.5 h1:/h1gH5Ce+VWNLSWqPzOVn6XBO+vJbCNGvjoaGBFW2IE= +github.com/klauspost/compress v1.18.5/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= +github.com/minio/highwayhash v1.0.4 h1:asJizugGgchQod2ja9NJlGOWq4s7KsAWr5XUc9Clgl4= +github.com/minio/highwayhash v1.0.4/go.mod h1:GGYsuwP/fPD6Y9hMiXuapVvlIUEhFhMTh0rxU3ik1LQ= +github.com/nats-io/jwt/v2 v2.8.1 h1:V0xpGuD/N8Mi+fQNDynXohVvp7ZztevW5io8CUWlPmU= +github.com/nats-io/jwt/v2 v2.8.1/go.mod h1:nWnOEEiVMiKHQpnAy4eXlizVEtSfzacZ1Q43LIRavZg= +github.com/nats-io/nats-server/v2 v2.11.17 h1:GKEghcFK6A+aFx11Yf1LjgLC3txAwvyhnYzhBIQZA8I= +github.com/nats-io/nats-server/v2 v2.11.17/go.mod h1:B1sFVz4StNosQ903ak4N1G01Fl/9f8e06mXpFIE2K24= +github.com/nats-io/nats.go v1.51.0 h1:ByW84XTz6W03GSSsygsZcA+xgKK8vPGaa/FCAAEHnAI= +github.com/nats-io/nats.go v1.51.0/go.mod h1:26HypzazeOkyO3/mqd1zZd53STJN0EjCYF9Uy2ZOBno= +github.com/nats-io/nkeys v0.4.15 h1:JACV5jRVO9V856KOapQ7x+EY8Jo3qw1vJt/9Jpwzkk4= +github.com/nats-io/nkeys v0.4.15/go.mod h1:CpMchTXC9fxA5zrMo4KpySxNjiDVvr8ANOSZdiNfUrs= +github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= +github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= +golang.org/x/crypto v0.50.0 h1:zO47/JPrL6vsNkINmLoo/PH1gcxpls50DNogFvB5ZGI= +golang.org/x/crypto v0.50.0/go.mod h1:3muZ7vA7PBCE6xgPX7nkzzjiUq87kRItoJQM1Yo8S+Q= +golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.43.0 h1:Rlag2XtaFTxp19wS8MXlJwTvoh8ArU6ezoyFsMyCTNI= +golang.org/x/sys v0.43.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= +golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= diff --git a/modules/mesh-vault/index.ts b/modules/mesh-vault/index.ts deleted file mode 100644 index 07aee59f..00000000 --- a/modules/mesh-vault/index.ts +++ /dev/null @@ -1,31 +0,0 @@ -// mesh-vault's events entrypoint, loaded by the per-node tool host (the provisioner runs in the same -// process — ADR 0052). The lifecycle events are EMITTED from the provisioner, where custody -// actually changes (novox/hq ADR 0041/0042): -// mesh-vault.provisioned — a consumer was granted a secret -// mesh-vault.rotated — that consumer's value changed (`rotate secret`) -// mesh-vault.deprovisioned — the consumer went away and its secret was withdrawn -// Here the vault reacts to them, keeping a lightweight audit line of who holds what and when it -// moved — the audit an owner of secrets is best placed to log. Fingerprints, never values. - -import { on } from "@novox/mesh-sdk/events"; - -interface SecretEvent { - as: string; - consumer?: string; - fingerprint?: string; - rotations?: number; -} - -await on("provisioned", async (e) => { - console.log(`[mesh-vault] secret provisioned for ${e.body?.as} on ${e.body?.consumer} (${e.body?.fingerprint})`); -}); - -await on("rotated", async (e) => { - console.log(`[mesh-vault] secret rotated for ${e.body?.as} — rotation ${e.body?.rotations} (${e.body?.fingerprint})`); -}); - -await on("deprovisioned", async (e) => { - console.log(`[mesh-vault] secret withdrawn from ${e.body?.as}`); -}); - -console.log("[mesh-vault] auditing secret lifecycle events"); diff --git a/modules/mesh-vault/module.json b/modules/mesh-vault/module.json index 87b6e6d7..83ee6cd4 100644 --- a/modules/mesh-vault/module.json +++ b/modules/mesh-vault/module.json @@ -1,6 +1,7 @@ { "module": "mesh-vault", "version": "1", + "runs-as": "mesh-vault", "provides": [ { "name": "secret", @@ -10,18 +11,13 @@ } } ], - "capabilities": [ - "container-runtime" - ], "emits": [ "provisioned", "rotated", - "deprovisioned" + "secret-changed" ], - "consumes": [ - "mesh-vault.provisioned", - "mesh-vault.rotated", - "mesh-vault.deprovisioned" + "reads": [ + "mesh-controller.assignments" ], "receives": { "secret": "${dir:grants}/mesh.json" @@ -49,6 +45,12 @@ "path": "${dir:root}", "class": "valuable", "why": "the vault's root material" + }, + { + "id": "store", + "path": "${dir:store}", + "class": "valuable", + "why": "every secret the vault holds, sealed to its own key and to the operator key" } ], "consumers": { @@ -59,27 +61,56 @@ } } }, + "own-secrets": { + "broker": "${dir:state}/broker" + }, + "secrets-owner": "mesh-vault", + "runtime-health": { + "kind": "tool", + "tool": "vault_health", + "interval": "60s", + "timeout": "10s", + "looks": 2, + "grace": "60s" + }, "resources": [ + { + "id": "account", + "type": "user", + "name": "mesh-vault", + "shell": "/usr/bin/nologin", + "home": "/var/lib/mesh-vault" + }, { "id": "state", "type": "directory", "mode": "0700", + "owner": "mesh-vault", "place": "." }, { "id": "grants", "type": "directory", - "mode": "0700" + "mode": "0700", + "owner": "mesh-vault" }, { "id": "ledger", "type": "directory", - "mode": "0700" + "mode": "0700", + "owner": "mesh-vault" }, { "id": "root", "type": "directory", - "mode": "0700" + "mode": "0700", + "owner": "mesh-vault" + }, + { + "id": "store", + "type": "directory", + "mode": "0700", + "owner": "mesh-vault" } ], "build": { @@ -87,29 +118,51 @@ { "name": "code", "kind": "bundle", - "language": "typescript", - "entrypoints": [ - "index.js", - "tools/index.js", - "provisioner/index.js" - ], + "language": "go", + "system": "arch", + "from": "cmd/mesh-vault", + "binary": "mesh-vault", "loads": [ - "index.js", - "tools/index.js", - "provisioner/index.js" + "mesh-vault" ], "env": { "MESH_RECEIVES": "${dir:grants}/mesh.json", "MESH_VAULT_LEDGER": "${dir:ledger}", - "MESH_VAULT_ROOT": "${dir:root}" + "MESH_VAULT_ROOT": "${dir:root}", + "MESH_VAULT_STORE": "${dir:store}", + "MESH_VAULT_STATE": "${dir:state}" } } ] }, + "tools": [ + "secret_export", + "secret_holders", + "secret_verify", + "secret_handed", + "secret_asks", + "secret_vault_export", + "provisioner_retirement", + "provisioner_retire_approve", + "provisioner_retire_reject", + "provisioner_delete", + "vault_status", + "vault_health" + ], "claims": [ { "name": "mesh-vault", - "scope": "mesh" + "scope": "mesh", + "serves": [ + "get", + "accept", + "secret-ask", + "issued", + "rotate", + "move-in", + "node-key", + "move-asks" + ] } ] } diff --git a/modules/mesh-vault/package.json b/modules/mesh-vault/package.json deleted file mode 100644 index 585de85b..00000000 --- a/modules/mesh-vault/package.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "name": "@novox/module-mesh-vault", - "version": "0.1.0", - "description": "mesh-vault — provides the mesh `secret` interface: a module's own secret as an ordinary pair credential, held, audited and rotated like any other (novox/hq ADR 0085). Its ledger, provisioner, tools and events live here (ADR 0039).", - "type": "module", - "private": true, - "scripts": { - "test": "node --test --experimental-strip-types 'test/*.test.ts'" - }, - "dependencies": { - "@novox/mesh-sdk": "^0.1.0" - }, - "devDependencies": { - "@types/node": "^22.0.0", - "typescript": "^5.6.0" - } -} diff --git a/modules/mesh-vault/provisioner/index.ts b/modules/mesh-vault/provisioner/index.ts deleted file mode 100644 index 37808a1e..00000000 --- a/modules/mesh-vault/provisioner/index.ts +++ /dev/null @@ -1,48 +0,0 @@ -// mesh-vault's provisioner — the adapter that makes vault a provider of the mesh `secret` interface. The -// reconcile loop, the contributions file, and reading the mesh's minted value are the sdk harness's; -// this writes only the per-service half (novox/hq ADR 0039/0040/0048) — and for a vault that half is -// taking custody, not creating anything. -// -// The `secret` interface (ADR 0085, design 24): a consumer requires a value for its own use — the -// password of a store it runs privately, an internal token — and reads it from the file the mesh -// writes on its machine. There is no server to create a login on. **The value is the pair -// credential itself**: the controller minted it, sealed it to both nodes, and delivered each its -// copy. What makes it *owned* is this: the vault records who holds it and its fingerprint, notices -// when `rotate secret` delivers a different one, and says so on the mesh. Rotation is not new -// machinery — it is the machinery that already moves a database password, pointed at a secret the -// vault provides (design 13). - -import { runProvisioner, type Provision } from "@novox/mesh-sdk/provisioner"; -import { emit } from "@novox/mesh-sdk/events"; -import { Ledger } from "../client.js"; - -const ledger = Ledger.fromEnv(); - -/** Emit a lifecycle event without letting a broker hiccup fail the custody itself. */ -async function announce(type: string, body: Record): Promise { - try { - await emit(type, body); - } catch (err) { - console.error(`[provisioner:secret] emit ${type} failed: ${err}`); - } -} - -runProvisioner("secret", { - async create(p: Provision): Promise { - const { held, outcome } = ledger.record(p.as, p.consumer ?? "", p.password); - if (outcome === "unchanged") return; // the harness re-runs create on restart; nothing happened - console.log(`[mesh-vault] ${outcome}: ${held.as} (${held.fingerprint.slice(0, 19)}…, rotations ${held.rotations})`); - await announce(`module.mesh-vault.secret.${outcome === "granted" ? "provisioned" : "rotated"}`, { - consumer: held.consumer, - as: held.as, - fingerprint: held.fingerprint, - rotations: held.rotations, - }); - }, - - async remove(p: { as: string }): Promise { - if (!ledger.withdraw(p.as)) return; - console.log(`[mesh-vault] withdrawn: ${p.as}`); - await announce("deprovisioned", { as: p.as }); - }, -}); diff --git a/modules/mesh-vault/test/ledger.test.ts b/modules/mesh-vault/test/ledger.test.ts deleted file mode 100644 index e2ca648f..00000000 --- a/modules/mesh-vault/test/ledger.test.ts +++ /dev/null @@ -1,80 +0,0 @@ -import { test } from "node:test"; -import assert from "node:assert/strict"; -import { mkdtempSync, readFileSync, readdirSync, statSync, writeFileSync } from "node:fs"; -import { tmpdir } from "node:os"; -import { join } from "node:path"; - -import { Ledger, fingerprint, contributions, deliveredFingerprint } from "../client.ts"; - -function fresh(): Ledger { - return new Ledger(mkdtempSync(join(tmpdir(), "vault-ledger-"))); -} - -test("a first delivery is a grant, the same value again is nothing, a new value is a rotation", () => { - const ledger = fresh(); - const t0 = new Date("2026-09-20T10:00:00Z"); - const t1 = new Date("2026-09-21T10:00:00Z"); - - const granted = ledger.record("anchor-redis", "anchor", "first-value", t0); - assert.equal(granted.outcome, "granted"); - assert.equal(granted.held.rotations, 0); - assert.equal(granted.held.since, t0.toISOString()); - assert.equal(granted.held.fingerprint, fingerprint("first-value")); - - assert.equal(ledger.record("anchor-redis", "anchor", "first-value", t1).outcome, "unchanged"); - assert.equal(ledger.get("anchor-redis")!.rotations, 0, "an unchanged delivery counted as a rotation"); - - const rotated = ledger.record("anchor-redis", "anchor", "second-value", t1); - assert.equal(rotated.outcome, "rotated"); - assert.equal(rotated.held.rotations, 1); - assert.equal(rotated.held.since, t0.toISOString(), "a rotation reset the grant date"); - assert.equal(rotated.held.changed, t1.toISOString()); - assert.equal(rotated.held.fingerprint, fingerprint("second-value")); - assert.deepEqual(rotated.held.history, [{ fingerprint: fingerprint("first-value"), until: t1.toISOString() }]); -}); - -test("the ledger holds fingerprints and never the value, in files nobody else can read", () => { - const dir = mkdtempSync(join(tmpdir(), "vault-ledger-")); - const ledger = new Ledger(dir); - ledger.record("anchor-redis", "anchor", "the-actual-password", new Date()); - ledger.record("anchor-redis", "anchor", "the-rotated-password", new Date()); - for (const name of readdirSync(dir)) { - const raw = readFileSync(join(dir, name), "utf8"); - assert.doesNotMatch(raw, /the-actual-password|the-rotated-password/, `${name} holds a value`); - assert.equal(statSync(join(dir, name)).mode & 0o777, 0o600, `${name} is readable by others`); - } -}); - -test("withdrawing forgets a holder, and listing is by login", () => { - const ledger = fresh(); - ledger.record("b-app", "b", "x", new Date()); - ledger.record("a-app", "a", "y", new Date()); - assert.deepEqual(ledger.list().map((h) => h.as), ["a-app", "b-app"]); - assert.equal(ledger.withdraw("a-app"), true); - assert.equal(ledger.withdraw("a-app"), false, "withdrawing twice said it found something"); - assert.deepEqual(ledger.list().map((h) => h.as), ["b-app"]); -}); - -test("a login is a name, not a path", () => { - const ledger = fresh(); - assert.throws(() => ledger.record("../etc/passwd", "n", "v"), /a login is a name/); -}); - -test("what the mesh delivers is read from the contributions file and fingerprinted, never returned", () => { - const dir = mkdtempSync(join(tmpdir(), "vault-grants-")); - const secret = join(dir, "anchor.redis.secret"); - writeFileSync(secret, "minted-value\n"); // the host may leave a trailing newline; the value has none - const receives = join(dir, "mesh.json"); - writeFileSync(receives, JSON.stringify({ - requirement: "secret", - given: [ - { from: "redis", node: "anchor", as: "anchor-redis", secret }, - { from: "offer-only", node: "anchor" }, // a contribution with no login grants nothing - ], - })); - const asked = contributions(receives); - assert.deepEqual(asked.map((c) => c.as), ["anchor-redis"]); - const seen = deliveredFingerprint(asked[0]); - assert.deepEqual(seen, { fingerprint: fingerprint("minted-value"), length: "minted-value".length }); - assert.match(JSON.stringify(deliveredFingerprint({ as: "x", secret: join(dir, "missing") })), /not readable/); -}); diff --git a/modules/mesh-vault/tools/index.ts b/modules/mesh-vault/tools/index.ts deleted file mode 100644 index a6437e60..00000000 --- a/modules/mesh-vault/tools/index.ts +++ /dev/null @@ -1,131 +0,0 @@ -// mesh-vault's tools — vault's own code (novox/hq ADR 0039), served through the sdk's tool harness. They -// return structured data about the secrets this vault provides, and **never a value**: a holder is -// identified by its login and a value by its fingerprint. That is the rule the source mesh's -// secret_locate / secret_verify were built on, after a secret printed into a transcript. - -import { readFileSync } from "node:fs"; -import { join } from "node:path"; -import { registerModuleTools, type ToolDefinition } from "@novox/mesh-sdk/tools"; -import { Ledger, contributions, deliveredFingerprint, type Held } from "../client.js"; - -/** The mesh's export of every operator-sealed secret, as the mesh wrote it into the root dir. */ -interface KeptExport { - export: number; - "operator-key": string; - fingerprint: string; - kept: { node: string; module: string; name: string; origin: string; sealed: string; key: string; "made-at": string }[]; - unrecoverable?: { node: string; module: string; name: string }[]; -} - -export function getVaultTools(ledger: Ledger, receives: string | undefined, root: string | undefined): ToolDefinition[] { - return [ - { - name: "secret_export", - description: - "The mesh's root secrets as this vault keeps them: every secret a module holds for itself, " + - "sealed to the operator's key (novox/hq ADR 0085, amended). Ciphertext — nothing here can " + - "open a line of it; the operator, holding the private key off the mesh, recovers one with " + - "`nox secret recover --from-export` on the control-node. Also lists what is NOT recoverable: secrets " + - "made before the mesh had an operator key.", - input: { - sealed: { - type: "boolean", - description: "include the sealed blobs (default true); false lists holders and the key only", - }, - }, - run: async (args) => { - if (!root) return { available: false, error: "this vault keeps no root secrets (MESH_VAULT_ROOT is not set)" }; - let doc: KeptExport; - try { - doc = JSON.parse(readFileSync(join(root, "export.json"), "utf8")) as KeptExport; - } catch (err) { - return { available: false, error: `the mesh has not written an export here yet: ${err}` }; - } - const withBlobs = args.sealed !== false; - return { - available: true, - export: doc.export, - "operator-key": doc["operator-key"], - fingerprint: doc.fingerprint, - count: doc.kept.length, - kept: doc.kept.map((k) => (withBlobs ? k : { node: k.node, module: k.module, name: k.name, origin: k.origin, "made-at": k["made-at"] })), - unrecoverable: doc.unrecoverable ?? [], - }; - }, - }, - { - name: "secret_holders", - description: - "Who holds a secret from this vault: each consumer's login, node and module, when it was " + - "granted, how many times it has been rotated and when, and the fingerprint of the current " + - "value. Fingerprints only — the value is never returned.", - input: {}, - run: async () => { - const asked = receives ? contributions(receives) : []; - const holders = ledger.list().map((h) => ({ - ...h, - module: asked.find((c) => c.as === h.as)?.from ?? null, - asked: asked.some((c) => c.as === h.as), - })); - return { holders, count: holders.length }; - }, - }, - { - name: "secret_verify", - description: - "Check one holder's secret without seeing it: the fingerprint the vault recorded against " + - "the fingerprint of the value the mesh currently delivers here, and optionally against a " + - "fingerprint computed on the holder's own machine (sha256 of the file, as `sha256:`). " + - "Two ends agreeing proves they agree, not that either works — the login itself is the test.", - input: { - as: { type: "string", description: "the holder's login, e.g. anchor-redis" }, - fingerprint: { - type: "string", - description: "optional: sha256: of the value as the holder reads it, computed there — never the value", - }, - }, - run: async (args) => { - const as = String(args.as ?? ""); - const recorded = ledger.get(as); - if (!recorded) return { as, known: false, error: `this vault holds nothing for ${as}` }; - const asked = receives ? contributions(receives).find((c) => c.as === as) : undefined; - const delivered = asked ? deliveredFingerprint(asked) : { error: "the mesh does not currently ask this vault to serve that login" }; - const given = args.fingerprint ? String(args.fingerprint) : undefined; - return verdict(recorded, delivered, given); - }, - }, - ]; -} - -function verdict( - recorded: Held, - delivered: { fingerprint: string; length: number } | { error: string }, - given: string | undefined, -): Record { - const deliveredMatches = "fingerprint" in delivered ? delivered.fingerprint === recorded.fingerprint : null; - const givenMatches = given === undefined ? null : given === recorded.fingerprint; - return { - as: recorded.as, - known: true, - recorded: recorded.fingerprint, - rotations: recorded.rotations, - changed: recorded.changed, - delivered: "fingerprint" in delivered ? delivered.fingerprint : null, - deliveredError: "error" in delivered ? delivered.error : null, - deliveredMatchesRecorded: deliveredMatches, - given: given ?? null, - givenMatchesRecorded: givenMatches, - givenIsAnEarlierValue: given === undefined ? null : recorded.history.some((h) => h.fingerprint === given), - ok: deliveredMatches !== false && givenMatches !== false, - }; -} - -// The tools exist only when the ledger can be reached from the environment; without it, vault -// contributes none rather than failing the whole tool runtime. -registerModuleTools("mesh-vault", (env) => { - try { - return getVaultTools(Ledger.fromEnv(env), env.MESH_RECEIVES, env.MESH_VAULT_ROOT); - } catch { - return []; - } -}); diff --git a/modules/mesh-vault/tsconfig.json b/modules/mesh-vault/tsconfig.json deleted file mode 100644 index 51f4046a..00000000 --- a/modules/mesh-vault/tsconfig.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "compilerOptions": { - "target": "ES2022", - "module": "NodeNext", - "moduleResolution": "NodeNext", - "strict": true, - "esModuleInterop": true, - "skipLibCheck": true, - "noEmit": true - }, - "include": ["client.ts", "index.ts", "provisioner/index.ts", "tools/index.ts"] -} diff --git a/modules/mesh-watcher/cmd/mesh-watcher/templates_test.go b/modules/mesh-watcher/cmd/mesh-watcher/templates_test.go new file mode 100644 index 00000000..7d0718b5 --- /dev/null +++ b/modules/mesh-watcher/cmd/mesh-watcher/templates_test.go @@ -0,0 +1,64 @@ +package main + +import ( + "encoding/json" + "os" + "path/filepath" + "testing" +) + +// The module's directory, where its manifest and its files/ are. +const moduleDir = "../.." + +// templateText is the text a file resource places on the machine: its template's, in the module's files/ +// (novox/hq ADR 0349 §2). A resource that names no template, or a template that is not there, fails the test. +func templateText(t *testing.T, r map[string]any) string { + t.Helper() + name, _ := r["template"].(string) + if name == "" { + t.Fatalf("resource %v names no template", r["id"]) + } + raw, err := os.ReadFile(filepath.Join(moduleDir, filepath.FromSlash(name))) + if err != nil { + t.Fatalf("resource %v names %s: %v", r["id"], name, err) + } + return string(raw) +} + +// No file resource carries its text inline: each names its template in files/, and the build packs files/ +// into the one archive the controller reads the templates back from (novox/hq ADR 0349 §2). +func TestEveryFileTakesItsTextFromATemplate(t *testing.T) { + raw, err := os.ReadFile(filepath.Join(moduleDir, "module.json")) + if err != nil { + t.Fatal(err) + } + var m struct { + Resources []map[string]any `json:"resources"` + Build struct { + Artifacts []map[string]any `json:"artifacts"` + } `json:"build"` + } + if err := json.Unmarshal(raw, &m); err != nil { + t.Fatal(err) + } + for _, r := range m.Resources { + if _, inline := r["content"]; inline { + t.Errorf("resource %v keeps its text inline: keep it in files/ and name it as the template", r["id"]) + } + if r["type"] == "file" { + templateText(t, r) + } + } + packs := 0 + for _, a := range m.Build.Artifacts { + if a["kind"] == "archive" && a["from"] == "files" { + packs++ + if a["name"] != "files" { + t.Errorf("the templates archive is named %v, not files", a["name"]) + } + } + } + if packs != 1 { + t.Errorf("the build packs files/ into %d archives, not one", packs) + } +} diff --git a/modules/mesh-watcher/files/settings.json b/modules/mesh-watcher/files/settings.json new file mode 100644 index 00000000..84b9ecc1 --- /dev/null +++ b/modules/mesh-watcher/files/settings.json @@ -0,0 +1,3 @@ +{ + "telegram-chat-id": "" +} diff --git a/modules/mesh-watcher/module.json b/modules/mesh-watcher/module.json index 6cbbfb33..798fcf20 100644 --- a/modules/mesh-watcher/module.json +++ b/modules/mesh-watcher/module.json @@ -18,6 +18,14 @@ "watcher_test", "watcher_ping" ], + "config": { + "telegram-chat-id": { + "type": "string", + "description": "The Telegram chat the watcher reports to: the chat's id, a number, or @name of a channel; empty reports to none.", + "trusted": true, + "default": "" + } + }, "resources": [ { "id": "state", @@ -30,8 +38,9 @@ "type": "file", "path": "${dir:state}/settings.json", "mode": "0600", + "takes-config": ["telegram-chat-id"], "merge": "json", - "content": "{\n \"telegram-chat-id\": \"\"\n}\n" + "template": "files/settings.json" } ], "build": { @@ -50,6 +59,11 @@ "MESH_WATCHER_SETTINGS": "${dir:state}/settings.json", "MESH_WATCHER_TELEGRAM_TOKEN_FILE": "${dir:state}/telegram-token" } + }, + { + "name": "files", + "kind": "archive", + "from": "files" } ] } diff --git a/modules/messenger/cmd/messenger/templates_test.go b/modules/messenger/cmd/messenger/templates_test.go new file mode 100644 index 00000000..7d0718b5 --- /dev/null +++ b/modules/messenger/cmd/messenger/templates_test.go @@ -0,0 +1,64 @@ +package main + +import ( + "encoding/json" + "os" + "path/filepath" + "testing" +) + +// The module's directory, where its manifest and its files/ are. +const moduleDir = "../.." + +// templateText is the text a file resource places on the machine: its template's, in the module's files/ +// (novox/hq ADR 0349 §2). A resource that names no template, or a template that is not there, fails the test. +func templateText(t *testing.T, r map[string]any) string { + t.Helper() + name, _ := r["template"].(string) + if name == "" { + t.Fatalf("resource %v names no template", r["id"]) + } + raw, err := os.ReadFile(filepath.Join(moduleDir, filepath.FromSlash(name))) + if err != nil { + t.Fatalf("resource %v names %s: %v", r["id"], name, err) + } + return string(raw) +} + +// No file resource carries its text inline: each names its template in files/, and the build packs files/ +// into the one archive the controller reads the templates back from (novox/hq ADR 0349 §2). +func TestEveryFileTakesItsTextFromATemplate(t *testing.T) { + raw, err := os.ReadFile(filepath.Join(moduleDir, "module.json")) + if err != nil { + t.Fatal(err) + } + var m struct { + Resources []map[string]any `json:"resources"` + Build struct { + Artifacts []map[string]any `json:"artifacts"` + } `json:"build"` + } + if err := json.Unmarshal(raw, &m); err != nil { + t.Fatal(err) + } + for _, r := range m.Resources { + if _, inline := r["content"]; inline { + t.Errorf("resource %v keeps its text inline: keep it in files/ and name it as the template", r["id"]) + } + if r["type"] == "file" { + templateText(t, r) + } + } + packs := 0 + for _, a := range m.Build.Artifacts { + if a["kind"] == "archive" && a["from"] == "files" { + packs++ + if a["name"] != "files" { + t.Errorf("the templates archive is named %v, not files", a["name"]) + } + } + } + if packs != 1 { + t.Errorf("the build packs files/ into %d archives, not one", packs) + } +} diff --git a/modules/messenger/files/settings.json b/modules/messenger/files/settings.json new file mode 100644 index 00000000..ce76334c --- /dev/null +++ b/modules/messenger/files/settings.json @@ -0,0 +1,4 @@ +{ + "time-zone": "", + "only-what-needs-me": [] +} diff --git a/modules/messenger/module.json b/modules/messenger/module.json index 3a50f885..89d0b3a6 100644 --- a/modules/messenger/module.json +++ b/modules/messenger/module.json @@ -112,6 +112,26 @@ "messenger_identities", "messenger_unlink" ], + "config": { + "time-zone": { + "type": "string", + "description": "The time zone every time a message says is said in, an IANA name such as Europe/Brussels; empty is this machine's own zone.", + "trusted": true, + "default": "" + }, + "desktop-machines": { + "type": "list", + "description": "The machines whose desktop showed the operator's messages, by node name. Nothing in this module reads it since the desk channel took the desktop over; it is taken so the file stays as a layer that still sets it makes it, until that layer drops it.", + "trusted": true, + "default": [] + }, + "only-what-needs-me": { + "type": "list", + "description": "The kinds of channel set to say only what needs the operator: a walk that delivered with nothing held and nothing open is not told there (ADR 0308 §4); empty is none.", + "trusted": true, + "default": [] + } + }, "resources": [ { "id": "account", @@ -132,8 +152,9 @@ "type": "file", "path": "${dir:state}/settings.json", "mode": "0600", + "takes-config": ["time-zone", "desktop-machines", "only-what-needs-me"], "merge": "json", - "content": "{\n \"time-zone\": \"\",\n \"only-what-needs-me\": []\n}\n", + "template": "files/settings.json", "owner": "messenger" } ], @@ -153,6 +174,11 @@ "MESH_MESSENGER_SETTINGS": "${dir:state}/settings.json", "MESH_MESSENGER_STATE": "${dir:state}" } + }, + { + "name": "files", + "kind": "archive", + "from": "files" } ] } diff --git a/modules/minio/files/root.env b/modules/minio/files/root.env new file mode 100644 index 00000000..2255260f --- /dev/null +++ b/modules/minio/files/root.env @@ -0,0 +1,2 @@ +MINIO_ROOT_USER=meshroot +MINIO_BROWSER_REDIRECT_URL=https://${bound:route:name-console} diff --git a/modules/minio/module.json b/modules/minio/module.json index 7d6f525e..fd59501e 100644 --- a/modules/minio/module.json +++ b/modules/minio/module.json @@ -104,7 +104,7 @@ "type": "file", "path": "${dir:state}/root.env", "mode": "0600", - "content": "MINIO_ROOT_USER=meshroot\nMINIO_BROWSER_REDIRECT_URL=https://${bound:route:name-console}\n" + "template": "files/root.env" }, { "id": "data", @@ -174,6 +174,11 @@ "MESH_MINIO_MC_CONFIG": "${dir:state}/mc", "MESH_RECEIVES": "${dir:grants}/mesh.json" } + }, + { + "name": "files", + "kind": "archive", + "from": "files" } ] } diff --git a/modules/model-usage/files/database.url b/modules/model-usage/files/database.url new file mode 100644 index 00000000..dc59da9a --- /dev/null +++ b/modules/model-usage/files/database.url @@ -0,0 +1 @@ +postgresql://${bound:postgres-database:as}:${secret:postgres-database}@${bound:postgres-database:at}:${bound:postgres-database:port}/${bound:postgres-database:as} diff --git a/modules/model-usage/module.json b/modules/model-usage/module.json index a167523f..0ab5703d 100644 --- a/modules/model-usage/module.json +++ b/modules/model-usage/module.json @@ -50,7 +50,7 @@ "type": "file", "path": "${dir:state}/database.url", "mode": "0600", - "content": "postgresql://${bound:postgres-database:as}:${secret:postgres-database}@${bound:postgres-database:at}:${bound:postgres-database:port}/${bound:postgres-database:as}\n" + "template": "files/database.url" }, { "id": "spool", @@ -94,6 +94,11 @@ "DATABASE_URL_FILE": "${dir:state}/database.url", "USAGE_SPOOL": "${dir:spool}" } + }, + { + "name": "files", + "kind": "archive", + "from": "files" } ] } diff --git a/modules/mongodb/files/server-root.secret b/modules/mongodb/files/server-root.secret new file mode 100644 index 00000000..0ccabeea --- /dev/null +++ b/modules/mongodb/files/server-root.secret @@ -0,0 +1 @@ +${secret:root} \ No newline at end of file diff --git a/modules/mongodb/module.json b/modules/mongodb/module.json index e204b863..de9d9105 100644 --- a/modules/mongodb/module.json +++ b/modules/mongodb/module.json @@ -109,7 +109,7 @@ "path": "${dir:state}/server-root.secret", "mode": "0400", "owner": "999:999", - "content": "${secret:root}" + "template": "files/server-root.secret" }, { "id": "server", @@ -155,6 +155,11 @@ "MESH_PROVISION_PASSWORD_FILE": "${dir:state}/root.secret", "MESH_RECEIVES": "${dir:grants}/mesh.json" } + }, + { + "name": "files", + "kind": "archive", + "from": "files" } ] } diff --git a/modules/mosquitto/files/bootstrap.env b/modules/mosquitto/files/bootstrap.env new file mode 100644 index 00000000..65b710eb --- /dev/null +++ b/modules/mosquitto/files/bootstrap.env @@ -0,0 +1,7 @@ +MESH_PROVISION_MQTT=127.0.0.1:${port:1883} +MESH_PROVISION_ADMIN_USER=mesh-admin +MESH_PROVISION_PASSWORD_FILE=${dir:mesh-state}/admin +MESH_DYNSEC_FILE=${dir:data}/dynamic-security.json +MESH_MQTT_CTRL_IMAGE=eclipse-mosquitto@sha256:38c0da4f2ef84284d47b3b3eeea1cb3bdeabe81ee10caf0cd5c5ff61ee3ea408 +MESH_MQTT_CTRL_CONTAINER=mosquitto +MESH_MQTT_ADMIN_APPLIED_FILE=${dir:state}/admin.applied diff --git a/modules/mosquitto/files/mosquitto.conf b/modules/mosquitto/files/mosquitto.conf new file mode 100644 index 00000000..500c2a69 --- /dev/null +++ b/modules/mosquitto/files/mosquitto.conf @@ -0,0 +1,20 @@ +persistence true +persistence_location /mosquitto/data + +log_dest stdout +log_type warning +log_type error +log_type notice + +# Every client authenticates; identities and their per-topic ACLs are managed +# at runtime by the dynamic security plugin, whose store the plugin itself owns. +allow_anonymous false +plugin /usr/lib/mosquitto_dynamic_security.so +plugin_opt_config_file /mosquitto/data/dynamic-security.json + +# MQTT listener +listener 1883 + +# MQTT-over-WebSockets listener +listener 8081 +protocol websockets diff --git a/modules/mosquitto/module.json b/modules/mosquitto/module.json index afa4172f..e9cb913c 100644 --- a/modules/mosquitto/module.json +++ b/modules/mosquitto/module.json @@ -103,7 +103,7 @@ "path": "${dir:state}/mosquitto.conf", "mode": "0600", "owner": "1883:1883", - "content": "persistence true\npersistence_location /mosquitto/data\n\nlog_dest stdout\nlog_type warning\nlog_type error\nlog_type notice\n\n# Every client authenticates; identities and their per-topic ACLs are managed\n# at runtime by the dynamic security plugin, whose store the plugin itself owns.\nallow_anonymous false\nplugin /usr/lib/mosquitto_dynamic_security.so\nplugin_opt_config_file /mosquitto/data/dynamic-security.json\n\n# MQTT listener\nlistener 1883\n\n# MQTT-over-WebSockets listener\nlistener 8081\nprotocol websockets\n" + "template": "files/mosquitto.conf" }, { "id": "net", @@ -115,7 +115,7 @@ "type": "file", "path": "${dir:state}/bootstrap.env", "mode": "0600", - "content": "MESH_PROVISION_MQTT=127.0.0.1:${port:1883}\nMESH_PROVISION_ADMIN_USER=mesh-admin\nMESH_PROVISION_PASSWORD_FILE=${dir:mesh-state}/admin\nMESH_DYNSEC_FILE=${dir:data}/dynamic-security.json\nMESH_MQTT_CTRL_IMAGE=eclipse-mosquitto@sha256:38c0da4f2ef84284d47b3b3eeea1cb3bdeabe81ee10caf0cd5c5ff61ee3ea408\nMESH_MQTT_CTRL_CONTAINER=mosquitto\nMESH_MQTT_ADMIN_APPLIED_FILE=${dir:state}/admin.applied\n" + "template": "files/bootstrap.env" }, { "id": "bootstrap", @@ -179,6 +179,11 @@ "MESH_PROVISION_PASSWORD_FILE": "${dir:mesh-state}/admin", "MESH_MQTT_CTRL_CONTAINER": "mosquitto" } + }, + { + "name": "files", + "kind": "archive", + "from": "files" } ] } diff --git a/modules/mounts/cmd/mounts/manifest_test.go b/modules/mounts/cmd/mounts/manifest_test.go index ea95543a..257ec6ce 100644 --- a/modules/mounts/cmd/mounts/manifest_test.go +++ b/modules/mounts/cmd/mounts/manifest_test.go @@ -16,7 +16,7 @@ type manifest struct { Invokes []string `json:"invokes"` Claims []map[string]any `json:"claims"` State []map[string]any `json:"state"` - Settings map[string]map[string]any `json:"settings"` + Config map[string]map[string]any `json:"config"` OwnSecrets map[string]map[string]string `json:"own-secrets"` Resources []map[string]any `json:"resources"` } @@ -31,7 +31,15 @@ func readManifest(t *testing.T) (manifest, string) { if err := json.Unmarshal(raw, &m); err != nil { t.Fatal(err) } - return m, string(raw) + // And the text of each file it places, kept in files/ as its template (novox/hq ADR 0349 §2): what the + // machine is given is judged as a whole. + text := string(raw) + for _, r := range m.Resources { + if _, has := r["template"]; has { + text += templateText(t, r) + } + } + return m, text } // The tools the bundle serves are the manifest's tools and the seat's verbs, each saying whether it reads @@ -91,13 +99,14 @@ func TestTheManifestNamesWhatTheBundleServesAndWhatItNeeds(t *testing.T) { if got := PasswordFile("${dir:state}", "games"); got != strings.Replace(sec["path"], "*", "games", 1) { t.Errorf("the process reads %s, the mesh places %s", got, sec["path"]) } - if _, has := m.Settings["smb-users"]; has || len(m.Settings) != 2 { - t.Errorf("an SMB source's username is in its entry of sources, not a setting of its own: %v", m.Settings) + if _, has := m.Config["smb-users"]; has || len(m.Config) != 2 { + t.Errorf("an SMB source's username is in its entry of sources, not a config value of its own: %v", m.Config) } for _, key := range []string{"shares", "sources"} { - s := m.Settings[key] - if s["kind"] != "preference" || s["default"] != None || s["why"] == "" || !strings.Contains(raw, "${setting:"+key+"}") { - t.Errorf("setting %s: %v", key, s) + s := m.Config[key] + if s["type"] != "string" || s["default"] != None || s["description"] == "" || s["trusted"] != true || + !strings.Contains(raw, "${config:"+key+"}") { + t.Errorf("config value %s: %v", key, s) } } // A machine's or a person's name, as a word: "novox/hq" names the record, not the machine. @@ -137,7 +146,7 @@ func TestTheProcessesAreRootAndJudgedByTheirUnitAndTheHealthTool(t *testing.T) { if byID["config"]["path"] != ConfigFile { t.Errorf("the config file is %v, the code reads %s", byID["config"]["path"], ConfigFile) } - content, _ := byID["config"]["content"].(string) + content := templateText(t, byID["config"]) cfg := ParseConfig(content) if len(cfg.Problems) != 0 && !strings.Contains(strings.Join(cfg.Problems, " "), "${") { t.Errorf("the config's template reads: %v", cfg.Problems) diff --git a/modules/mounts/cmd/mounts/templates_test.go b/modules/mounts/cmd/mounts/templates_test.go new file mode 100644 index 00000000..7d0718b5 --- /dev/null +++ b/modules/mounts/cmd/mounts/templates_test.go @@ -0,0 +1,64 @@ +package main + +import ( + "encoding/json" + "os" + "path/filepath" + "testing" +) + +// The module's directory, where its manifest and its files/ are. +const moduleDir = "../.." + +// templateText is the text a file resource places on the machine: its template's, in the module's files/ +// (novox/hq ADR 0349 §2). A resource that names no template, or a template that is not there, fails the test. +func templateText(t *testing.T, r map[string]any) string { + t.Helper() + name, _ := r["template"].(string) + if name == "" { + t.Fatalf("resource %v names no template", r["id"]) + } + raw, err := os.ReadFile(filepath.Join(moduleDir, filepath.FromSlash(name))) + if err != nil { + t.Fatalf("resource %v names %s: %v", r["id"], name, err) + } + return string(raw) +} + +// No file resource carries its text inline: each names its template in files/, and the build packs files/ +// into the one archive the controller reads the templates back from (novox/hq ADR 0349 §2). +func TestEveryFileTakesItsTextFromATemplate(t *testing.T) { + raw, err := os.ReadFile(filepath.Join(moduleDir, "module.json")) + if err != nil { + t.Fatal(err) + } + var m struct { + Resources []map[string]any `json:"resources"` + Build struct { + Artifacts []map[string]any `json:"artifacts"` + } `json:"build"` + } + if err := json.Unmarshal(raw, &m); err != nil { + t.Fatal(err) + } + for _, r := range m.Resources { + if _, inline := r["content"]; inline { + t.Errorf("resource %v keeps its text inline: keep it in files/ and name it as the template", r["id"]) + } + if r["type"] == "file" { + templateText(t, r) + } + } + packs := 0 + for _, a := range m.Build.Artifacts { + if a["kind"] == "archive" && a["from"] == "files" { + packs++ + if a["name"] != "files" { + t.Errorf("the templates archive is named %v, not files", a["name"]) + } + } + } + if packs != 1 { + t.Errorf("the build packs files/ into %d archives, not one", packs) + } +} diff --git a/modules/mounts/files/mounts.conf b/modules/mounts/files/mounts.conf new file mode 100644 index 00000000..1d5f87dd --- /dev/null +++ b/modules/mounts/files/mounts.conf @@ -0,0 +1,11 @@ +# Written by the mesh (module mounts, novox/hq ADR 0263) from this machine's assignment: the settings +# shares and sources, and the binding of nfs-share. Replaced on every push; change the settings, not this. +# The module's process reads it as root every 30 seconds and writes one .mount and .automount per entry. +shares=${config:shares} +sources=${config:sources} +server=${bound:nfs-share:from} +at=${bound:nfs-share:at} +account=${machine:account} +node=${machine:name} +passwords=${dir:state} +state=${dir:state} diff --git a/modules/mounts/module.json b/modules/mounts/module.json index 108133c9..9424dc1c 100644 --- a/modules/mounts/module.json +++ b/modules/mounts/module.json @@ -38,18 +38,20 @@ "per-machine": true } ], - "settings": { - "shares": { - "kind": "preference", - "default": "none", - "why": "a machine mounts no share of another until its assignment names one: space-separated name=mountpoint, each optionally :ro (hq ADR 0263 rule 5)" + "config": { + "shares": { + "type": "string", + "description": "The shares of another machine this machine mounts: space-separated name=mountpoint, each optionally :ro, or none (novox/hq ADR 0263 rule 5).", + "trusted": true, + "default": "none" + }, + "sources": { + "type": "string", + "description": "The occasional sources this machine mounts: space-separated name=smb://[@]/@ or name=disk:@, each optionally :ro, or none (novox/hq ADR 0263 rule 6); an SMB source's username is in its entry, its password the secret smb-password- (novox/hq ADR 0283).", + "trusted": true, + "default": "none" + } }, - "sources": { - "kind": "preference", - "default": "none", - "why": "a machine has no occasional source until its assignment names one: space-separated name=smb://[@]/@ or name=disk:@, each optionally :ro (hq ADR 0263 rule 6); an SMB source's username is in its entry, its password the secret smb-password- (hq ADR 0283)" - } - }, "own-secrets": { "smb-password-*": { "path": "${dir:state}/smb-password-*.secret", @@ -84,7 +86,7 @@ "type": "file", "path": "/etc/mesh-mounts/mounts.conf", "mode": "0644", - "content": "# Written by the mesh (module mounts, novox/hq ADR 0263) from this machine's assignment: the settings\n# shares and sources, and the binding of nfs-share. Replaced on every push; change the settings, not this.\n# The module's process reads it as root every 30 seconds and writes one .mount and .automount per entry.\nshares=${setting:shares}\nsources=${setting:sources}\nserver=${bound:nfs-share:from}\nat=${bound:nfs-share:at}\naccount=${machine:account}\nnode=${machine:name}\npasswords=${dir:state}\nstate=${dir:state}\n" + "template": "files/mounts.conf" }, { "id": "bus-dir-parent", @@ -143,6 +145,11 @@ "loads": [ "mounts" ] + }, + { + "name": "files", + "kind": "archive", + "from": "files" } ] } diff --git a/modules/mssql/files/backup.sql b/modules/mssql/files/backup.sql new file mode 100644 index 00000000..fd2eccec --- /dev/null +++ b/modules/mssql/files/backup.sql @@ -0,0 +1,14 @@ +SET NOCOUNT ON; +DECLARE @n sysname, @s nvarchar(max); +DECLARE c CURSOR LOCAL FAST_FORWARD FOR + SELECT name FROM sys.databases WHERE database_id > 4 AND state = 0 AND source_database_id IS NULL; +OPEN c; +FETCH NEXT FROM c INTO @n; +WHILE @@FETCH_STATUS = 0 +BEGIN + SET @s = N'BACKUP DATABASE ' + QUOTENAME(@n) + N' TO DISK = N''/var/opt/mssql/backup/' + REPLACE(@n, N'''', N'''''') + N'.bak'' WITH INIT, COPY_ONLY, CHECKSUM'; + EXEC (@s); + FETCH NEXT FROM c INTO @n; +END +CLOSE c; +DEALLOCATE c; diff --git a/modules/mssql/files/sa.env b/modules/mssql/files/sa.env new file mode 100644 index 00000000..e7dd4445 --- /dev/null +++ b/modules/mssql/files/sa.env @@ -0,0 +1,2 @@ +ACCEPT_EULA=Y +MSSQL_SA_PASSWORD=${secret:sa} diff --git a/modules/mssql/module.json b/modules/mssql/module.json index 2ccbba55..e30276c9 100644 --- a/modules/mssql/module.json +++ b/modules/mssql/module.json @@ -92,7 +92,7 @@ "type": "file", "path": "${dir:state}/sa.env", "mode": "0600", - "content": "ACCEPT_EULA=Y\nMSSQL_SA_PASSWORD=${secret:sa}\n" + "template": "files/sa.env" }, { "id": "data", @@ -134,7 +134,7 @@ "type": "file", "path": "${dir:state}/backup.sql", "mode": "0600", - "content": "SET NOCOUNT ON;\nDECLARE @n sysname, @s nvarchar(max);\nDECLARE c CURSOR LOCAL FAST_FORWARD FOR\n SELECT name FROM sys.databases WHERE database_id > 4 AND state = 0 AND source_database_id IS NULL;\nOPEN c;\nFETCH NEXT FROM c INTO @n;\nWHILE @@FETCH_STATUS = 0\nBEGIN\n SET @s = N'BACKUP DATABASE ' + QUOTENAME(@n) + N' TO DISK = N''/var/opt/mssql/backup/' + REPLACE(@n, N'''', N'''''') + N'.bak'' WITH INIT, COPY_ONLY, CHECKSUM';\n EXEC (@s);\n FETCH NEXT FROM c INTO @n;\nEND\nCLOSE c;\nDEALLOCATE c;\n" + "template": "files/backup.sql" } ], "build": { @@ -159,6 +159,11 @@ "MESH_RECEIVES": "${dir:grants}/mesh.json", "MESH_MSSQL_READER_PASSWORD_FILE": "${dir:state}/reader.secret" } + }, + { + "name": "files", + "kind": "archive", + "from": "files" } ] } diff --git a/modules/n8n/files/n8n-database.secret b/modules/n8n/files/n8n-database.secret new file mode 100644 index 00000000..e811e882 --- /dev/null +++ b/modules/n8n/files/n8n-database.secret @@ -0,0 +1 @@ +${secret:postgres-database} \ No newline at end of file diff --git a/modules/n8n/files/server.env b/modules/n8n/files/server.env new file mode 100644 index 00000000..2f74e0f8 --- /dev/null +++ b/modules/n8n/files/server.env @@ -0,0 +1,12 @@ +N8N_HOST=${bound:route:name} +N8N_PORT=5678 +N8N_PROTOCOL=https +WEBHOOK_URL=https://${bound:route:name}/ +NODE_FUNCTION_ALLOW_BUILTIN=* +NODE_FUNCTION_ALLOW_EXTERNAL=* +DB_TYPE=postgresdb +DB_POSTGRESDB_HOST=${bound:postgres-database:at} +DB_POSTGRESDB_PORT=${bound:postgres-database:port} +DB_POSTGRESDB_DATABASE=${bound:postgres-database:as} +DB_POSTGRESDB_USER=${bound:postgres-database:as} +DB_POSTGRESDB_PASSWORD_FILE=/run/secrets/database diff --git a/modules/n8n/module.json b/modules/n8n/module.json index 8e89145f..23874b61 100644 --- a/modules/n8n/module.json +++ b/modules/n8n/module.json @@ -80,14 +80,14 @@ "path": "${dir:state}/n8n-database.secret", "mode": "0400", "owner": "1000:1000", - "content": "${secret:postgres-database}" + "template": "files/n8n-database.secret" }, { "id": "server-env", "type": "file", "path": "${dir:state}/server.env", "mode": "0600", - "content": "N8N_HOST=${bound:route:name}\nN8N_PORT=5678\nN8N_PROTOCOL=https\nWEBHOOK_URL=https://${bound:route:name}/\nNODE_FUNCTION_ALLOW_BUILTIN=*\nNODE_FUNCTION_ALLOW_EXTERNAL=*\nDB_TYPE=postgresdb\nDB_POSTGRESDB_HOST=${bound:postgres-database:at}\nDB_POSTGRESDB_PORT=${bound:postgres-database:port}\nDB_POSTGRESDB_DATABASE=${bound:postgres-database:as}\nDB_POSTGRESDB_USER=${bound:postgres-database:as}\nDB_POSTGRESDB_PASSWORD_FILE=/run/secrets/database\n" + "template": "files/server.env" }, { "id": "net", @@ -159,6 +159,11 @@ "name": "server", "kind": "image", "from": "Dockerfile" + }, + { + "name": "files", + "kind": "archive", + "from": "files" } ] } diff --git a/modules/nats/files/nats.conf b/modules/nats/files/nats.conf new file mode 100644 index 00000000..e1d33eb7 --- /dev/null +++ b/modules/nats/files/nats.conf @@ -0,0 +1,46 @@ +# The nats module's own server settings. Declared by the module, because a port, a TLS path +# and a store directory are properties of the container this module raises: they live in its +# image and its mounts and change when it does. +# +# The mesh writes accounts.conf beside this one and nothing else. A controller that wrote the +# whole file would have to be kept in step with a Dockerfile it never sees. + +port: 4222 +# Monitoring on every interface *inside* the container, so the publish below can reach it; the publish +# is 127.0.0.1:8222 on the machine, so nothing beyond the machine reaches it, and the module guards 8222 +# too. Bound to the container's own loopback until 2026-10-04, the published port answered nothing +# (connection reset), and the only way in was `docker exec`. +http: 0.0.0.0:8222 + +# The mesh's own broker certificate — the one every machine already pins by fingerprint and the +# controller already trusts (MESH_BROKER_CERTIFICATE). Serving the new bus with it means no +# machine's pin changes when it moves, and no second certificate exists to be wrong about. +tls { + cert_file: "/tls/tls.crt" + key_file: "/tls/tls.key" +} + +# **No `verify`, deliberately, and it was `verify: true` until a probe ran this image.** That +# setting makes the server demand a *client* certificate, and nothing in the mesh presents one: a +# host pins this server's exact certificate and authenticates with the password the mesh minted +# (novox/hq ADR 0004, design 25 §4), and so does a module's runtime. With it on, every connection +# in the mesh is refused at the TLS handshake, before any password is looked at — and the error is +# "client didn't provide a certificate", which reads as a client fault. +# +# TLS is still required: a tls block is what makes it required, and verify only decides whether +# client certificates are checked. What is given up is a second factor the mesh has no machinery +# to issue or rotate — a certificate per module per node — and what is kept is stronger than a +# name check in both directions: an exact pin outward, a per-user password inward. + +jetstream { + store_dir: "/data" +} + +# Every user of the mesh, composed by the controller and rewritten whenever a module is +# assigned, a node enrols or a person's access changes. +# +# **Relative, and in this same directory, because it has to be.** An absolute include path is +# resolved relative to the including file's directory, not from the root: nats-server given +# `include /etc/nats/accounts.conf` from /etc/nats-server/nats.conf looks for +# /etc/nats-server/etc/nats/accounts.conf and refuses to start. Verified against the server. +include accounts.conf diff --git a/modules/nats/module.json b/modules/nats/module.json index d8647b3e..3c633473 100644 --- a/modules/nats/module.json +++ b/modules/nats/module.json @@ -98,7 +98,7 @@ "id": "server-conf", "type": "file", "path": "/var/lib/nats-module/conf/nats.conf", - "content": "# The nats module's own server settings. Declared by the module, because a port, a TLS path\n# and a store directory are properties of the container this module raises: they live in its\n# image and its mounts and change when it does.\n#\n# The mesh writes accounts.conf beside this one and nothing else. A controller that wrote the\n# whole file would have to be kept in step with a Dockerfile it never sees.\n\nport: 4222\n# Monitoring on every interface *inside* the container, so the publish below can reach it; the publish\n# is 127.0.0.1:8222 on the machine, so nothing beyond the machine reaches it, and the module guards 8222\n# too. Bound to the container's own loopback until 2026-10-04, the published port answered nothing\n# (connection reset), and the only way in was `docker exec`.\nhttp: 0.0.0.0:8222\n\n# The mesh's own broker certificate — the one every machine already pins by fingerprint and the\n# controller already trusts (MESH_BROKER_CERTIFICATE). Serving the new bus with it means no\n# machine's pin changes when it moves, and no second certificate exists to be wrong about.\ntls {\n cert_file: \"/tls/tls.crt\"\n key_file: \"/tls/tls.key\"\n}\n\n# **No `verify`, deliberately, and it was `verify: true` until a probe ran this image.** That\n# setting makes the server demand a *client* certificate, and nothing in the mesh presents one: a\n# host pins this server's exact certificate and authenticates with the password the mesh minted\n# (novox/hq ADR 0004, design 25 §4), and so does a module's runtime. With it on, every connection\n# in the mesh is refused at the TLS handshake, before any password is looked at — and the error is\n# \"client didn't provide a certificate\", which reads as a client fault.\n#\n# TLS is still required: a tls block is what makes it required, and verify only decides whether\n# client certificates are checked. What is given up is a second factor the mesh has no machinery\n# to issue or rotate — a certificate per module per node — and what is kept is stronger than a\n# name check in both directions: an exact pin outward, a per-user password inward.\n\njetstream {\n store_dir: \"/data\"\n}\n\n# Every user of the mesh, composed by the controller and rewritten whenever a module is\n# assigned, a node enrols or a person's access changes.\n#\n# **Relative, and in this same directory, because it has to be.** An absolute include path is\n# resolved relative to the including file's directory, not from the root: nats-server given\n# `include /etc/nats/accounts.conf` from /etc/nats-server/nats.conf looks for\n# /etc/nats-server/etc/nats/accounts.conf and refuses to start. Verified against the server.\ninclude accounts.conf\n", + "template": "files/nats.conf", "mode": "0644" }, { @@ -156,6 +156,11 @@ "MESH_NATS_CONTAINER": "mesh-broker-nats", "MESH_NATS_USERS_FILE": "/etc/nats/accounts.conf" } + }, + { + "name": "files", + "kind": "archive", + "from": "files" } ] } diff --git a/modules/nats/snapshot/templates_test.go b/modules/nats/snapshot/templates_test.go new file mode 100644 index 00000000..fbd1490a --- /dev/null +++ b/modules/nats/snapshot/templates_test.go @@ -0,0 +1,64 @@ +package main + +import ( + "encoding/json" + "os" + "path/filepath" + "testing" +) + +// The module's directory, where its manifest and its files/ are. +const moduleDir = ".." + +// templateText is the text a file resource places on the machine: its template's, in the module's files/ +// (novox/hq ADR 0349 §2). A resource that names no template, or a template that is not there, fails the test. +func templateText(t *testing.T, r map[string]any) string { + t.Helper() + name, _ := r["template"].(string) + if name == "" { + t.Fatalf("resource %v names no template", r["id"]) + } + raw, err := os.ReadFile(filepath.Join(moduleDir, filepath.FromSlash(name))) + if err != nil { + t.Fatalf("resource %v names %s: %v", r["id"], name, err) + } + return string(raw) +} + +// No file resource carries its text inline: each names its template in files/, and the build packs files/ +// into the one archive the controller reads the templates back from (novox/hq ADR 0349 §2). +func TestEveryFileTakesItsTextFromATemplate(t *testing.T) { + raw, err := os.ReadFile(filepath.Join(moduleDir, "module.json")) + if err != nil { + t.Fatal(err) + } + var m struct { + Resources []map[string]any `json:"resources"` + Build struct { + Artifacts []map[string]any `json:"artifacts"` + } `json:"build"` + } + if err := json.Unmarshal(raw, &m); err != nil { + t.Fatal(err) + } + for _, r := range m.Resources { + if _, inline := r["content"]; inline { + t.Errorf("resource %v keeps its text inline: keep it in files/ and name it as the template", r["id"]) + } + if r["type"] == "file" { + templateText(t, r) + } + } + packs := 0 + for _, a := range m.Build.Artifacts { + if a["kind"] == "archive" && a["from"] == "files" { + packs++ + if a["name"] != "files" { + t.Errorf("the templates archive is named %v, not files", a["name"]) + } + } + } + if packs != 1 { + t.Errorf("the build packs files/ into %d archives, not one", packs) + } +} diff --git a/modules/networkmanager/cmd/uplink-tools/templates_test.go b/modules/networkmanager/cmd/uplink-tools/templates_test.go new file mode 100644 index 00000000..7d0718b5 --- /dev/null +++ b/modules/networkmanager/cmd/uplink-tools/templates_test.go @@ -0,0 +1,64 @@ +package main + +import ( + "encoding/json" + "os" + "path/filepath" + "testing" +) + +// The module's directory, where its manifest and its files/ are. +const moduleDir = "../.." + +// templateText is the text a file resource places on the machine: its template's, in the module's files/ +// (novox/hq ADR 0349 §2). A resource that names no template, or a template that is not there, fails the test. +func templateText(t *testing.T, r map[string]any) string { + t.Helper() + name, _ := r["template"].(string) + if name == "" { + t.Fatalf("resource %v names no template", r["id"]) + } + raw, err := os.ReadFile(filepath.Join(moduleDir, filepath.FromSlash(name))) + if err != nil { + t.Fatalf("resource %v names %s: %v", r["id"], name, err) + } + return string(raw) +} + +// No file resource carries its text inline: each names its template in files/, and the build packs files/ +// into the one archive the controller reads the templates back from (novox/hq ADR 0349 §2). +func TestEveryFileTakesItsTextFromATemplate(t *testing.T) { + raw, err := os.ReadFile(filepath.Join(moduleDir, "module.json")) + if err != nil { + t.Fatal(err) + } + var m struct { + Resources []map[string]any `json:"resources"` + Build struct { + Artifacts []map[string]any `json:"artifacts"` + } `json:"build"` + } + if err := json.Unmarshal(raw, &m); err != nil { + t.Fatal(err) + } + for _, r := range m.Resources { + if _, inline := r["content"]; inline { + t.Errorf("resource %v keeps its text inline: keep it in files/ and name it as the template", r["id"]) + } + if r["type"] == "file" { + templateText(t, r) + } + } + packs := 0 + for _, a := range m.Build.Artifacts { + if a["kind"] == "archive" && a["from"] == "files" { + packs++ + if a["name"] != "files" { + t.Errorf("the templates archive is named %v, not files", a["name"]) + } + } + } + if packs != 1 { + t.Errorf("the build packs files/ into %d archives, not one", packs) + } +} diff --git a/modules/networkmanager/files/50-mesh.conf b/modules/networkmanager/files/50-mesh.conf new file mode 100644 index 00000000..f8c8e8a2 --- /dev/null +++ b/modules/networkmanager/files/50-mesh.conf @@ -0,0 +1,63 @@ +# Managed by the mesh (module networkmanager). Replaced on every push; edit the +# catalogue instead. +# +# This machine's uplink is NetworkManager's, and this file is the whole of what +# the mesh asks of it (novox/hq ADR 0117): leave the resolver file to the mesh, +# and leave the private network's interface alone. Nothing more. The mesh never +# declares a connection profile, an address, a route, a wireless network or its +# credentials — those are joined at the machine, by the person using it, and +# the link they make is the only channel the mesh reaches this machine over. A +# push that got a link wrong could not be undone by the next one. +# +# A drop-in of the mesh's own, beside NetworkManager.conf and whatever else the +# operator keeps in this directory. NetworkManager reads the files here sorted by +# name and a later one wins a key it sets again — so a file of the operator's +# that sorts after this one (any name starting with a letter does) and sets dns= +# or unmanaged-devices= overrides it. That is the operator's to decide, and the +# reason this file sets nothing but the two keys it must. +# +# NetworkManager itself is the machine's: the mesh never starts, stops, enables +# or disables it (its service is declared with no state), because stopping it +# takes every link down, this machine's channel to the mesh included — and a +# module unassigned by mistake must not be able to do that. When this file +# changes, a running NetworkManager is reloaded (its D-Bus Reload call, which +# re-reads its configuration — NetworkManager(8)), never restarted. + +[main] +# The resolver file is the mesh's: this module writes /etc/resolv.conf itself, +# listing the mesh's resolvers (novox/hq ADR 0223). Without this line +# NetworkManager rewrites that file on +# every connectivity change — every network joined, every lease renewed — +# and the mesh's resolver is silently replaced while every surface of the mesh +# still reads green. none: "NetworkManager will not modify resolv.conf. This +# implies rc-manager unmanaged" (NetworkManager.conf(5), 1.58). On an adopted +# machine the predecessor wrote the same line in a file of its own; both say one +# thing, and the predecessor's is retired by hand after the take. +# +# Not NetworkManager's own global DNS ([global-dns-domain-*] with rc-manager=file): +# it writes its own header and composes the options line itself, so it cannot +# write the mesh's file byte for byte, and the three uplink modules would write +# three different files for one fact. dns=none, and the file declared beside it. +dns=none + +[keyfile] +# mesh0 is the private network's interface: the mesh brings it up and the mesh +# alone configures it. A manager that considers every interface its own could +# try to configure it, or tear it down on a profile change. +# +# unmanaged-devices rather than a [device-mesh0] section with managed=0, because +# NetworkManager.conf(5) says a device unmanaged by this key "is strictly +# unmanaged and cannot be overruled by using the API like nmcli device set +# $IFNAME managed yes", while device*.managed "can be overruled at runtime via +# D-Bus". The same page adds that device*.managed "may be a better choice" for +# exactly those reasons — for an interface the operator might want to hand back +# at runtime. For the mesh's own interface, strict is the point. +# +# += rather than =: the same page documents appending to a list-valued key set +# earlier ("plugins+=another-plugin") as an extension of its key file format, +# and unmanaged-devices is a device list. = would replace whatever devices the +# operator already keeps NetworkManager away from; += adds this one to them +# (novox/hq ADR 0102: a list is added to, never replaced). A file of the +# operator's read after this one that sets the key with = replaces it again; +# that is the operator's to decide. +unmanaged-devices+=interface-name:mesh0 diff --git a/modules/networkmanager/module.json b/modules/networkmanager/module.json index 6d0cbaed..68ace345 100644 --- a/modules/networkmanager/module.json +++ b/modules/networkmanager/module.json @@ -32,6 +32,11 @@ "loads": [ "uplink-tools" ] + }, + { + "name": "files", + "kind": "archive", + "from": "files" } ] }, @@ -52,7 +57,7 @@ "type": "file", "path": "/etc/NetworkManager/conf.d/50-mesh.conf", "mode": "0644", - "content": "# Managed by the mesh (module networkmanager). Replaced on every push; edit the\n# catalogue instead.\n#\n# This machine's uplink is NetworkManager's, and this file is the whole of what\n# the mesh asks of it (novox/hq ADR 0117): leave the resolver file to the mesh,\n# and leave the private network's interface alone. Nothing more. The mesh never\n# declares a connection profile, an address, a route, a wireless network or its\n# credentials \u2014 those are joined at the machine, by the person using it, and\n# the link they make is the only channel the mesh reaches this machine over. A\n# push that got a link wrong could not be undone by the next one.\n#\n# A drop-in of the mesh's own, beside NetworkManager.conf and whatever else the\n# operator keeps in this directory. NetworkManager reads the files here sorted by\n# name and a later one wins a key it sets again \u2014 so a file of the operator's\n# that sorts after this one (any name starting with a letter does) and sets dns=\n# or unmanaged-devices= overrides it. That is the operator's to decide, and the\n# reason this file sets nothing but the two keys it must.\n#\n# NetworkManager itself is the machine's: the mesh never starts, stops, enables\n# or disables it (its service is declared with no state), because stopping it\n# takes every link down, this machine's channel to the mesh included \u2014 and a\n# module unassigned by mistake must not be able to do that. When this file\n# changes, a running NetworkManager is reloaded (its D-Bus Reload call, which\n# re-reads its configuration \u2014 NetworkManager(8)), never restarted.\n\n[main]\n# The resolver file is the mesh's: this module writes /etc/resolv.conf itself,\n# listing the mesh's resolvers (novox/hq ADR 0223). Without this line\n# NetworkManager rewrites that file on\n# every connectivity change \u2014 every network joined, every lease renewed \u2014\n# and the mesh's resolver is silently replaced while every surface of the mesh\n# still reads green. none: \"NetworkManager will not modify resolv.conf. This\n# implies rc-manager unmanaged\" (NetworkManager.conf(5), 1.58). On an adopted\n# machine the predecessor wrote the same line in a file of its own; both say one\n# thing, and the predecessor's is retired by hand after the take.\n#\n# Not NetworkManager's own global DNS ([global-dns-domain-*] with rc-manager=file):\n# it writes its own header and composes the options line itself, so it cannot\n# write the mesh's file byte for byte, and the three uplink modules would write\n# three different files for one fact. dns=none, and the file declared beside it.\ndns=none\n\n[keyfile]\n# mesh0 is the private network's interface: the mesh brings it up and the mesh\n# alone configures it. A manager that considers every interface its own could\n# try to configure it, or tear it down on a profile change.\n#\n# unmanaged-devices rather than a [device-mesh0] section with managed=0, because\n# NetworkManager.conf(5) says a device unmanaged by this key \"is strictly\n# unmanaged and cannot be overruled by using the API like nmcli device set\n# $IFNAME managed yes\", while device*.managed \"can be overruled at runtime via\n# D-Bus\". The same page adds that device*.managed \"may be a better choice\" for\n# exactly those reasons \u2014 for an interface the operator might want to hand back\n# at runtime. For the mesh's own interface, strict is the point.\n#\n# += rather than =: the same page documents appending to a list-valued key set\n# earlier (\"plugins+=another-plugin\") as an extension of its key file format,\n# and unmanaged-devices is a device list. = would replace whatever devices the\n# operator already keeps NetworkManager away from; += adds this one to them\n# (novox/hq ADR 0102: a list is added to, never replaced). A file of the\n# operator's read after this one that sets the key with = replaces it again;\n# that is the operator's to decide.\nunmanaged-devices+=interface-name:mesh0\n" + "template": "files/50-mesh.conf" }, { "id": "service", diff --git a/modules/nextcloud/files/config.json b/modules/nextcloud/files/config.json new file mode 100644 index 00000000..0967ef42 --- /dev/null +++ b/modules/nextcloud/files/config.json @@ -0,0 +1 @@ +{} diff --git a/modules/nextcloud/files/server.env b/modules/nextcloud/files/server.env new file mode 100644 index 00000000..df6e20f6 --- /dev/null +++ b/modules/nextcloud/files/server.env @@ -0,0 +1,14 @@ +POSTGRES_HOST=${bound:postgres-database:at}:${bound:postgres-database:port} +POSTGRES_DB=${bound:postgres-database:as} +POSTGRES_USER=${bound:postgres-database:as} +POSTGRES_PASSWORD=${secret:postgres-database} +NEXTCLOUD_ADMIN_USER=mesh-admin +NEXTCLOUD_ADMIN_PASSWORD=${secret:admin} +OBJECTSTORE_S3_HOST=${bound:s3-bucket:at} +OBJECTSTORE_S3_PORT=${bound:s3-bucket:port} +OBJECTSTORE_S3_BUCKET=${bound:s3-bucket:bucket} +OBJECTSTORE_S3_KEY=${bound:s3-bucket:as} +OBJECTSTORE_S3_SECRET=${secret:s3-bucket} +OBJECTSTORE_S3_SSL=false +OBJECTSTORE_S3_USEPATH_STYLE=true +OBJECTSTORE_S3_REGION=${bound:s3-bucket:region} diff --git a/modules/nextcloud/module.json b/modules/nextcloud/module.json index 2d40594c..c7d32627 100644 --- a/modules/nextcloud/module.json +++ b/modules/nextcloud/module.json @@ -54,6 +54,26 @@ } ] }, + "config": { + "container": { + "type": "string", + "description": "The name of the container the module's tools run Nextcloud's occ in; empty is nextcloud.", + "trusted": true, + "default": "" + }, + "url": { + "type": "string", + "description": "The address the module's tools reach Nextcloud's sharing API at, as a URL; empty is this machine's Nextcloud on its port.", + "trusted": true, + "default": "" + }, + "user": { + "type": "string", + "description": "The Nextcloud account the module's tools act as; empty is admin.", + "trusted": true, + "default": "" + } + }, "resources": [ { "id": "mesh-state", @@ -72,7 +92,7 @@ "type": "file", "path": "${dir:state}/server.env", "mode": "0600", - "content": "POSTGRES_HOST=${bound:postgres-database:at}:${bound:postgres-database:port}\nPOSTGRES_DB=${bound:postgres-database:as}\nPOSTGRES_USER=${bound:postgres-database:as}\nPOSTGRES_PASSWORD=${secret:postgres-database}\nNEXTCLOUD_ADMIN_USER=mesh-admin\nNEXTCLOUD_ADMIN_PASSWORD=${secret:admin}\nOBJECTSTORE_S3_HOST=${bound:s3-bucket:at}\nOBJECTSTORE_S3_PORT=${bound:s3-bucket:port}\nOBJECTSTORE_S3_BUCKET=${bound:s3-bucket:bucket}\nOBJECTSTORE_S3_KEY=${bound:s3-bucket:as}\nOBJECTSTORE_S3_SECRET=${secret:s3-bucket}\nOBJECTSTORE_S3_SSL=false\nOBJECTSTORE_S3_USEPATH_STYLE=true\nOBJECTSTORE_S3_REGION=${bound:s3-bucket:region}\n" + "template": "files/server.env" }, { "id": "html", @@ -109,7 +129,8 @@ "type": "file", "path": "${dir:mesh-state}/config.json", "mode": "0600", - "content": "{}\n", + "template": "files/config.json", + "takes-config": ["container", "url", "user"], "merge": "json" } ], @@ -133,6 +154,11 @@ "MESH_NEXTCLOUD_ADMIN_USER": "mesh-admin", "MESH_NEXTCLOUD_ADMIN_PASSWORD_FILE": "${dir:state}/admin.secret" } + }, + { + "name": "files", + "kind": "archive", + "from": "files" } ] } diff --git a/modules/nfs-server/cmd/nfs-server/manifest_test.go b/modules/nfs-server/cmd/nfs-server/manifest_test.go index f9e12319..d43d5776 100644 --- a/modules/nfs-server/cmd/nfs-server/manifest_test.go +++ b/modules/nfs-server/cmd/nfs-server/manifest_test.go @@ -22,8 +22,8 @@ type manifest struct { Protocol string `json:"protocol"` From string `json:"from"` } `json:"listens"` - Settings map[string]any `json:"settings"` - Resources []map[string]any `json:"resources"` + Config map[string]map[string]any `json:"config"` + Resources []map[string]any `json:"resources"` } func readManifest(t *testing.T) (manifest, string) { @@ -83,28 +83,34 @@ func TestItListensOnTwoThousandFortyNineFromTheMeshOnlyAndProvidesTheShare(t *te t.Errorf("the module opens %s", port) } } - if !strings.Contains(raw, "vers3=n") { + nfsConf := "" + for _, r := range m.Resources { + if r["id"] == "nfs-conf" { + nfsConf = templateText(t, r) + } + } + if !strings.Contains(nfsConf, "vers3=n") { t.Errorf("NFS version 3 is not switched off, so 2049 alone would not do") } } // The definition names no folder, no machine and no person (novox/hq ADR 0112): the shares are the -// assignment's setting, and the range the mesh's. +// assignment's config value, required and with no default (ADR 0348 §4), and the range the mesh's. func TestTheDefinitionNamesNoFolderNoMachineAndGivesTheSharesNoDefault(t *testing.T) { m, raw := readManifest(t) if hit := regexp.MustCompile(`\b(g14|novox|ace|shanks|jochen)\b|/home/|/storage|/srv/`).FindString(strings.ReplaceAll(raw, "novox/hq", "")); hit != "" { t.Errorf("the manifest says %q", hit) } - if _, declared := m.Settings["shares"]; declared { - t.Errorf("the shares are the operator's value and have no default (ADR 0262)") + if s := m.Config["shares"]; s["required"] != true || s["default"] != nil || s["type"] != "string" { + t.Errorf("the shares are the operator's value: required, with no default (ADR 0348 §4): %v", s) } var config string for _, r := range m.Resources { if r["id"] == "config" { - config, _ = r["content"].(string) + config = templateText(t, r) } } - if !strings.Contains(config, "shares=${setting:shares}") || !strings.Contains(config, "range=${machine:mesh-range}") { + if !strings.Contains(config, "shares=${config:shares}") || !strings.Contains(config, "range=${machine:mesh-range}") { t.Errorf("the config file: %q", config) } // Every long-running resource says how it is ready (ADR 0240 rule 8). @@ -124,9 +130,9 @@ func TestTheDefinitionNamesNoFolderNoMachineAndGivesTheSharesNoDefault(t *testin func TestTheManifestCarriesGrantsTheRosterTheWantsAndTheRemovalOrder(t *testing.T) { m, raw := readManifest(t) var full struct { - Reads []string `json:"reads"` - Facts map[string]map[string]string `json:"facts"` - Settings map[string]map[string]any `json:"settings"` + Reads []string `json:"reads"` + Facts map[string]map[string]string `json:"facts"` + Config map[string]map[string]any `json:"config"` } if err := json.Unmarshal([]byte(raw), &full); err != nil { t.Fatal(err) @@ -138,7 +144,7 @@ func TestTheManifestCarriesGrantsTheRosterTheWantsAndTheRemovalOrder(t *testing. if roster["path"] != RosterFile || !strings.Contains(roster["template"], "{{range .Machines}}{{.Name}} {{.Address}}") { t.Errorf("the roster fact: %v", roster) } - if g := full.Settings["grants"]; g["kind"] != "preference" || g["default"] != "none" { + if g := full.Config["grants"]; g["type"] != "string" || g["default"] != "none" || g["trusted"] != true { t.Errorf("grants: %v", g) } order := map[string]int{} @@ -146,13 +152,13 @@ func TestTheManifestCarriesGrantsTheRosterTheWantsAndTheRemovalOrder(t *testing. for i, r := range m.Resources { order[r["id"].(string)] = i if r["id"] == "config" { - config, _ = r["content"].(string) + config = templateText(t, r) } if r["path"] == WantsDir && r["owner"] != "${machine:account}" { t.Errorf("the tools, as the operator account, cannot write the wants: %v", r) } } - if !strings.Contains(config, "grants=${setting:grants}") { + if !strings.Contains(config, "grants=${config:grants}") { t.Errorf("the config lacks the grants: %q", config) } if _, ok := order["wants-dir"]; !ok { diff --git a/modules/nfs-server/cmd/nfs-server/templates_test.go b/modules/nfs-server/cmd/nfs-server/templates_test.go new file mode 100644 index 00000000..7d0718b5 --- /dev/null +++ b/modules/nfs-server/cmd/nfs-server/templates_test.go @@ -0,0 +1,64 @@ +package main + +import ( + "encoding/json" + "os" + "path/filepath" + "testing" +) + +// The module's directory, where its manifest and its files/ are. +const moduleDir = "../.." + +// templateText is the text a file resource places on the machine: its template's, in the module's files/ +// (novox/hq ADR 0349 §2). A resource that names no template, or a template that is not there, fails the test. +func templateText(t *testing.T, r map[string]any) string { + t.Helper() + name, _ := r["template"].(string) + if name == "" { + t.Fatalf("resource %v names no template", r["id"]) + } + raw, err := os.ReadFile(filepath.Join(moduleDir, filepath.FromSlash(name))) + if err != nil { + t.Fatalf("resource %v names %s: %v", r["id"], name, err) + } + return string(raw) +} + +// No file resource carries its text inline: each names its template in files/, and the build packs files/ +// into the one archive the controller reads the templates back from (novox/hq ADR 0349 §2). +func TestEveryFileTakesItsTextFromATemplate(t *testing.T) { + raw, err := os.ReadFile(filepath.Join(moduleDir, "module.json")) + if err != nil { + t.Fatal(err) + } + var m struct { + Resources []map[string]any `json:"resources"` + Build struct { + Artifacts []map[string]any `json:"artifacts"` + } `json:"build"` + } + if err := json.Unmarshal(raw, &m); err != nil { + t.Fatal(err) + } + for _, r := range m.Resources { + if _, inline := r["content"]; inline { + t.Errorf("resource %v keeps its text inline: keep it in files/ and name it as the template", r["id"]) + } + if r["type"] == "file" { + templateText(t, r) + } + } + packs := 0 + for _, a := range m.Build.Artifacts { + if a["kind"] == "archive" && a["from"] == "files" { + packs++ + if a["name"] != "files" { + t.Errorf("the templates archive is named %v, not files", a["name"]) + } + } + } + if packs != 1 { + t.Errorf("the build packs files/ into %d archives, not one", packs) + } +} diff --git a/modules/nfs-server/files/50-mesh.conf b/modules/nfs-server/files/50-mesh.conf new file mode 100644 index 00000000..2640a8b2 --- /dev/null +++ b/modules/nfs-server/files/50-mesh.conf @@ -0,0 +1,12 @@ +# Written by the mesh (module nfs-server, novox/hq ADR 0263). Replaced on every push; a drop-in of +# the operator's that sorts after this one overrides it, and is theirs. +# +# NFS version 4 only: a client needs port 2049 and nothing else, so the module opens nothing more +# than that, to the private network. Version 3 needs rpcbind and mountd, on ports the mesh does not open. +[nfsd] +vers2=n +vers3=n +vers4=y +vers4.0=n +vers4.1=y +vers4.2=y diff --git a/modules/nfs-server/files/shares.conf b/modules/nfs-server/files/shares.conf new file mode 100644 index 00000000..f25f0261 --- /dev/null +++ b/modules/nfs-server/files/shares.conf @@ -0,0 +1,10 @@ +# Written by the mesh (module nfs-server, novox/hq ADR 0263) from this machine's assignment. +# Replaced on every push; change the `shares` setting, never this file. +# +# The shares: name=folder, or name=folder:ro, one share per folder. The grants: share=node:rw|ro[,…], +# which nodes may mount each and how. The module's process exports a share to a node's private address +# only when it is granted here and that node's mounts module wants it, every client mapped to the +# folder's owner, and only to addresses inside the range below. +shares=${config:shares} +grants=${config:grants} +range=${machine:mesh-range} diff --git a/modules/nfs-server/module.json b/modules/nfs-server/module.json index d67cddde..70b7f924 100644 --- a/modules/nfs-server/module.json +++ b/modules/nfs-server/module.json @@ -51,13 +51,20 @@ "seat:mesh-controller.seats", "seat:mesh-controller.data" ], - "settings": { - "grants": { - "kind": "preference", - "default": "none", - "why": "which nodes may mount which share, and how, is this machine's to say (hq ADR 0263, review of mesh-catalog #136): share=node:rw|ro[,node:rw|ro], entries separated by spaces; none exports to no node, whatever a node wants" - } - }, + "config": { + "shares": { + "type": "string", + "description": "The folders this machine exports, as space-separated name=path; the operator's, with no default (novox/hq ADR 0263).", + "trusted": true, + "required": true + }, + "grants": { + "type": "string", + "description": "Which nodes may mount which share, and how: share=node:rw|ro[,node:rw|ro], entries separated by spaces, or none to export to no node whatever a node wants (novox/hq ADR 0263).", + "trusted": true, + "default": "none" + } + }, "facts": { "machines": { "path": "/etc/nfs-server/machines", @@ -88,7 +95,7 @@ "type": "file", "path": "/etc/nfs.conf.d/50-mesh.conf", "mode": "0644", - "content": "# Written by the mesh (module nfs-server, novox/hq ADR 0263). Replaced on every push; a drop-in of\n# the operator's that sorts after this one overrides it, and is theirs.\n#\n# NFS version 4 only: a client needs port 2049 and nothing else, so the module opens nothing more\n# than that, to the private network. Version 3 needs rpcbind and mountd, on ports the mesh does not open.\n[nfsd]\nvers2=n\nvers3=n\nvers4=y\nvers4.0=n\nvers4.1=y\nvers4.2=y\n" + "template": "files/50-mesh.conf" }, { "id": "run-dir", @@ -154,7 +161,7 @@ "type": "file", "path": "/etc/nfs-server/shares.conf", "mode": "0644", - "content": "# Written by the mesh (module nfs-server, novox/hq ADR 0263) from this machine's assignment.\n# Replaced on every push; change the `shares` setting, never this file.\n#\n# The shares: name=folder, or name=folder:ro, one share per folder. The grants: share=node:rw|ro[,…],\n# which nodes may mount each and how. The module's process exports a share to a node's private address\n# only when it is granted here and that node's mounts module wants it, every client mapped to the\n# folder's owner, and only to addresses inside the range below.\nshares=${setting:shares}\ngrants=${setting:grants}\nrange=${machine:mesh-range}\n" + "template": "files/shares.conf" } ], "build": { @@ -169,6 +176,11 @@ "loads": [ "nfs-server" ] + }, + { + "name": "files", + "kind": "archive", + "from": "files" } ] } diff --git a/modules/nftables/files/mesh-filter.service b/modules/nftables/files/mesh-filter.service new file mode 100644 index 00000000..5a6ccedb --- /dev/null +++ b/modules/nftables/files/mesh-filter.service @@ -0,0 +1,14 @@ +[Unit] +Description=The mesh's packet filter, derived from what is assigned to this node +Wants=network-pre.target +Before=network-pre.target + +[Service] +Type=oneshot +RemainAfterExit=yes +ExecStart=nft -f /etc/nftables.conf +ExecReload=nft -f /etc/nftables.conf +ExecStop=nft delete table inet mesh + +[Install] +WantedBy=multi-user.target diff --git a/modules/nftables/files/mesh.conf b/modules/nftables/files/mesh.conf new file mode 100644 index 00000000..f1a7bb0d --- /dev/null +++ b/modules/nftables/files/mesh.conf @@ -0,0 +1,5 @@ +# The mesh: stopping the stock unit deletes only the mesh's table, never the whole ruleset +# (novox/hq ADR 0100) — a flush would take the container runtime's rules and any firewall with it. +[Service] +ExecStop= +ExecStop=nft delete table inet mesh diff --git a/modules/nftables/module.json b/modules/nftables/module.json index 2a80c58b..b7b0c83f 100644 --- a/modules/nftables/module.json +++ b/modules/nftables/module.json @@ -37,14 +37,14 @@ "id": "unit", "type": "file", "path": "/etc/systemd/system/mesh-filter.service", - "content": "[Unit]\nDescription=The mesh's packet filter, derived from what is assigned to this node\nWants=network-pre.target\nBefore=network-pre.target\n\n[Service]\nType=oneshot\nRemainAfterExit=yes\nExecStart=nft -f /etc/nftables.conf\nExecReload=nft -f /etc/nftables.conf\nExecStop=nft delete table inet mesh\n\n[Install]\nWantedBy=multi-user.target\n", + "template": "files/mesh-filter.service", "mode": "0644" }, { "id": "stock-unit-stop", "type": "file", "path": "/etc/systemd/system/nftables.service.d/mesh.conf", - "content": "# The mesh: stopping the stock unit deletes only the mesh's table, never the whole ruleset\n# (novox/hq ADR 0100) — a flush would take the container runtime's rules and any firewall with it.\n[Service]\nExecStop=\nExecStop=nft delete table inet mesh\n", + "template": "files/mesh.conf", "mode": "0644" }, { @@ -80,6 +80,11 @@ "entrypoints": [ "tools/index.js" ] + }, + { + "name": "files", + "kind": "archive", + "from": "files" } ] } diff --git a/modules/nftables/test/remove.test.ts b/modules/nftables/test/remove.test.ts index 8c705c95..17003d2f 100644 --- a/modules/nftables/test/remove.test.ts +++ b/modules/nftables/test/remove.test.ts @@ -77,7 +77,15 @@ test("which chains jump to a target is read from a listing", () => { test("the filter's commands run as given by root and through sudo without a prompt by anyone else", () => { assert.deepEqual(escalated("nft", ["list", "ruleset"], 0), ["nft", ["list", "ruleset"]]); assert.deepEqual(escalated("nft", ["-f", "/etc/nftables.conf"], 1000), ["sudo", ["-n", "nft", "-f", "/etc/nftables.conf"]]); - assert.deepEqual(escalated("iptables-legacy", ["-S"], undefined), ["sudo", ["-n", "iptables-legacy", "-S"]]); + assert.deepEqual(escalated("iptables-legacy", ["-S"], 1000), ["sudo", ["-n", "iptables-legacy", "-S"]]); +}); + +test("without a uid given, the process's own decides: as given for root, through sudo for anyone else", () => { + // Whoever runs the check: the default is the uid this process has, so the expectation is derived from it. + const own = process.getuid?.(); + const want: [string, string[]] = own === 0 ? ["iptables-legacy", ["-S"]] : ["sudo", ["-n", "iptables-legacy", "-S"]]; + assert.deepEqual(escalated("iptables-legacy", ["-S"]), want); + assert.deepEqual(escalated("iptables-legacy", ["-S"], undefined), want); }); test("the filter file is the one the manifest's filtering names", () => { diff --git a/modules/node-env/files/50-mesh.conf b/modules/node-env/files/50-mesh.conf new file mode 100644 index 00000000..3aebc873 --- /dev/null +++ b/modules/node-env/files/50-mesh.conf @@ -0,0 +1,4 @@ +# The operator account's environment for its service manager and graphical session, generated by +# the mesh (module node-env, novox/hq ADR 0203). Do not edit: this file is replaced at every push. +# The same facts as ~/.config/mesh/environment.sh, in environment.d(5) syntax. +${collected:environment:systemd} \ No newline at end of file diff --git a/modules/node-env/files/environment.sh b/modules/node-env/files/environment.sh new file mode 100644 index 00000000..c28adf34 --- /dev/null +++ b/modules/node-env/files/environment.sh @@ -0,0 +1,5 @@ +# The operator account's environment, generated by the mesh (module node-env, novox/hq ADR 0203). +# Do not edit: this file is replaced at every push. Every line names the module that contributed it. +# Sourced by the login shell from its always-read startup file (for zsh, ~/.zshenv), so a script, a +# login and the shell's execute verb all see it. Your own variables belong in your shell's own lines. +${collected:environment:posix} \ No newline at end of file diff --git a/modules/node-env/module.json b/modules/node-env/module.json index 43b27e88..a933578c 100644 --- a/modules/node-env/module.json +++ b/modules/node-env/module.json @@ -1,6 +1,7 @@ { "module": "node-env", "version": "1", + "moved": true, "claims": [ { "name": "node-environment", @@ -28,7 +29,7 @@ "path": "${machine:account-home}/.config/mesh/environment.sh", "owner": "${machine:account}", "mode": "0644", - "content": "# The operator account's environment, generated by the mesh (module node-env, novox/hq ADR 0203).\n# Do not edit: this file is replaced at every push. Every line names the module that contributed it.\n# Sourced by the login shell from its always-read startup file (for zsh, ~/.zshenv), so a script, a\n# login and the shell's execute verb all see it. Your own variables belong in your shell's own lines.\n${environment:posix}" + "template": "files/environment.sh" }, { "id": "systemd", @@ -36,7 +37,16 @@ "path": "${machine:account-home}/.config/environment.d/50-mesh.conf", "owner": "${machine:account}", "mode": "0644", - "content": "# The operator account's environment for its service manager and graphical session, generated by\n# the mesh (module node-env, novox/hq ADR 0203). Do not edit: this file is replaced at every push.\n# The same facts as ~/.config/mesh/environment.sh, in environment.d(5) syntax.\n${environment:systemd}" + "template": "files/50-mesh.conf" } - ] + ], + "build": { + "artifacts": [ + { + "name": "files", + "kind": "archive", + "from": "files" + } + ] + } } diff --git a/modules/nodered/files/config.json b/modules/nodered/files/config.json new file mode 100644 index 00000000..a396fbd1 --- /dev/null +++ b/modules/nodered/files/config.json @@ -0,0 +1,3 @@ +{ + "token": "${secret:api-token}" +} diff --git a/modules/nodered/files/mqtt.env b/modules/nodered/files/mqtt.env new file mode 100644 index 00000000..5331f3cf --- /dev/null +++ b/modules/nodered/files/mqtt.env @@ -0,0 +1,4 @@ +MESH_NODERED_URL=http://127.0.0.1:${port:1880} +MESH_NODERED_CONFIG_FILE=${dir:mesh-state}/config.json +MESH_PROVISIONS_DIR=${dir:state} +MESH_WRITTEN_DIR=${dir:written} diff --git a/modules/nodered/files/settings.js b/modules/nodered/files/settings.js new file mode 100644 index 00000000..0d0f3515 --- /dev/null +++ b/modules/nodered/files/settings.js @@ -0,0 +1,48 @@ +// Node-RED's settings, written by the mesh from the nodered module. What an assignment may change +// is settings.json beside this file (merged key by key); the credentials are the mesh's secrets and +// reach Node-RED only through this file. The flows' own credentials stay encrypted in the user +// directory under the key Node-RED keeps there (.config.runtime.json), which is data, not this. +const fs = require("fs"); +const path = require("path"); +const crypto = require("crypto"); + +const ADMIN_PASSWORD = "${secret:admin}"; +const API_TOKEN = "${secret:api-token}"; +const ADMIN = { username: "admin", permissions: "*" }; + +const settings = JSON.parse(fs.readFileSync(path.join(__dirname, "settings.json"), "utf8")); +// The mesh's keys, not Node-RED's: endpoints lands in every merged file; timeZone is the +// assignment's way to set the zone flows schedule and format in; mqtt names the broker nodes the +// module's MQTT step keeps pointed at the mesh's broker; topics is what nodered asks the broker for. +if (settings.timeZone) process.env.TZ = settings.timeZone; +delete settings.timeZone; +delete settings.endpoints; +delete settings.mqtt; +delete settings.topics; + +function same(a, b) { + const x = crypto.createHash("sha256").update(String(a)).digest(); + const y = crypto.createHash("sha256").update(String(b)).digest(); + return crypto.timingSafeEqual(x, y); +} + +// The admin secret is a password, or, accepted from an existing install, the bcrypt hash its +// settings held, so the password people already use keeps working. +function passwordMatches(given) { + if (/^\$2[aby]\$\d\d\$/.test(ADMIN_PASSWORD)) return require("bcryptjs").compare(String(given), ADMIN_PASSWORD); + return Promise.resolve(same(given, ADMIN_PASSWORD)); +} + +module.exports = Object.assign(settings, { + uiPort: 1880, + adminAuth: { + type: "credentials", + users: (username) => Promise.resolve(username === ADMIN.username ? ADMIN : null), + authenticate: (username, password) => + username === ADMIN.username + ? passwordMatches(password).then((ok) => (ok ? ADMIN : null)) + : Promise.resolve(null), + // The module's own tools call the admin API with this bearer token. + tokens: (token) => Promise.resolve(same(token, API_TOKEN) ? { username: "mesh", permissions: "*" } : null), + }, +}); diff --git a/modules/nodered/files/settings.json b/modules/nodered/files/settings.json new file mode 100644 index 00000000..93cf1572 --- /dev/null +++ b/modules/nodered/files/settings.json @@ -0,0 +1,14 @@ +{ + "flowFile": "flows.json", + "flowFilePretty": true, + "diagnostics": { "enabled": true, "ui": true }, + "runtimeState": { "enabled": false, "ui": false }, + "logging": { "console": { "level": "info", "metrics": false, "audit": false } }, + "exportGlobalContextKeys": false, + "externalModules": {}, + "editorTheme": { "projects": { "enabled": false } }, + "functionExternalModules": true, + "debugMaxLength": 1000, + "mqttReconnectTime": 15000, + "serialReconnectTime": 15000 +} diff --git a/modules/nodered/module.json b/modules/nodered/module.json index 3bd4a552..48c65dd0 100644 --- a/modules/nodered/module.json +++ b/modules/nodered/module.json @@ -36,6 +36,108 @@ } ] }, + "config": { + "flow-file": { + "type": "string", + "description": "Node-RED's flowFile setting: the file in the user directory the flows are kept in.", + "trusted": true, + "default": "flows.json" + }, + "flow-file-pretty": { + "type": "boolean", + "description": "Node-RED's flowFilePretty setting: whether the flows file is written indented.", + "trusted": true, + "default": true + }, + "diagnostics": { + "type": "object", + "description": "Node-RED's diagnostics setting: whether its diagnostics report is on, and shown in the editor.", + "trusted": true, + "default": { + "enabled": true, + "ui": true + } + }, + "runtime-state": { + "type": "object", + "description": "Node-RED's runtimeState setting: whether flows may be started and stopped from the editor.", + "trusted": true, + "default": { + "enabled": false, + "ui": false + } + }, + "logging": { + "type": "object", + "description": "Node-RED's logging setting: the console log's level, and whether metrics and audit events are logged.", + "trusted": true, + "default": { + "console": { + "level": "info", + "metrics": false, + "audit": false + } + } + }, + "export-global-context-keys": { + "type": "boolean", + "description": "Node-RED's exportGlobalContextKeys setting: whether a function node may list the global context's keys.", + "trusted": true, + "default": false + }, + "external-modules": { + "type": "object", + "description": "Node-RED's externalModules setting: which modules the editor may install, and from where.", + "trusted": true, + "default": {} + }, + "editor-theme": { + "type": "object", + "description": "Node-RED's editorTheme setting: the editor's look and features, projects among them.", + "trusted": true, + "default": { + "projects": { + "enabled": false + } + } + }, + "function-external-modules": { + "type": "boolean", + "description": "Node-RED's functionExternalModules setting: whether a function node may load modules of its own.", + "trusted": true, + "default": true + }, + "debug-max-length": { + "type": "number", + "description": "Node-RED's debugMaxLength setting: the most characters a debug message shows, a count.", + "trusted": true, + "default": 1000 + }, + "mqtt-reconnect-time": { + "type": "number", + "description": "Node-RED's mqttReconnectTime setting: how long an MQTT node waits before reconnecting, in milliseconds.", + "trusted": true, + "default": 15000 + }, + "serial-reconnect-time": { + "type": "number", + "description": "Node-RED's serialReconnectTime setting: how long a serial node waits before reconnecting, in milliseconds.", + "trusted": true, + "default": 15000 + }, + "mqtt": { + "type": "object", + "description": "The broker nodes the module's MQTT step keeps pointed at the mesh's broker, as {\"brokers\": [their ids]}; empty, none.", + "trusted": true, + "default": {} + }, + "time-zone": { + "type": "string", + "description": "The time zone the flows schedule and format in, an IANA name such as Europe/Brussels; unset, the container's UTC.", + "trusted": true, + "default": "" + } + }, "resources": [ { "id": "mesh-state", @@ -66,7 +168,7 @@ "path": "${dir:state}/settings.js", "mode": "0600", "owner": "1000:1000", - "content": "// Node-RED's settings, written by the mesh from the nodered module. What an assignment may change\n// is settings.json beside this file (merged key by key); the credentials are the mesh's secrets and\n// reach Node-RED only through this file. The flows' own credentials stay encrypted in the user\n// directory under the key Node-RED keeps there (.config.runtime.json), which is data, not this.\nconst fs = require(\"fs\");\nconst path = require(\"path\");\nconst crypto = require(\"crypto\");\n\nconst ADMIN_PASSWORD = \"${secret:admin}\";\nconst API_TOKEN = \"${secret:api-token}\";\nconst ADMIN = { username: \"admin\", permissions: \"*\" };\n\nconst settings = JSON.parse(fs.readFileSync(path.join(__dirname, \"settings.json\"), \"utf8\"));\n// The mesh's keys, not Node-RED's: endpoints lands in every merged file; timeZone is the\n// assignment's way to set the zone flows schedule and format in; mqtt names the broker nodes the\n// module's MQTT step keeps pointed at the mesh's broker; topics is what nodered asks the broker for.\nif (settings.timeZone) process.env.TZ = settings.timeZone;\ndelete settings.timeZone;\ndelete settings.endpoints;\ndelete settings.mqtt;\ndelete settings.topics;\n\nfunction same(a, b) {\n const x = crypto.createHash(\"sha256\").update(String(a)).digest();\n const y = crypto.createHash(\"sha256\").update(String(b)).digest();\n return crypto.timingSafeEqual(x, y);\n}\n\n// The admin secret is a password, or, accepted from an existing install, the bcrypt hash its\n// settings held, so the password people already use keeps working.\nfunction passwordMatches(given) {\n if (/^\\$2[aby]\\$\\d\\d\\$/.test(ADMIN_PASSWORD)) return require(\"bcryptjs\").compare(String(given), ADMIN_PASSWORD);\n return Promise.resolve(same(given, ADMIN_PASSWORD));\n}\n\nmodule.exports = Object.assign(settings, {\n uiPort: 1880,\n adminAuth: {\n type: \"credentials\",\n users: (username) => Promise.resolve(username === ADMIN.username ? ADMIN : null),\n authenticate: (username, password) =>\n username === ADMIN.username\n ? passwordMatches(password).then((ok) => (ok ? ADMIN : null))\n : Promise.resolve(null),\n // The module's own tools call the admin API with this bearer token.\n tokens: (token) => Promise.resolve(same(token, API_TOKEN) ? { username: \"mesh\", permissions: \"*\" } : null),\n },\n});\n" + "template": "files/settings.js" }, { "id": "settings", @@ -75,7 +177,8 @@ "mode": "0600", "owner": "1000:1000", "merge": "json", - "content": "{\n \"flowFile\": \"flows.json\",\n \"flowFilePretty\": true,\n \"diagnostics\": { \"enabled\": true, \"ui\": true },\n \"runtimeState\": { \"enabled\": false, \"ui\": false },\n \"logging\": { \"console\": { \"level\": \"info\", \"metrics\": false, \"audit\": false } },\n \"exportGlobalContextKeys\": false,\n \"externalModules\": {},\n \"editorTheme\": { \"projects\": { \"enabled\": false } },\n \"functionExternalModules\": true,\n \"debugMaxLength\": 1000,\n \"mqttReconnectTime\": 15000,\n \"serialReconnectTime\": 15000\n}\n" + "takes-config": [{"key": "flow-file", "as": "flowFile"}, {"key": "flow-file-pretty", "as": "flowFilePretty"}, "diagnostics", {"key": "runtime-state", "as": "runtimeState"}, "logging", {"key": "export-global-context-keys", "as": "exportGlobalContextKeys"}, {"key": "external-modules", "as": "externalModules"}, {"key": "editor-theme", "as": "editorTheme"}, {"key": "function-external-modules", "as": "functionExternalModules"}, {"key": "debug-max-length", "as": "debugMaxLength"}, {"key": "mqtt-reconnect-time", "as": "mqttReconnectTime"}, {"key": "serial-reconnect-time", "as": "serialReconnectTime"}, "mqtt", {"key": "time-zone", "as": "timeZone"}], + "template": "files/settings.json" }, { "id": "server", @@ -110,14 +213,14 @@ "type": "file", "path": "${dir:mesh-state}/config.json", "mode": "0600", - "content": "{\n \"token\": \"${secret:api-token}\"\n}\n" + "template": "files/config.json" }, { "id": "mqtt-env", "type": "file", "path": "${dir:state}/mqtt.env", "mode": "0600", - "content": "MESH_NODERED_URL=http://127.0.0.1:${port:1880}\nMESH_NODERED_CONFIG_FILE=${dir:mesh-state}/config.json\nMESH_PROVISIONS_DIR=${dir:state}\nMESH_WRITTEN_DIR=${dir:written}\n" + "template": "files/mqtt.env" }, { "id": "mqtt", @@ -179,6 +282,11 @@ "MESH_NODERED_URL": "http://127.0.0.1:${port:1880}", "MESH_NODERED_CONFIG_FILE": "${dir:mesh-state}/config.json" } + }, + { + "name": "files", + "kind": "archive", + "from": "files" } ] } diff --git a/modules/only-office/files/server.env b/modules/only-office/files/server.env new file mode 100644 index 00000000..19fab119 --- /dev/null +++ b/modules/only-office/files/server.env @@ -0,0 +1,5 @@ +JWT_ENABLED=true +JWT_SECRET=${secret:jwt} +JWT_HEADER=Authorization +JWT_IN_BODY=true +ALLOW_PRIVATE_IP_ADDRESS=true diff --git a/modules/only-office/module.json b/modules/only-office/module.json index 49f10efd..ac0eed75 100644 --- a/modules/only-office/module.json +++ b/modules/only-office/module.json @@ -87,7 +87,7 @@ "type": "file", "path": "${dir:state}/server.env", "mode": "0600", - "content": "JWT_ENABLED=true\nJWT_SECRET=${secret:jwt}\nJWT_HEADER=Authorization\nJWT_IN_BODY=true\nALLOW_PRIVATE_IP_ADDRESS=true\n" + "template": "files/server.env" }, { "id": "logs", @@ -152,5 +152,14 @@ ], "secrets-in-environment": "run-document-server.sh regenerates JWT_SECRET from the environment on every start and overwrites local.json; not convertible" } - ] + ], + "build": { + "artifacts": [ + { + "name": "files", + "kind": "archive", + "from": "files" + } + ] + } } diff --git a/modules/pacman/cmd/pacman-tools/manifest_test.go b/modules/pacman/cmd/pacman-tools/manifest_test.go index 4b461587..b55f6908 100644 --- a/modules/pacman/cmd/pacman-tools/manifest_test.go +++ b/modules/pacman/cmd/pacman-tools/manifest_test.go @@ -34,7 +34,7 @@ func TestItHoldsThePackageManagerSeatAndDeclaresItsPackage(t *testing.T) { func TestPacmanConfIsWholeTheUnionOfRepositoriesAndTheImprovedOptions(t *testing.T) { f := manifest(t).resource(t, "config") - content := f["content"].(string) + content := templateText(t, f) if f["path"] != "/etc/pacman.conf" || f["into"] != nil || !strings.HasPrefix(content, MeshHeader) { t.Fatalf("%v", f) } @@ -75,7 +75,7 @@ func TestPacmanConfIsWholeTheUnionOfRepositoriesAndTheImprovedOptions(t *testing } func TestReflectorWritesTheListPacmanReads(t *testing.T) { - content := manifest(t).resource(t, "mirrors")["content"].(string) + content := templateText(t, manifest(t).resource(t, "mirrors")) opts := map[string]string{} for _, l := range strings.Split(content, "\n") { if l == "" || strings.HasPrefix(l, "#") { diff --git a/modules/pacman/cmd/pacman-tools/templates_test.go b/modules/pacman/cmd/pacman-tools/templates_test.go new file mode 100644 index 00000000..7d0718b5 --- /dev/null +++ b/modules/pacman/cmd/pacman-tools/templates_test.go @@ -0,0 +1,64 @@ +package main + +import ( + "encoding/json" + "os" + "path/filepath" + "testing" +) + +// The module's directory, where its manifest and its files/ are. +const moduleDir = "../.." + +// templateText is the text a file resource places on the machine: its template's, in the module's files/ +// (novox/hq ADR 0349 §2). A resource that names no template, or a template that is not there, fails the test. +func templateText(t *testing.T, r map[string]any) string { + t.Helper() + name, _ := r["template"].(string) + if name == "" { + t.Fatalf("resource %v names no template", r["id"]) + } + raw, err := os.ReadFile(filepath.Join(moduleDir, filepath.FromSlash(name))) + if err != nil { + t.Fatalf("resource %v names %s: %v", r["id"], name, err) + } + return string(raw) +} + +// No file resource carries its text inline: each names its template in files/, and the build packs files/ +// into the one archive the controller reads the templates back from (novox/hq ADR 0349 §2). +func TestEveryFileTakesItsTextFromATemplate(t *testing.T) { + raw, err := os.ReadFile(filepath.Join(moduleDir, "module.json")) + if err != nil { + t.Fatal(err) + } + var m struct { + Resources []map[string]any `json:"resources"` + Build struct { + Artifacts []map[string]any `json:"artifacts"` + } `json:"build"` + } + if err := json.Unmarshal(raw, &m); err != nil { + t.Fatal(err) + } + for _, r := range m.Resources { + if _, inline := r["content"]; inline { + t.Errorf("resource %v keeps its text inline: keep it in files/ and name it as the template", r["id"]) + } + if r["type"] == "file" { + templateText(t, r) + } + } + packs := 0 + for _, a := range m.Build.Artifacts { + if a["kind"] == "archive" && a["from"] == "files" { + packs++ + if a["name"] != "files" { + t.Errorf("the templates archive is named %v, not files", a["name"]) + } + } + } + if packs != 1 { + t.Errorf("the build packs files/ into %d archives, not one", packs) + } +} diff --git a/modules/pacman/files/pacman.conf b/modules/pacman/files/pacman.conf new file mode 100644 index 00000000..0390e254 --- /dev/null +++ b/modules/pacman/files/pacman.conf @@ -0,0 +1,29 @@ +# The mesh's (module pacman, novox/hq to-be 42): the package manager's configuration. Written +# whole at every push: an edit here is overwritten, and the file a machine had before is kept once by +# the host. Owned whole because [options] cannot take a block by appending: anything added at the end +# of the file lands in the last repository's section. +# +# The repositories are the union of what the machines had enabled when the module was written +# (core, extra, multilib). The options are the distribution's defaults with four more: colour on a +# terminal, parallel downloads, package lists in columns, and downloads run as the unprivileged +# alpm user, which pacman 7 creates. + +[options] +HoldPkg = pacman glibc +Architecture = auto +CheckSpace +Color +VerbosePkgLists +ParallelDownloads = 5 +DownloadUser = alpm +SigLevel = Required DatabaseOptional +LocalFileSigLevel = Optional + +[core] +Include = /etc/pacman.d/mirrorlist + +[extra] +Include = /etc/pacman.d/mirrorlist + +[multilib] +Include = /etc/pacman.d/mirrorlist diff --git a/modules/pacman/files/reflector.conf b/modules/pacman/files/reflector.conf new file mode 100644 index 00000000..eb9e731b --- /dev/null +++ b/modules/pacman/files/reflector.conf @@ -0,0 +1,8 @@ +# The mesh's (module pacman, novox/hq to-be 42): how reflector refreshes the mirror list, weekly, +# through reflector.timer. Written whole at every push. Before the module, every machine's list was +# generated once and never again. +--save /etc/pacman.d/mirrorlist +--protocol https +--country Belgium,Netherlands,Luxembourg,Germany,France +--latest 20 +--sort rate diff --git a/modules/pacman/module.json b/modules/pacman/module.json index 98ef2440..746c8c5a 100644 --- a/modules/pacman/module.json +++ b/modules/pacman/module.json @@ -39,7 +39,7 @@ "type": "file", "path": "/etc/pacman.conf", "mode": "0644", - "content": "# The mesh's (module pacman, novox/hq to-be 42): the package manager's configuration. Written\n# whole at every push: an edit here is overwritten, and the file a machine had before is kept once by\n# the host. Owned whole because [options] cannot take a block by appending: anything added at the end\n# of the file lands in the last repository's section.\n#\n# The repositories are the union of what the machines had enabled when the module was written\n# (core, extra, multilib). The options are the distribution's defaults with four more: colour on a\n# terminal, parallel downloads, package lists in columns, and downloads run as the unprivileged\n# alpm user, which pacman 7 creates.\n\n[options]\nHoldPkg = pacman glibc\nArchitecture = auto\nCheckSpace\nColor\nVerbosePkgLists\nParallelDownloads = 5\nDownloadUser = alpm\nSigLevel = Required DatabaseOptional\nLocalFileSigLevel = Optional\n\n[core]\nInclude = /etc/pacman.d/mirrorlist\n\n[extra]\nInclude = /etc/pacman.d/mirrorlist\n\n[multilib]\nInclude = /etc/pacman.d/mirrorlist\n" + "template": "files/pacman.conf" }, { "id": "contrib", @@ -56,7 +56,7 @@ "type": "file", "path": "/etc/xdg/reflector/reflector.conf", "mode": "0644", - "content": "# The mesh's (module pacman, novox/hq to-be 42): how reflector refreshes the mirror list, weekly,\n# through reflector.timer. Written whole at every push. Before the module, every machine's list was\n# generated once and never again.\n--save /etc/pacman.d/mirrorlist\n--protocol https\n--country Belgium,Netherlands,Luxembourg,Germany,France\n--latest 20\n--sort rate\n" + "template": "files/reflector.conf" }, { "id": "mirror-refresh", @@ -85,6 +85,11 @@ "loads": [ "pacman-tools" ] + }, + { + "name": "files", + "kind": "archive", + "from": "files" } ] } diff --git a/modules/picom/cmd/picom-tools/manifest_helpers_test.go b/modules/picom/cmd/picom-tools/manifest_helpers_test.go index a098c63c..a652d238 100644 --- a/modules/picom/cmd/picom-tools/manifest_helpers_test.go +++ b/modules/picom/cmd/picom-tools/manifest_helpers_test.go @@ -12,27 +12,35 @@ import ( // every desktop module that carries it. type manifest struct { - Module string `json:"module"` - Version string `json:"version"` - Capabilities []string `json:"capabilities"` - Requires []string `json:"requires"` - Claims []claim `json:"claims"` - Seats []any `json:"seats"` - Tools []string `json:"tools"` - Settings map[string]setting `json:"settings"` - Environment *environment `json:"environment"` - Shell []shellCode `json:"shell"` - Resources []map[string]any `json:"resources"` + Module string `json:"module"` + Version string `json:"version"` + Capabilities []string `json:"capabilities"` + Requires []string `json:"requires"` + Claims []claim `json:"claims"` + Seats []any `json:"seats"` + Tools []string `json:"tools"` + Config map[string]config `json:"config"` + Environment *environment `json:"environment"` + Shell []shellCode `json:"shell"` + Resources []map[string]any `json:"resources"` Build struct { Artifacts []map[string]any `json:"artifacts"` } `json:"build"` } -// setting is one key's default, as the definition gives it (novox/hq ADR 0262). -type setting struct { - Kind string `json:"kind"` - Default any `json:"default"` - Why string `json:"why"` +// config is one config value as the definition declares it (novox/hq ADR 0348 §4). +type config struct { + Type string `json:"type"` + Description string `json:"description"` + Enum []any `json:"enum"` + Minimum *float64 `json:"minimum"` + Maximum *float64 `json:"maximum"` + Pattern string `json:"pattern"` + Trusted bool `json:"trusted"` + Predefined string `json:"predefined"` + Default any `json:"default"` + Required bool `json:"required"` + Runtime bool `json:"runtime"` } type claim struct { @@ -91,26 +99,66 @@ func (m manifest) packages() (present, absent []string) { return present, absent } -// sameAsSource checks that a file resource's content is byte for byte the module's source file, so -// the readable file in the repository is what the machine gets. -func (m manifest) sameAsSource(t *testing.T, id, source string) { +// fromTemplate checks that a file resource takes its text from its template file in the module's +// repository (novox/hq ADR 0349 §2): it names the template, carries no text of its own beside it, and the +// template is there. The readable file in the repository is the one place the machine's text is written. +func (m manifest) fromTemplate(t *testing.T, id, template string) { t.Helper() - want, err := os.ReadFile(filepath.Join("..", "..", source)) - if err != nil { - t.Fatal(err) - } r := m.resource(t, id) if r["type"] != "file" { t.Fatalf("%s is a %v, not a file", id, r["type"]) } - if got, _ := r["content"].(string); got != string(want) { - t.Fatalf("resource %s's content is not %s: edit the source and copy it into module.json", id, source) + if r["template"] != template { + t.Fatalf("resource %s does not take its text from %s: template %v", id, template, r["template"]) + } + if _, inline := r["content"]; inline { + t.Fatalf("resource %s keeps its text inline beside its template: a file's text is in one place", id) + } + if _, err := os.Stat(filepath.Join("..", "..", template)); err != nil { + t.Fatalf("resource %s names %s: %v", id, template, err) } if r["owner"] != "${machine:account}" && !strings.HasPrefix(r["path"].(string), "/etc/") { t.Fatalf("%s under the home is the account's", id) } } +// everyFileIsATemplate checks that no file resource carries its text inline, and that the build packs +// files/ into the one archive the controller reads the templates back from (novox/hq ADR 0349 §2). +func (m manifest) everyFileIsATemplate(t *testing.T) { + t.Helper() + for _, r := range m.Resources { + if _, inline := r["content"]; inline { + t.Errorf("resource %v keeps its text inline: keep it in files/ and name it as the template", r["id"]) + } + } + packs := 0 + for _, a := range m.Build.Artifacts { + if a["kind"] == "archive" && a["from"] == "files" { + packs++ + if a["name"] != "files" { + t.Errorf("the templates archive is named %v, not files", a["name"]) + } + } + } + if packs != 1 { + t.Errorf("the build packs files/ into %d archives, not one", packs) + } +} + +// text is the text a file resource places on the machine: its template file's. +func (m manifest) text(t *testing.T, id string) string { + t.Helper() + name, _ := m.resource(t, id)["template"].(string) + if name == "" { + t.Fatalf("resource %s names no template", id) + } + raw, err := os.ReadFile(filepath.Join("..", "..", name)) + if err != nil { + t.Fatal(err) + } + return string(raw) +} + // checkTheToolsAgree checks that the manifest lists the module's own tools exactly, that the bundle // serves each seat verb the claims promise as ., and that the Go bundle is declared. func checkTheToolsAgree(t *testing.T, m manifest) { diff --git a/modules/picom/cmd/picom-tools/manifest_test.go b/modules/picom/cmd/picom-tools/manifest_test.go index b82d8ccc..5c6a397d 100644 --- a/modules/picom/cmd/picom-tools/manifest_test.go +++ b/modules/picom/cmd/picom-tools/manifest_test.go @@ -28,7 +28,7 @@ func TestItClaimsTheCompositorSeatAndRequiresTheXDisplay(t *testing.T) { func TestItOwnsItsConfigurationAsWrittenInTheModule(t *testing.T) { m := readManifest(t) - m.sameAsSource(t, "configuration", "files/picom.conf") + m.fromTemplate(t, "configuration", "files/picom.conf") if p := m.resource(t, "configuration")["path"]; p != "${machine:account-home}/.config/picom/picom.conf" { t.Fatalf("path: %v", p) } @@ -62,16 +62,19 @@ func TestTheToolsAgreeWithTheManifest(t *testing.T) { // The backend and damage tracking are each machine's (novox/hq research 037: the backend depends on the // machine's graphics driver; issue 345: a window left unpainted under glx is the first suspect). Both -// are settings whose defaults are what every machine ran before, so the file filled with them is today's. -func TestTheBackendAndDamageTrackingAreSettingsWithTodaysDefaults(t *testing.T) { +// are config values whose defaults are what every machine ran before, so the file filled with them is today's +// (novox/hq ADR 0348 §4): the backend a string of picom's backends, damage tracking a boolean, both trusted +// as the file is. +func TestTheBackendAndDamageTrackingAreConfigValuesWithTodaysDefaults(t *testing.T) { m := readManifest(t) - b, d := m.Settings["backend"], m.Settings["use-damage"] - if b.Kind != "preference" || b.Default != "glx" || b.Why == "" || d.Kind != "preference" || d.Default != true || d.Why == "" { - t.Fatalf("settings: %+v", m.Settings) + b, d := m.Config["backend"], m.Config["use-damage"] + if b.Type != "string" || b.Default != "glx" || b.Description == "" || !b.Trusted || len(b.Enum) == 0 || + d.Type != "boolean" || d.Default != true || d.Description == "" || !d.Trusted { + t.Fatalf("config: %+v", m.Config) } r := m.resource(t, "configuration") - c := r["content"].(string) - for _, want := range []string{`backend = "${setting:backend}";`, "use-damage = ${setting:use-damage};"} { + c := m.text(t, "configuration") + for _, want := range []string{`backend = "${config:backend}";`, "use-damage = ${config:use-damage};"} { if !strings.Contains(c, want) { t.Errorf("the file asks for %q", want) } @@ -88,7 +91,12 @@ func TestTheBackendAndDamageTrackingAreSettingsWithTodaysDefaults(t *testing.T) } } -// withDefaults fills the file's settings with their defaults, as a machine without its own values gets it. +// withDefaults fills the file's config values with their defaults, as a machine without its own values gets it. func withDefaults(c string) string { - return strings.NewReplacer("${setting:backend}", "glx", "${setting:use-damage}", "true").Replace(c) + return strings.NewReplacer("${config:backend}", "glx", "${config:use-damage}", "true").Replace(c) +} + +// No file resource carries its text inline (novox/hq ADR 0349 §2). +func TestEveryFileTakesItsTextFromATemplate(t *testing.T) { + readManifest(t).everyFileIsATemplate(t) } diff --git a/modules/picom/files/picom.conf b/modules/picom/files/picom.conf index 1c49876f..c6b85551 100644 --- a/modules/picom/files/picom.conf +++ b/modules/picom/files/picom.conf @@ -7,8 +7,8 @@ # machine ran before: glx, and picom's own default of repainting only what changed. xrender, or # use-damage false (repaint the whole screen each frame), is what to try on a machine where a window is # left unpainted or stale. -backend = "${setting:backend}"; -use-damage = ${setting:use-damage}; +backend = "${config:backend}"; +use-damage = ${config:use-damage}; vsync = true; fading = true; diff --git a/modules/picom/module.json b/modules/picom/module.json index 99d95644..2f747784 100644 --- a/modules/picom/module.json +++ b/modules/picom/module.json @@ -13,18 +13,26 @@ "scope": "node" } ], - "settings": { - "backend": { - "kind": "preference", - "default": "glx", - "why": "glx draws with the graphics card and has served every machine so far; xrender draws on the CPU and is what to try where glx leaves a window unpainted or stale on the machine's driver (novox/hq research 037, issue 345)" + "config": { + "backend": { + "type": "string", + "description": "How picom draws: glx with the graphics card, which has served every machine so far, or xrender on the CPU where glx leaves a window unpainted or stale on the machine's driver (novox/hq research 037, issue 345); xr_glx_hybrid and egl are picom's other backends.", + "enum": [ + "glx", + "xrender", + "xr_glx_hybrid", + "egl" + ], + "trusted": true, + "default": "glx" + }, + "use-damage": { + "type": "boolean", + "description": "Whether picom repaints only what changed (true, picom's own default) or the whole screen each frame (false), which costs work and cures a window left stale or unpainted on some drivers (novox/hq issue 345).", + "trusted": true, + "default": true + } }, - "use-damage": { - "kind": "preference", - "default": true, - "why": "true repaints only what changed, picom's own default; false repaints the whole screen each frame, which costs work and cures a window left stale or unpainted on some drivers (novox/hq issue 345)" - } - }, "tools": [ "picom_restart", "picom_rules", @@ -55,7 +63,7 @@ "path": "${machine:account-home}/.config/picom/picom.conf", "owner": "${machine:account}", "mode": "0644", - "content": "# picom, the X compositor (module picom, novox/hq ADR 0208). Owned by the mesh: this file is\n# replaced at every push. Adopted from the two workstations' file of 2026-10-04 and moved to\n# picom's window rules, which supersede opacity-rule, inactive-opacity and inactive-dim.\n\n# The backend and damage tracking are each machine's settings (backend, use-damage; novox/hq research\n# 037, issue 345): the right ones depend on the machine's graphics driver. The defaults are what every\n# machine ran before: glx, and picom's own default of repainting only what changed. xrender, or\n# use-damage false (repaint the whole screen each frame), is what to try on a machine where a window is\n# left unpainted or stale.\nbackend = \"${setting:backend}\";\nuse-damage = ${setting:use-damage};\nvsync = true;\n\nfading = true;\nfade-in-step = 0.05;\nfade-out-step = 0.05;\n\n# Tiled windows hide their shadows and corners, so neither is drawn.\nshadow = false;\ncorner-radius = 0;\nblur-method = \"none\";\n\n# \"Focused\" is i3's _NET_ACTIVE_WINDOW, not X focus events: under i3 those do not reliably reach\n# the toplevel picom tracks, and a terminal then never lifts to its focused opacity.\nuse-ewmh-active-win = true;\n\n# Window rules are applied in order, and a later match wins. Only terminals are translucent:\n# nothing else on screen (browsers, video, games) is touched. A terminal is matched by its class;\n# the node's terminal emulator today is xterm (class XTerm). A window's own opacity property is\n# used wherever no rule sets one, which is what the window-opacity tool sets.\nrules = (\n { match = \"window_type = 'tooltip'\"; fade = false; opacity = 0.95; },\n { match = \"window_type = 'dock' || window_type = 'desktop'\"; fade = false; },\n { match = \"class_g = 'XTerm' && focused\"; opacity = 0.95; },\n { match = \"class_g = 'XTerm' && !focused\"; opacity = 0.75; },\n # A full-screen window is opaque, a terminal included: a video or a game is never see-through.\n { match = \"fullscreen\"; opacity = 1; }\n);\n" + "template": "files/picom.conf" } ], "build": { @@ -70,6 +78,11 @@ "loads": [ "picom-tools" ] + }, + { + "name": "files", + "kind": "archive", + "from": "files" } ] } diff --git a/modules/polychromatic/cmd/polychromatic-tools/manifest_test.go b/modules/polychromatic/cmd/polychromatic-tools/manifest_test.go index 5baaba42..aef16b5b 100644 --- a/modules/polychromatic/cmd/polychromatic-tools/manifest_test.go +++ b/modules/polychromatic/cmd/polychromatic-tools/manifest_test.go @@ -25,11 +25,13 @@ type manifest struct { Claims []any `json:"claims"` Seats []any `json:"seats"` Shell []any `json:"shell"` - Contributions []struct { - Seat string `json:"seat"` - Kind string `json:"kind"` - Content string `json:"content"` - } `json:"contributions"` + Moved bool `json:"moved"` + Contributions []any `json:"contributions"` + Items []struct { + Collection string `json:"collection"` + Name string `json:"name"` + Template string `json:"template"` + } `json:"items"` Environment any `json:"environment"` Build struct { Artifacts []map[string]any `json:"artifacts"` @@ -69,7 +71,7 @@ func TestTheToolsAgreeWithTheManifest(t *testing.T) { for name := range served { t.Errorf("the bundle serves %s, which module.json does not list", name) } - if len(m.Build.Artifacts) != 1 { + if len(m.Build.Artifacts) != 2 { // the tools bundle and the archive of its template t.Fatalf("%v", m.Build.Artifacts) } b := m.Build.Artifacts[0] @@ -93,11 +95,8 @@ func TestItDeclaresNoPackageAndContributesTheTraysOneStart(t *testing.T) { if m.Resources != nil || m.Capabilities != nil || m.Claims != nil || m.Seats != nil || m.Environment != nil || m.Shell != nil { t.Fatal("no package (outside the official repositories), no seat, no environment, no xinitrc start") } - if len(m.Contributions) != 1 || m.Contributions[0].Seat != "node-display-session" || m.Contributions[0].Kind != "config" { - t.Fatalf("%+v", m.Contributions) - } var code []string - for _, l := range strings.Split(m.Contributions[0].Content, "\n") { + for _, l := range strings.Split(displayLines(t, m), "\n") { if l = strings.TrimSpace(l); l != "" && !strings.HasPrefix(l, "#") { code = append(code, l) } @@ -121,8 +120,23 @@ func TestTheContributionParsesInI3(t *testing.T) { } m, _ := readManifest(t) path := filepath.Join(t.TempDir(), "config") - os.WriteFile(path, []byte("set $mod Mod4\n"+m.Contributions[0].Content), 0o644) + os.WriteFile(path, []byte("set $mod Mod4\n"+displayLines(t, m)), 0o644) if out, err := exec.Command(i3, "-C", "-c", path).CombinedOutput(); err != nil || strings.Contains(string(out), "ERROR") { t.Fatalf("i3 -C: %v %s", err, out) } } + +// displayLines is the module's lines for the window manager: the text of the template of its one item of the +// collection contribution:node-display-session:config (design 57 L3, issue 848). +func displayLines(t *testing.T, m manifest) string { + t.Helper() + if !m.Moved || len(m.Contributions) != 0 || len(m.Items) != 1 || + m.Items[0].Collection != "contribution:node-display-session:config" { + t.Fatalf("moved %v, contributions %v, items %+v", m.Moved, m.Contributions, m.Items) + } + text, err := os.ReadFile(filepath.Join("..", "..", m.Items[0].Template)) + if err != nil { + t.Fatal(err) + } + return string(text) +} diff --git a/modules/polychromatic/files/display-session b/modules/polychromatic/files/display-session new file mode 100644 index 00000000..bbdbd6f2 --- /dev/null +++ b/modules/polychromatic/files/display-session @@ -0,0 +1,6 @@ +# The Razer peripherals' tray (module polychromatic, novox/hq ADR 0208, ADR 0212). Owned by the mesh: +# replaced at every push. This line is the tray's one start, at the session's start (exec, not +# exec_always, so a reload starts nothing). The package's own login helper starts it too while the +# application's setting "Start the tray applet when I log on" is ticked; polychromatic_check names +# that as a second start. +exec --no-startup-id polychromatic-tray-applet diff --git a/modules/polychromatic/module.json b/modules/polychromatic/module.json index eacb0bfe..4d6ba4b3 100644 --- a/modules/polychromatic/module.json +++ b/modules/polychromatic/module.json @@ -1,6 +1,7 @@ { "module": "polychromatic", "version": "1", + "moved": true, "requires": [ "x11-display" ], @@ -9,11 +10,11 @@ "polychromatic_restart", "polychromatic_check" ], - "contributions": [ + "items": [ { - "seat": "node-display-session", - "kind": "config", - "content": "# The Razer peripherals' tray (module polychromatic, novox/hq ADR 0208, ADR 0212). Owned by the mesh:\n# replaced at every push. This line is the tray's one start, at the session's start (exec, not\n# exec_always, so a reload starts nothing). The package's own login helper starts it too while the\n# application's setting \"Start the tray applet when I log on\" is ticked; polychromatic_check names\n# that as a second start.\nexec --no-startup-id polychromatic-tray-applet\n" + "collection": "contribution:node-display-session:config", + "name": "display-session", + "template": "files/display-session" } ], "build": { @@ -28,6 +29,11 @@ "loads": [ "polychromatic-tools" ] + }, + { + "name": "files", + "kind": "archive", + "from": "files" } ] } diff --git a/modules/postgres/cmd/postgres-provider/main.go b/modules/postgres/cmd/postgres-provider/main.go index af26205c..4caf857f 100644 --- a/modules/postgres/cmd/postgres-provider/main.go +++ b/modules/postgres/cmd/postgres-provider/main.go @@ -33,7 +33,32 @@ func main() { return } var h *Harness - if receives := os.Getenv("MESH_RECEIVES"); receives == "" { + if consumers := handedConsumersFile(os.Getenv, os.Stat); consumers != "" { + // Handed to the provisioner (novox/hq ADR 0373): the provider helper answers the consumers its + // machine's client fills the file with; the grant directory is not read. Retirement is the + // harness's, never run, so its guards and its tools stand (issue 780). + h = &Harness{ + Resource: ProvisionName, + Adapter: provisioner{pg: pg, announce: announce}, + Log: func(format string, args ...any) { fmt.Fprintf(os.Stderr, format+"\n", args...) }, + Announce: func(event string, body map[string]any) { + if err := stdio.Emit(event, body); err != nil { + say("emit %s failed: %v", event, err) + } + }, + Node: os.Getenv("MESH_NODE"), + } + run, p, bus, err := moved(consumers, h) + if err != nil { + say("%v", err) + } else { + go func() { + if err := run.Run(context.Background(), p, bus); err != nil { + say("the provider helper stopped: %v", err) + } + }() + } + } else if receives := os.Getenv("MESH_RECEIVES"); receives == "" { say("MESH_RECEIVES is not set — the provisioner cannot run without it") } else { h = &Harness{ @@ -52,6 +77,15 @@ func main() { } go h.Run(context.Background()) } + // Started on the grant path while the module is to be handed, it restarts itself when its client first fills + // the consumers' file, so the first fill moves it onto the helper without a hand restart (issue 780): `restart-on` + // names a service or a container, and the only one here is the database. + if at := os.Getenv("MESH_CONSUMERS_FILE"); at != "" && handedConsumersFile(os.Getenv, os.Stat) == "" { + go exitWhenFilled(at, os.Stat, 5*time.Second, func() { + say("%s is filled: restarting to answer from it", at) + os.Exit(0) + }) + } go audit() // The store's reader is narrowed to what modules declare readable, at once and every minute, whether // or not the provisioner runs: without a contributions file it is given nothing (novox/hq issue 337). @@ -64,6 +98,31 @@ func main() { } } +// handedConsumersFile is the consumers' file when postgres is handed to the provisioner: the file MESH_CONSUMERS_FILE +// names, which the controller places only where it hands the module. Empty, where it is not set or the file is not +// there, and postgres then reads the grant directory exactly as it always did (issue 780). +func handedConsumersFile(getenv func(string) string, stat func(string) (os.FileInfo, error)) string { + at := getenv("MESH_CONSUMERS_FILE") + if at == "" { + return "" + } + if _, err := stat(at); err != nil { + return "" + } + return at +} + +// exitWhenFilled calls filled once the file is there, asking every so often. +func exitWhenFilled(at string, stat func(string) (os.FileInfo, error), every time.Duration, filled func()) { + for { + if _, err := stat(at); err == nil { + filled() + return + } + time.Sleep(every) + } +} + // announce emits a lifecycle event without letting a broker hiccup fail the provisioning itself. func announce(event string, body map[string]string) { if err := stdio.Emit(event, body); err != nil { diff --git a/modules/postgres/cmd/postgres-provider/moved.go b/modules/postgres/cmd/postgres-provider/moved.go new file mode 100644 index 00000000..4f8e8642 --- /dev/null +++ b/modules/postgres/cmd/postgres-provider/moved.go @@ -0,0 +1,253 @@ +package main + +// postgres handed to the provisioner (novox/hq ADR 0373, ADR 0349 clause 12, issue 780): where its consumers file +// is set, postgres runs on the SDK's provider helper instead of the harness reading the controller's grant files. +// +// The helper learns who binds postgres from the plans, composes each consumer's entry and reads its credential +// through ConsumersFile, the file this machine's provisioner client fills. It leaves the rest to the two hooks +// here: Make brings the consumer's database and role into being exactly as the harness's adapter does, and Gone +// retires what was kept for a consumer that no longer binds, the reversible way (retire_pg.go): the login is +// locked and marked to delete, nothing is dropped. + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "sync" + "time" + + "git.novox.be/novox/mesh-sdk/go/provider" +) + +// ProvisionName is what postgres provides, as its manifest says. +const ProvisionName = "postgres-database" + +// servedPort is the port its manifest serves (`serves.postgres-database.port`). +const servedPort = 5432 + +// movedRun is postgres on the provider helper, with the harness's own memory and retirement behind it: what is held, +// what is counted unasked, what waits for a person (retirement.go). The harness is never Run here; the helper's +// Make and Gone and the loop below drive the same state, so the guards are the harness's, not a copy. +type movedRun struct { + h *Harness + + mu sync.Mutex // guards assignments and loaded + assignments map[string][]byte + loaded bool + verified map[string]time.Time +} + +func newMovedRun(h *Harness) *movedRun { + h.mu.Lock() + h.init() + h.mu.Unlock() + return &movedRun{h: h, assignments: map[string][]byte{}, verified: map[string]time.Time{}} +} + +// Make brings one consumer's database and role into being under its login and the vault's credential, as the +// harness does: at once when it is new or changed, and otherwise only when the server no longer holds it, which is +// asked every VerifyEvery. +func (m *movedRun) Make(ctx context.Context, c provider.Consumer) error { + h, e := m.h, c.Entry + if e.User == "" { + return fmt.Errorf("%s has no login to make", e.Key()) + } + if len(c.Credential) == 0 { + return fmt.Errorf("%s has no credential to make its login with", e.Key()) + } + h.mu.Lock() + defer h.mu.Unlock() + as, password := e.User, string(c.Credential) + p := Provision{As: as, Password: password, Values: orEmpty(e.Contributes), Consumer: e.Consumer} + hash := hashOf(as, password, p.Values, nil) + if was, ok := h.applied[as]; ok && was.hash == hash { + if h.Now().Sub(m.verified[as]) < h.VerifyEvery { + return nil + } + m.verified[as] = h.Now() + hctx, cancel := context.WithTimeout(ctx, h.HoldsTimeout) + held, err := h.Adapter.Holds(hctx, p) + cancel() + if err != nil { + text := scrub(err, password) + h.failed(as, e.Consumer, h.classOf(err, text), text) + return errors.New(text) + } + if held { + h.succeeded(as) + return nil + } + h.say("%s: the backend no longer holds it; applying again", as) + } + if err := h.Adapter.Create(ctx, p); err != nil { + text := scrub(err, password) + h.failed(as, e.Consumer, h.classOf(err, text), text) + return errors.New(text) + } + h.succeeded(as) + h.applied[as] = appliedEntry{hash: hash, node: e.Consumer} + m.verified[as] = h.Now() + h.reenabled(as, e.Consumer) + return nil +} + +// Gone is the helper asking whether it may forget a consumer that no longer binds. **It never retires**: retiring is +// the harness's rule, the same unasked set over StablePasses passes and StableFor, at most RetireAtOnce at a time +// (retirement.go, ADR 0230), run by retire below. So a consumer missing from one read of the plans loses nothing: +// the helper is told no, and asks again at its next pass, until the login is retired, or asked for again. +func (m *movedRun) Gone(_ context.Context, e provider.Entry) error { + h := m.h + as := e.User + if as == "" { + return nil // refused: nothing was made for it + } + // The plans as they are now, not as the last retirement pass read them: with none read, nothing is let go. + want, read := m.asked(h.Node) + asked := want[as] + h.mu.Lock() + defer h.mu.Unlock() + _, held := h.applied[as] + _, retired := h.r.retired[as] + switch { + case !read: + case asked: + return nil // another holder of the same login still binds: nothing to retire + case retired, !held && h.r.seeded: + return nil + } + return fmt.Errorf("%s is kept until its retirement is stable (%d passes and %s) and within the bound, or a "+ + "person approves it", as, h.StablePasses, h.StableFor) +} + +// plan is as much of an assignment as is read to learn who binds postgres. +type plan struct { + Node string `json:"node"` + Module string `json:"module"` + Provisioning *struct { + Provisions []struct { + Provision string `json:"provision"` + Provider string `json:"provider"` + ProviderModule string `json:"provider-module"` + // As is who the consumer is at its provider's end, its slug first (issue 788). + As string `json:"as"` + } `json:"provisions"` + } `json:"provisioning"` +} + +// asked is the logins the plans bind postgres for, and whether they could all be read: a plan that is not the +// controller's, or no read yet, is not an answer, so nothing is counted unasked from it (novox/hq issue 241). +func (m *movedRun) asked(node string) (map[string]bool, bool) { + m.mu.Lock() + defer m.mu.Unlock() + if !m.loaded || len(m.assignments) == 0 { + return nil, false + } + want := map[string]bool{} + for key, raw := range m.assignments { + var a plan + if err := json.Unmarshal(raw, &a); err != nil || a.Node+"."+a.Module != key { + return nil, false + } + if a.Provisioning == nil { + continue + } + for _, b := range a.Provisioning.Provisions { + if b.Provider == node && b.ProviderModule == "postgres" && b.Provision == ProvisionName { + as := b.As + if as == "" { + as = provider.ConsumerIdentity(a.Node, a.Module) // a plan that names none + } + want[as] = true + } + } + } + return want, true +} + +// retire is one pass of the harness's retirement over what the plans bind now. +func (m *movedRun) retire(ctx context.Context) { + h := m.h + want, ok := m.asked(h.Node) + h.mu.Lock() + defer h.mu.Unlock() + if !ok { + // Not a result: the passes that must agree before anything is retired start again. + h.r.key, h.r.count = "", 0 + return + } + h.seed(ctx, want) + h.retireUnasked(ctx, want) + h.r.lastWant = want + for as := range h.failing { + if !want[as] { + delete(h.failing, as) + } + } + for as := range h.trouble { + if _, held := h.applied[as]; !want[as] && !held { + h.recovered(as, "no longer asked for") + } + } +} + +// watch keeps the plans as the bus holds them, from the first read on. +func (m *movedRun) watch(ctx context.Context, bus provider.Bus) error { + err := bus.Watch(ctx, provider.AssignmentsBucket, func(key string, value []byte) { + m.mu.Lock() + defer m.mu.Unlock() + if value == nil { + delete(m.assignments, key) + } else { + m.assignments[key] = append([]byte(nil), value...) + } + }) + if err == nil { + m.mu.Lock() + m.loaded = true + m.mu.Unlock() + } + return err +} + +// Run answers the consumers through the helper and retires through the harness's rules, until ctx ends. +func (m *movedRun) Run(ctx context.Context, p *provider.Provider, bus provider.Bus) error { + if err := m.watch(ctx, bus); err != nil { + return err + } + go func() { + for { + m.retire(ctx) + select { + case <-ctx.Done(): + return + case <-time.After(m.h.Every): + } + } + }() + return p.Run(ctx) +} + +// moved is postgres handed to the provisioner: the helper answering its consumers, and the harness (never run) whose +// retirement and tools it shares. consumersFile is where its machine's client fills its consumers. +func moved(consumersFile string, h *Harness) (*movedRun, *provider.Provider, provider.Bus, error) { + if consumersFile == "" { + return nil, nil, nil, errors.New("no consumers' file") + } + run := newMovedRun(h) + bus := provider.StateBus("postgres") + return run, provider.New(provider.Options{ + Node: h.Node, + Module: "postgres", + Offers: map[string]provider.Offer{ProvisionName: { + Credential: provider.PerConsumer, + Serves: map[string]any{"port": servedPort}, + }}, + Bus: bus, + Vault: provider.ConsumersFile{Paths: []string{consumersFile}}, + Make: run.Make, + Gone: run.Gone, + Every: h.Every, + Logf: func(f string, a ...any) { h.say(f, a...) }, + }), bus, nil +} diff --git a/modules/postgres/cmd/postgres-provider/moved_test.go b/modules/postgres/cmd/postgres-provider/moved_test.go new file mode 100644 index 00000000..db51aed4 --- /dev/null +++ b/modules/postgres/cmd/postgres-provider/moved_test.go @@ -0,0 +1,358 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + "os" + "path/filepath" + "reflect" + "sync" + "testing" + "time" + + "git.novox.be/novox/mesh-sdk/go/provider" +) + +// bucketBus is the helper's bus over in-memory buckets. +type bucketBus struct { + mu sync.Mutex + buckets map[string]map[string][]byte +} + +func (b *bucketBus) Watch(_ context.Context, bucket string, handle func(string, []byte)) error { + b.mu.Lock() + defer b.mu.Unlock() + for k, v := range b.buckets[bucket] { + handle(k, v) + } + return nil +} + +func (b *bucketBus) Put(_ context.Context, bucket, key string, value []byte) error { + b.mu.Lock() + defer b.mu.Unlock() + if b.buckets[bucket] == nil { + b.buckets[bucket] = map[string][]byte{} + } + b.buckets[bucket][key] = append([]byte(nil), value...) + return nil +} + +func (b *bucketBus) Delete(_ context.Context, bucket, key string) error { + b.mu.Lock() + defer b.mu.Unlock() + delete(b.buckets[bucket], key) + return nil +} + +// bind is one consumer's assignment, whose plan names this machine's postgres as the provider and carries what the +// consumer contributes (issue 776). +func (b *bucketBus) bind(node, module string, contributes map[string]any) { + plan, _ := json.Marshal(map[string]any{"node": node, "module": module, "provisioning": map[string]any{ + "provisions": []any{map[string]any{"provision": ProvisionName, "provider": "novox", "provider-module": "postgres", + "contributes": contributes}}}}) + if err := b.Put(context.Background(), provider.AssignmentsBucket, node+"."+module, plan); err != nil { + panic(err) + } +} + +// consumersFile writes the file the machine's client fills: one credential per consumer. +func consumersFile(t *testing.T, bound map[[2]string]string) string { + var list []provider.ConsumerCredential + for k, password := range bound { + list = append(list, provider.ConsumerCredential{Consumer: k[0], Module: k[1], + Credential: provider.PairName("novox", ProvisionName, k[0], k[1], ""), Value: password}) + } + raw, _ := json.Marshal(list) + at := filepath.Join(t.TempDir(), "consumers.json") + if err := os.WriteFile(at, raw, 0o600); err != nil { + t.Fatal(err) + } + return at +} + +// movedWorld is postgres on the helper over an in-memory bus, with a recorder for a server and a clock to move. +type movedWorld struct { + t *testing.T + a *recorder + bus *bucketBus + h *Harness + run *movedRun + p *provider.Provider + now time.Time + file string +} + +func newMovedWorld(t *testing.T, bound map[[2]string]string) *movedWorld { + w := &movedWorld{t: t, a: &recorder{}, bus: &bucketBus{buckets: map[string]map[string][]byte{}}, + now: time.Date(2026, 10, 15, 12, 0, 0, 0, time.UTC)} + w.file = consumersFile(t, bound) + w.h = &Harness{Resource: ProvisionName, Adapter: w.a, Node: "novox", Now: func() time.Time { return w.now }, + Log: func(string, ...any) {}} + var bus provider.Bus + w.run, w.p, bus, _ = moved(w.file, w.h) + _ = bus + if err := w.run.watch(ctx, w.bus); err != nil { + t.Fatal(err) + } + return w +} + +// pass is one helper pass and one retirement pass, five seconds after the last. +func (w *movedWorld) pass() []provider.Outcome { + w.now = w.now.Add(5 * time.Second) + // the plans as the bus holds them now, as the watch would hand them + w.run.assignments = map[string][]byte{} + for k, v := range w.bus.buckets[provider.AssignmentsBucket] { + w.run.assignments[k] = v + } + // the helper reads its buckets from the in-memory bus + w.p = provider.New(provider.Options{Node: "novox", Module: "postgres", Bus: w.bus, + Vault: provider.ConsumersFile{Paths: []string{w.file}}, + Offers: map[string]provider.Offer{ProvisionName: {Credential: provider.PerConsumer, + Serves: map[string]any{"port": servedPort}}}, + Make: w.run.Make, Gone: w.run.Gone, Logf: func(string, ...any) {}}) + out, err := w.p.Pass(ctx) + if err != nil { + w.t.Fatal(err) + } + w.run.retire(ctx) + return out +} + +func bindings(n int) (map[[2]string]string, []string) { + bound := map[[2]string]string{} + var logins []string + for i := 1; i <= n; i++ { + module := fmt.Sprintf("app%d", i) + bound[[2]string{"ace", module}] = "pw-" + module + logins = append(logins, provider.ConsumerIdentity("ace", module)) + } + return bound, logins +} + +// The consumers' file makes the same calls on the adapter as the equivalent grant files do (the harness). +func TestTheConsumersFileMakesWhatTheGrantFilesMake(t *testing.T) { + letta := provider.ConsumerIdentity("ace", "letta") + plain := provider.ConsumerIdentity("ace", "plain") + + // Today: the grant files, through the harness. + g := newWorld(t) + g.give( + map[string]any{"as": letta, "node": "ace", "values": map[string]any{"name": "letta", "extensions": []any{"vector"}}}, + map[string]any{"as": plain, "node": "ace", "values": map[string]any{"name": "plain"}}, + ) + g.h.Reconcile(ctx) + + // The consumers' file, through the provider helper. + w := newMovedWorld(t, map[[2]string]string{{"ace", "letta"}: "pw-" + letta, {"ace", "plain"}: "pw-" + plain}) + w.bus.bind("ace", "letta", map[string]any{"name": "letta", "extensions": []any{"vector"}}) + w.bus.bind("ace", "plain", map[string]any{"name": "plain"}) + for _, o := range w.pass() { + if o.Status != provider.Answered { + t.Fatalf("%+v", o) + } + } + byAs := map[string]Provision{} + for _, c := range w.a.created { + byAs[c.As] = c + } + if len(w.a.created) != 2 { + t.Fatalf("created %d, want 2", len(w.a.created)) + } + for _, want := range g.a.created { + c := byAs[want.As] + if c.As != want.As || c.Password != want.Password || c.Consumer != want.Consumer || + !reflect.DeepEqual(orEmpty(c.Values), orEmpty(want.Values)) { + t.Fatalf("made %+v, the grant files make %+v", c, want) + } + } + // Unchanged, and held: nothing is made again, within a minute or after it. + w.a.held = true + w.pass() + w.now = w.now.Add(2 * time.Minute) + w.pass() + if len(w.a.created) != 2 { + t.Fatalf("an unchanged pass created again: %d", len(w.a.created)) + } + // No longer held: made again. + w.a.held = false + w.now = w.now.Add(2 * time.Minute) + w.pass() + if len(w.a.created) != 4 { + t.Fatalf("not made again: %d", len(w.a.created)) + } +} + +// One read without a consumer retires nothing: it takes the harness's stable passes and time, and then the entry +// goes and the login is retired, never dropped. +func TestAConsumerMissingFromOneReadRetiresNothing(t *testing.T) { + bound, logins := bindings(2) + w := newMovedWorld(t, bound) + w.bus.bind("ace", "app1", nil) + w.bus.bind("ace", "app2", nil) + w.pass() + + // One read without app2. + saved := w.bus.buckets[provider.AssignmentsBucket]["ace.app2"] + w.bus.Delete(ctx, provider.AssignmentsBucket, "ace.app2") + w.pass() + if len(w.a.removed) != 0 { + t.Fatalf("retired %v on one read", w.a.removed) + } + if _, kept := w.bus.buckets[provider.BucketOf("postgres")]["ace.app2.postgres-database"]; !kept { + t.Fatal("the entry went on one read") + } + // It is back: nothing was lost, and the entry never left the bucket. + w.bus.Put(ctx, provider.AssignmentsBucket, "ace.app2", saved) + for i := 0; i < 8; i++ { + w.pass() + } + w.now = w.now.Add(15 * time.Minute) + w.pass() + if len(w.a.removed) != 0 || len(w.bus.buckets[provider.BucketOf("postgres")]) != 2 { + t.Fatalf("retired %v, entries %d", w.a.removed, len(w.bus.buckets[provider.BucketOf("postgres")])) + } + + // Gone for good: not retired before StablePasses and StableFor, then retired and its entry removed. + w.bus.Delete(ctx, provider.AssignmentsBucket, "ace.app2") + for i := 0; i < 6; i++ { + w.pass() + } + if len(w.a.removed) != 0 { + t.Fatalf("retired %v before the time was out", w.a.removed) + } + w.now = w.now.Add(11 * time.Minute) + w.pass() + w.pass() + if !reflect.DeepEqual(w.a.removed, []string{logins[1]}) || len(w.a.deleted) != 0 { + t.Fatalf("retired %v, deleted %v", w.a.removed, w.a.deleted) + } + if _, kept := w.bus.buckets[provider.BucketOf("postgres")]["ace.app2.postgres-database"]; kept { + t.Fatal("the entry stayed after the login was retired") + } +} + +// More than three at once waits for a person, as the harness's bound says. +func TestMoreThanTheBoundWaitsForAPerson(t *testing.T) { + bound, _ := bindings(5) + w := newMovedWorld(t, bound) + for i := 1; i <= 5; i++ { + w.bus.bind("ace", fmt.Sprintf("app%d", i), nil) + } + w.pass() + for i := 1; i <= 4; i++ { + w.bus.Delete(ctx, provider.AssignmentsBucket, fmt.Sprintf("ace.app%d", i)) + } + for i := 0; i < 6; i++ { + w.pass() + } + w.now = w.now.Add(11 * time.Minute) + w.pass() + w.pass() + if len(w.a.removed) != 0 || w.h.r.waiting == nil { + t.Fatalf("retired %v, waiting %v", w.a.removed, w.h.r.waiting) + } +} + +// A plan list that was never read is not nobody asking. +func TestNoPlansReadRetiresNothing(t *testing.T) { + bound, _ := bindings(1) + w := newMovedWorld(t, bound) + w.bus.bind("ace", "app1", nil) + w.pass() + w.bus.Delete(ctx, provider.AssignmentsBucket, "ace.app1") + for i := 0; i < 8; i++ { + w.pass() + } + w.now = w.now.Add(20 * time.Minute) + w.pass() + if len(w.a.removed) != 0 { + t.Fatalf("retired %v with no plans read", w.a.removed) + } +} + +// A consumer whose credential is not in the file yet is waited for: nothing is made for it. +func TestAConsumerWithoutACredentialIsWaitedFor(t *testing.T) { + w := newMovedWorld(t, map[[2]string]string{}) + w.bus.bind("ace", "letta", map[string]any{"name": "letta"}) + outcomes := w.pass() + if len(outcomes) != 1 || outcomes[0].Status != provider.Waiting || len(w.a.created) != 0 { + t.Fatalf("%+v, created %v", outcomes, w.a.created) + } +} + +// Where the file is not there, postgres takes the grant directory, as before; where it is, the helper. +func TestTheGrantPathIsTakenWhereTheConsumersFileIsAbsent(t *testing.T) { + dir := t.TempDir() + at := filepath.Join(dir, "consumers.json") + env := func(k string) string { + if k == "MESH_CONSUMERS_FILE" { + return at + } + return "" + } + if got := handedConsumersFile(env, os.Stat); got != "" { + t.Fatalf("took %q for a file that is not there", got) + } + if got := handedConsumersFile(func(string) string { return "" }, os.Stat); got != "" { + t.Fatalf("took %q with no environment", got) + } + if err := os.WriteFile(at, []byte("[]"), 0o600); err != nil { + t.Fatal(err) + } + if got := handedConsumersFile(env, os.Stat); got != at { + t.Fatalf("took %q, want %q", got, at) + } +} + +// The grant path restarts itself when the consumers file is first filled. +func TestTheGrantPathLeavesWhenTheConsumersFileIsFilled(t *testing.T) { + at := filepath.Join(t.TempDir(), "consumers.json") + done := make(chan struct{}) + go exitWhenFilled(at, os.Stat, time.Millisecond, func() { close(done) }) + select { + case <-done: + t.Fatal("left before the file was there") + case <-time.After(20 * time.Millisecond): + } + if err := os.WriteFile(at, []byte("[]"), 0o600); err != nil { + t.Fatal(err) + } + select { + case <-done: + case <-time.After(2 * time.Second): + t.Fatal("did not leave when the file was filled") + } +} + +// A consumer whose module has a slug keeps the login its plan names, the slug first (issue 788): the adapter makes it +// under that login, and the retirement counts it as asked for, so a pass retires nothing. +func TestASluggedConsumerKeepsTheLoginItsPlanNames(t *testing.T) { + agent := provider.ConsumerIdentity("ace", "agent") + w := newMovedWorld(t, map[[2]string]string{{"ace", "claude-code"}: "pw-agent"}) + plan, _ := json.Marshal(map[string]any{"node": "ace", "module": "claude-code", "provisioning": map[string]any{ + "provisions": []any{map[string]any{"provision": ProvisionName, "provider": "novox", + "provider-module": "postgres", "as": agent}}}}) + if err := w.bus.Put(ctx, provider.AssignmentsBucket, "ace.claude-code", plan); err != nil { + t.Fatal(err) + } + for _, o := range w.pass() { + if o.Status != provider.Answered { + t.Fatalf("%+v", o) + } + } + if len(w.a.created) != 1 || w.a.created[0].As != agent { + t.Fatalf("made %+v, want one under %s", w.a.created, agent) + } + for i := 0; i < 8; i++ { + w.pass() + } + w.now = w.now.Add(15 * time.Minute) + w.pass() + w.pass() + if len(w.a.removed) != 0 { + t.Fatalf("retired %v: the plan's login was counted unasked", w.a.removed) + } +} diff --git a/modules/postgres/files/consumers.json b/modules/postgres/files/consumers.json new file mode 100644 index 00000000..30d160e4 --- /dev/null +++ b/modules/postgres/files/consumers.json @@ -0,0 +1 @@ +${consumers:postgres-database} diff --git a/modules/postgres/go.mod b/modules/postgres/go.mod index d4bf4ff3..820d0e67 100644 --- a/modules/postgres/go.mod +++ b/modules/postgres/go.mod @@ -3,7 +3,7 @@ module postgres go 1.25.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.7 + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261015210729-d3e182503c19 github.com/jackc/pgx/v5 v5.11.0 ) diff --git a/modules/postgres/go.sum b/modules/postgres/go.sum index 79d630df..929f7d28 100644 --- a/modules/postgres/go.sum +++ b/modules/postgres/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.7 h1:C0sTQmtTiyYH7bnqZb7PusXnqA37gKuT7Nqjn9gG47w= -git.novox.be/novox/mesh-sdk/go v0.1.7/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261015210729-d3e182503c19 h1:mAMt3KFmrHzDiBv4MLL94opWcG9y73ISMrBUyisapQE= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261015210729-d3e182503c19/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= diff --git a/modules/postgres/module.json b/modules/postgres/module.json index b55ed746..934c2ef4 100644 --- a/modules/postgres/module.json +++ b/modules/postgres/module.json @@ -1,6 +1,7 @@ { "module": "postgres", "version": "1", + "moved": true, "upgrade": { "policy": "record", "why": "a provider whose restart drops every consumer on its machine, and whose new major version changes its data's format: a person takes each build, after a backup (hq ADR 0236)" @@ -102,6 +103,13 @@ "type": "directory", "mode": "0700" }, + { + "id": "consumers", + "type": "file", + "path": "${dir:state}/consumers.json", + "mode": "0600", + "template": "files/consumers.json" + }, { "id": "store-data", "type": "directory", @@ -159,9 +167,21 @@ "MESH_PROVISION_POSTGRES": "postgres://postgres@127.0.0.1:${port:5432}/postgres?sslmode=disable", "MESH_PROVISION_PASSWORD_FILE": "${dir:state}/superuser.secret", "MESH_RECEIVES": "${dir:grants}/mesh.json", - "MESH_POSTGRES_READER_PASSWORD_FILE": "${dir:state}/reader.secret" + "MESH_POSTGRES_READER_PASSWORD_FILE": "${dir:state}/reader.secret", + "MESH_CONSUMERS_FILE": "${dir:state}/consumers.json" } + }, + { + "name": "files", + "kind": "archive", + "from": "files" } ] - } + }, + "state": [ + "bound" + ], + "reads": [ + "mesh-controller.assignments" + ] } diff --git a/modules/power/README.md b/modules/power/README.md index d1076525..e353aaa0 100644 --- a/modules/power/README.md +++ b/modules/power/README.md @@ -35,8 +35,9 @@ resume units, is not a contribution. Its module keeps its own drop-ins on the sl ## The bar's battery and power draw (novox/hq ADR 0255, ADR 0260) The module holds the machine's power everywhere, so it gives the bar what is generic about power, on -the machines that have it. Both are contributions to `node-bar` in the bar's own neutral terms, so any -bar renders them, and each names the capability the node-engine reports for it: +the machines that have it. Both are items of the collection `contribution:node-bar:block`, each the block text in +i3status-rust's grammar (`templates/bar-battery`, `templates/bar-power-draw`), placed on the bottom bar's status +place, and each names the capability the node-engine reports for it: | block | where | what it shows | |---|---|---| diff --git a/modules/power/cmd/power/templates_test.go b/modules/power/cmd/power/templates_test.go new file mode 100644 index 00000000..db4b3f77 --- /dev/null +++ b/modules/power/cmd/power/templates_test.go @@ -0,0 +1,129 @@ +package main + +import ( + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" +) + +// Each file's text is a template in templates/, the directory the manifest names (novox/hq ADR 0349 §2, issue 731), +// never in files/: files/ is the archive the machine unpacks, and stays exactly what the machine is given. +func TestEveryFileTakesItsTextFromATemplateOutsideTheUnpackedFiles(t *testing.T) { + raw, err := os.ReadFile("../../module.json") + if err != nil { + t.Fatal(err) + } + var m struct { + Templates string `json:"templates"` + Resources []map[string]any `json:"resources"` + Build struct { + Artifacts []map[string]any `json:"artifacts"` + } `json:"build"` + } + if err := json.Unmarshal(raw, &m); err != nil { + t.Fatal(err) + } + if m.Templates != "templates" { + t.Fatalf("the manifest names the templates directory %q, not templates", m.Templates) + } + files := 0 + for _, r := range m.Resources { + if _, inline := r["content"]; inline { + t.Errorf("resource %v keeps its text inline: keep it in templates/ and name it as the template", r["id"]) + } + if r["type"] != "file" { + continue + } + files++ + name, _ := r["template"].(string) + if !strings.HasPrefix(name, "templates/") { + t.Errorf("resource %v names the template %q, not one in templates/", r["id"], name) + continue + } + if st, err := os.Lstat(filepath.Join("../..", name)); err != nil || !st.Mode().IsRegular() { + t.Errorf("resource %v names the template %s, which is not a plain file: %v", r["id"], name, err) + } + } + if files == 0 { + t.Fatal("no file resource") + } + packs := 0 + for _, a := range m.Build.Artifacts { + if a["kind"] == "archive" && a["from"] == "templates" { + packs++ + if a["name"] != "templates" { + t.Errorf("the templates archive is named %v, not templates", a["name"]) + } + } + } + if packs != 1 { + t.Fatalf("the build packs templates/ into %d archives, not one", packs) + } + // The templates archive is read by the controller, never unpacked onto the machine. + for _, r := range m.Resources { + if r["artifact"] == "templates" { + t.Errorf("resource %v places the templates archive on the machine", r["id"]) + } + } + // And files/ holds what the machine runs, no template. + err = filepath.WalkDir("../../files", func(at string, d os.DirEntry, err error) error { + if err != nil { + return err + } + if !d.IsDir() && !strings.HasPrefix(filepath.ToSlash(at), "../../files/bin/") { + t.Errorf("files/ holds %s, which is not one of the scripts the machine unpacks", at) + } + return nil + }) + if err != nil { + t.Fatal(err) + } +} + +// The bar's blocks are two items of the collection the bar places (design 57 L3, issue 781): the text i3status-rust's +// bars hold for the battery and for the power draw, byte for byte as the bar's own template rendered them before, the +// draw's key being the machine's own name. +func TestTheBarBlocksAreTwoItemsOfTheBarsCollection(t *testing.T) { + raw, err := os.ReadFile("../../module.json") + if err != nil { + t.Fatal(err) + } + var m struct { + Contributions []any `json:"contributions"` + Items []struct { + Collection string `json:"collection"` + Name string `json:"name"` + IfCapability string `json:"if-capability"` + Attributes map[string]string `json:"attributes"` + Template string `json:"template"` + } `json:"items"` + } + if err := json.Unmarshal(raw, &m); err != nil { + t.Fatal(err) + } + if len(m.Contributions) != 0 || len(m.Items) != 2 { + t.Fatalf("%d contributions and %d items", len(m.Contributions), len(m.Items)) + } + want := []struct{ name, capability, order, template, text string }{ + {"battery", "battery", "40", "templates/bar-battery", "[[block]]\nblock = \"battery\"\n" + + "format = \"$icon $percentage{ $time|}\"\ncharging_format = \"$icon $percentage{ $time|}\"\n" + + "not_charging_format = \"$icon $percentage\"\nfull_format = \"$icon $percentage\"\n" + + "empty_format = \"$icon $percentage\"\nmissing_format = \"\"\n\n"}, + {"power-draw", "power-meter", "45", "templates/bar-power-draw", "[[block]]\nblock = \"custom\"\n" + + "command = \"~/.local/bin/i3status-value power.draw ${machine:name}\"\ninterval = 5\nhide_when_empty = true\n\n"}, + } + for i, w := range want { + it := m.Items[i] + if it.Collection != "contribution:node-bar:block" || it.Name != w.name || it.IfCapability != w.capability || + it.Template != w.template || it.Attributes["bar"] != "bottom" || it.Attributes["place"] != "status" || + it.Attributes["order"] != w.order || len(it.Attributes) != 3 { + t.Errorf("item %d is %+v", i, it) + } + text, err := os.ReadFile(filepath.Join("../..", w.template)) + if err != nil || string(text) != w.text { + t.Errorf("%s holds %q (%v), want %q", w.template, text, err, w.text) + } + } +} diff --git a/modules/power/module.json b/modules/power/module.json index 6411982c..cea72ad7 100644 --- a/modules/power/module.json +++ b/modules/power/module.json @@ -1,6 +1,8 @@ { "module": "power", "version": "1", + "moved": true, + "templates": "templates", "capabilities": [ "package-manager", "service-manager" @@ -34,6 +36,88 @@ "power_run", "power_check" ], + "config": { + "handle-power-key": { + "type": "string", + "description": "What logind does when the power key is pressed: one of logind's actions, such as suspend, ignore or poweroff (logind.conf(5)); the operator's choice for this machine, with no default.", + "enum": [ + "ignore", + "poweroff", + "reboot", + "halt", + "kexec", + "suspend", + "hibernate", + "hybrid-sleep", + "suspend-then-hibernate", + "lock", + "factory-reset", + "sleep" + ], + "trusted": true, + "required": true + }, + "handle-lid-switch": { + "type": "string", + "description": "What logind does when the lid is closed on battery: one of logind's actions, such as suspend, ignore or poweroff (logind.conf(5)); the operator's choice for this machine, with no default.", + "enum": [ + "ignore", + "poweroff", + "reboot", + "halt", + "kexec", + "suspend", + "hibernate", + "hybrid-sleep", + "suspend-then-hibernate", + "lock", + "factory-reset", + "sleep" + ], + "trusted": true, + "required": true + }, + "handle-lid-switch-external-power": { + "type": "string", + "description": "What logind does when the lid is closed on external power: one of logind's actions, such as suspend, ignore or poweroff (logind.conf(5)); the operator's choice for this machine, with no default.", + "enum": [ + "ignore", + "poweroff", + "reboot", + "halt", + "kexec", + "suspend", + "hibernate", + "hybrid-sleep", + "suspend-then-hibernate", + "lock", + "factory-reset", + "sleep" + ], + "trusted": true, + "required": true + }, + "handle-lid-switch-docked": { + "type": "string", + "description": "What logind does when the lid is closed while docked or with another screen attached: one of logind's actions, such as suspend, ignore or poweroff (logind.conf(5)); the operator's choice for this machine, with no default.", + "enum": [ + "ignore", + "poweroff", + "reboot", + "halt", + "kexec", + "suspend", + "hibernate", + "hybrid-sleep", + "suspend-then-hibernate", + "lock", + "factory-reset", + "sleep" + ], + "trusted": true, + "required": true + } + }, "resources": [ { "id": "polkit", @@ -63,42 +147,42 @@ "type": "file", "path": "/etc/mesh-power/moments/after-boot", "mode": "0644", - "content": "# What every module on this machine runs at the moment after-boot, as the mesh placed it (module power,\n# novox/hq ADR 0211). Replaced on every push: code is added by the module it belongs to, as a\n# contribution for this moment. Each module's piece runs on its own, as root, bounded in time.\n${shell:after-boot:first}${shell:after-boot:normal}${shell:after-boot:last}" + "template": "templates/mesh-power/moments/after-boot" }, { "id": "moment-before-sleep", "type": "file", "path": "/etc/mesh-power/moments/before-sleep", "mode": "0644", - "content": "# What every module on this machine runs at the moment before-sleep, as the mesh placed it (module power,\n# novox/hq ADR 0211). Replaced on every push: code is added by the module it belongs to, as a\n# contribution for this moment. Each module's piece runs on its own, as root, bounded in time.\n${shell:before-sleep:first}${shell:before-sleep:normal}${shell:before-sleep:last}" + "template": "templates/mesh-power/moments/before-sleep" }, { "id": "moment-after-wake", "type": "file", "path": "/etc/mesh-power/moments/after-wake", "mode": "0644", - "content": "# What every module on this machine runs at the moment after-wake, as the mesh placed it (module power,\n# novox/hq ADR 0211). Replaced on every push: code is added by the module it belongs to, as a\n# contribution for this moment. Each module's piece runs on its own, as root, bounded in time.\n${shell:after-wake:first}${shell:after-wake:normal}${shell:after-wake:last}" + "template": "templates/mesh-power/moments/after-wake" }, { "id": "moment-before-shutdown", "type": "file", "path": "/etc/mesh-power/moments/before-shutdown", "mode": "0644", - "content": "# What every module on this machine runs at the moment before-shutdown, as the mesh placed it (module power,\n# novox/hq ADR 0211). Replaced on every push: code is added by the module it belongs to, as a\n# contribution for this moment. Each module's piece runs on its own, as root, bounded in time.\n${shell:before-shutdown:first}${shell:before-shutdown:normal}${shell:before-shutdown:last}" + "template": "templates/mesh-power/moments/before-shutdown" }, { "id": "moment-on-mains", "type": "file", "path": "/etc/mesh-power/moments/on-mains", "mode": "0644", - "content": "# What every module on this machine runs at the moment on-mains, as the mesh placed it (module power,\n# novox/hq ADR 0211). Replaced on every push: code is added by the module it belongs to, as a\n# contribution for this moment. Each module's piece runs on its own, as root, bounded in time.\n${shell:on-mains:first}${shell:on-mains:normal}${shell:on-mains:last}" + "template": "templates/mesh-power/moments/on-mains" }, { "id": "moment-on-battery", "type": "file", "path": "/etc/mesh-power/moments/on-battery", "mode": "0644", - "content": "# What every module on this machine runs at the moment on-battery, as the mesh placed it (module power,\n# novox/hq ADR 0211). Replaced on every push: code is added by the module it belongs to, as a\n# contribution for this moment. Each module's piece runs on its own, as root, bounded in time.\n${shell:on-battery:first}${shell:on-battery:normal}${shell:on-battery:last}" + "template": "templates/mesh-power/moments/on-battery" }, { "id": "logind-drop-ins", @@ -111,7 +195,7 @@ "type": "file", "path": "/etc/systemd/logind.conf.d/power.conf", "mode": "0644", - "content": "# Written by the mesh (module power, novox/hq ADR 0211). Replaced on every push.\n#\n# The power key and the lid, from this module's settings: the mesh's, then this machine's.\n[Login]\nHandlePowerKey=${setting:handle-power-key}\nHandleLidSwitch=${setting:handle-lid-switch}\nHandleLidSwitchExternalPower=${setting:handle-lid-switch-external-power}\nHandleLidSwitchDocked=${setting:handle-lid-switch-docked}\n" + "template": "templates/systemd/logind.conf.d/power.conf" }, { "id": "logind", @@ -126,35 +210,35 @@ "type": "file", "path": "/etc/systemd/system/mesh-power-after-boot.service", "mode": "0644", - "content": "# Written by the mesh (module power, novox/hq ADR 0211). Replaced on every push.\n[Unit]\nDescription=Every module's code after boot (mesh power)\nWants=network-online.target\nAfter=network-online.target\n\n[Service]\nType=oneshot\nRemainAfterExit=yes\nExecStart=/usr/local/lib/mesh-power/bin/power-moment after-boot\n\n[Install]\nWantedBy=multi-user.target\n" + "template": "templates/systemd/system/mesh-power-after-boot.service" }, { "id": "before-shutdown-unit", "type": "file", "path": "/etc/systemd/system/mesh-power-before-shutdown.service", "mode": "0644", - "content": "# Written by the mesh (module power, novox/hq ADR 0211). Replaced on every push.\n#\n# Started at boot and left active; stopping it is the shutdown. Ordered after the network, so at\n# shutdown it stops, and runs every module's code, while the network is still up.\n[Unit]\nDescription=Every module's code before shutdown (mesh power)\nWants=network-online.target\nAfter=network-online.target\n\n[Service]\nType=oneshot\nRemainAfterExit=yes\nExecStart=/bin/true\nExecStop=/usr/local/lib/mesh-power/bin/power-moment before-shutdown\nTimeoutStopSec=120\n\n[Install]\nWantedBy=multi-user.target\n" + "template": "templates/systemd/system/mesh-power-before-shutdown.service" }, { "id": "before-sleep-unit", "type": "file", "path": "/etc/systemd/system/mesh-power-before-sleep.service", "mode": "0644", - "content": "# Written by the mesh (module power, novox/hq ADR 0211). Replaced on every push.\n#\n# Pulled in by sleep.target through this module's drop-in on it, never enabled: a one-shot waiting\n# for a sleep is not a service whose state the mesh can declare.\n[Unit]\nDescription=Every module's code before sleep (mesh power)\nBefore=sleep.target\n\n[Service]\nType=oneshot\nExecStart=/usr/local/lib/mesh-power/bin/power-moment before-sleep\n" + "template": "templates/systemd/system/mesh-power-before-sleep.service" }, { "id": "after-wake-unit", "type": "file", "path": "/etc/systemd/system/mesh-power-after-wake.service", "mode": "0644", - "content": "# Written by the mesh (module power, novox/hq ADR 0211). Replaced on every push.\n#\n# Pulled in by the four sleep targets through this module's drop-ins on them, and ordered after\n# them, so it runs once the machine is awake again. Never enabled.\n[Unit]\nDescription=Every module's code after waking (mesh power)\nAfter=suspend.target hibernate.target hybrid-sleep.target suspend-then-hibernate.target\n\n[Service]\nType=oneshot\nExecStart=/usr/local/lib/mesh-power/bin/power-moment after-wake\n" + "template": "templates/systemd/system/mesh-power-after-wake.service" }, { "id": "supply-unit", "type": "file", "path": "/etc/systemd/system/mesh-power-supply.service", "mode": "0644", - "content": "# Written by the mesh (module power, novox/hq ADR 0211). Replaced on every push.\n#\n# Started by the module's udev rule when a power supply changes; runs on-mains or on-battery once per\n# change of source.\n[Unit]\nDescription=Every module's code for the power source (mesh power)\n\n[Service]\nType=oneshot\nExecStart=/usr/local/lib/mesh-power/bin/power-moment supply\n" + "template": "templates/systemd/system/mesh-power-supply.service" }, { "id": "sleep-drop-ins", @@ -167,7 +251,7 @@ "type": "file", "path": "/etc/systemd/system/sleep.target.d/mesh-power.conf", "mode": "0644", - "content": "# Written by the mesh (module power, novox/hq ADR 0211). Replaced on every push.\n[Unit]\nWants=mesh-power-before-sleep.service\n" + "template": "templates/systemd/system/sleep.target.d/mesh-power.conf" }, { "id": "suspend-drop-ins", @@ -180,7 +264,7 @@ "type": "file", "path": "/etc/systemd/system/suspend.target.d/mesh-power.conf", "mode": "0644", - "content": "# Written by the mesh (module power, novox/hq ADR 0211). Replaced on every push.\n[Unit]\nWants=mesh-power-after-wake.service\n" + "template": "templates/systemd/system/suspend.target.d/mesh-power.conf" }, { "id": "hibernate-drop-ins", @@ -193,7 +277,7 @@ "type": "file", "path": "/etc/systemd/system/hibernate.target.d/mesh-power.conf", "mode": "0644", - "content": "# Written by the mesh (module power, novox/hq ADR 0211). Replaced on every push.\n[Unit]\nWants=mesh-power-after-wake.service\n" + "template": "templates/systemd/system/hibernate.target.d/mesh-power.conf" }, { "id": "hybrid-sleep-drop-ins", @@ -206,7 +290,7 @@ "type": "file", "path": "/etc/systemd/system/hybrid-sleep.target.d/mesh-power.conf", "mode": "0644", - "content": "# Written by the mesh (module power, novox/hq ADR 0211). Replaced on every push.\n[Unit]\nWants=mesh-power-after-wake.service\n" + "template": "templates/systemd/system/hybrid-sleep.target.d/mesh-power.conf" }, { "id": "suspend-then-hibernate-drop-ins", @@ -219,14 +303,14 @@ "type": "file", "path": "/etc/systemd/system/suspend-then-hibernate.target.d/mesh-power.conf", "mode": "0644", - "content": "# Written by the mesh (module power, novox/hq ADR 0211). Replaced on every push.\n[Unit]\nWants=mesh-power-after-wake.service\n" + "template": "templates/systemd/system/suspend-then-hibernate.target.d/mesh-power.conf" }, { "id": "udev-rule", "type": "file", "path": "/etc/udev/rules.d/90-mesh-power.rules", "mode": "0644", - "content": "# Written by the mesh (module power, novox/hq ADR 0211). Replaced on every push.\n# Only the charger's change can switch the machine between mains and battery. A battery reports its\n# level many times a minute, and USB-C supplies come and go; matching them started the supply unit\n# every second or two on the laptop.\nSUBSYSTEM==\"power_supply\", ACTION==\"change\", ATTR{type}==\"Mains\", RUN+=\"/usr/bin/systemctl --no-block start mesh-power-supply.service\"\n" + "template": "templates/udev/rules.d/90-mesh-power.rules" }, { "id": "after-boot", @@ -279,34 +363,36 @@ "name": "scripts", "kind": "archive", "from": "files" + }, + { + "name": "templates", + "kind": "archive", + "from": "templates" } ] }, - "contributions": [ + "items": [ { - "seat": "node-bar", - "kind": "block", + "collection": "contribution:node-bar:block", + "name": "battery", "if-capability": "battery", - "data": { + "attributes": { "bar": "bottom", "place": "status", - "order": 40, - "shows": "battery" - } + "order": "40" + }, + "template": "templates/bar-battery" }, { - "seat": "node-bar", - "kind": "block", + "collection": "contribution:node-bar:block", + "name": "power-draw", "if-capability": "power-meter", - "data": { + "attributes": { "bar": "bottom", "place": "status", - "order": 45, - "shows": "state", - "options": { - "state": "power.draw" - } - } + "order": "45" + }, + "template": "templates/bar-power-draw" } ] } diff --git a/modules/power/templates/bar-battery b/modules/power/templates/bar-battery new file mode 100644 index 00000000..83b15e0a --- /dev/null +++ b/modules/power/templates/bar-battery @@ -0,0 +1,8 @@ +[[block]] +block = "battery" +format = "$icon $percentage{ $time|}" +charging_format = "$icon $percentage{ $time|}" +not_charging_format = "$icon $percentage" +full_format = "$icon $percentage" +empty_format = "$icon $percentage" +missing_format = "" diff --git a/modules/power/templates/bar-power-draw b/modules/power/templates/bar-power-draw new file mode 100644 index 00000000..e78da74d --- /dev/null +++ b/modules/power/templates/bar-power-draw @@ -0,0 +1,5 @@ +[[block]] +block = "custom" +command = "~/.local/bin/i3status-value power.draw ${machine:name}" +interval = 5 +hide_when_empty = true diff --git a/modules/power/templates/mesh-power/moments/after-boot b/modules/power/templates/mesh-power/moments/after-boot new file mode 100644 index 00000000..c5a90f6b --- /dev/null +++ b/modules/power/templates/mesh-power/moments/after-boot @@ -0,0 +1,4 @@ +# What every module on this machine runs at the moment after-boot, as the mesh placed it (module power, +# novox/hq ADR 0211). Replaced on every push: code is added by the module it belongs to, as a +# contribution for this moment. Each module's piece runs on its own, as root, bounded in time. +${shell:after-boot:first}${shell:after-boot:normal}${shell:after-boot:last} \ No newline at end of file diff --git a/modules/power/templates/mesh-power/moments/after-wake b/modules/power/templates/mesh-power/moments/after-wake new file mode 100644 index 00000000..2655c226 --- /dev/null +++ b/modules/power/templates/mesh-power/moments/after-wake @@ -0,0 +1,4 @@ +# What every module on this machine runs at the moment after-wake, as the mesh placed it (module power, +# novox/hq ADR 0211). Replaced on every push: code is added by the module it belongs to, as a +# contribution for this moment. Each module's piece runs on its own, as root, bounded in time. +${shell:after-wake:first}${shell:after-wake:normal}${shell:after-wake:last} \ No newline at end of file diff --git a/modules/power/templates/mesh-power/moments/before-shutdown b/modules/power/templates/mesh-power/moments/before-shutdown new file mode 100644 index 00000000..65cc76d3 --- /dev/null +++ b/modules/power/templates/mesh-power/moments/before-shutdown @@ -0,0 +1,4 @@ +# What every module on this machine runs at the moment before-shutdown, as the mesh placed it (module power, +# novox/hq ADR 0211). Replaced on every push: code is added by the module it belongs to, as a +# contribution for this moment. Each module's piece runs on its own, as root, bounded in time. +${shell:before-shutdown:first}${shell:before-shutdown:normal}${shell:before-shutdown:last} \ No newline at end of file diff --git a/modules/power/templates/mesh-power/moments/before-sleep b/modules/power/templates/mesh-power/moments/before-sleep new file mode 100644 index 00000000..e5fc2a5f --- /dev/null +++ b/modules/power/templates/mesh-power/moments/before-sleep @@ -0,0 +1,4 @@ +# What every module on this machine runs at the moment before-sleep, as the mesh placed it (module power, +# novox/hq ADR 0211). Replaced on every push: code is added by the module it belongs to, as a +# contribution for this moment. Each module's piece runs on its own, as root, bounded in time. +${shell:before-sleep:first}${shell:before-sleep:normal}${shell:before-sleep:last} \ No newline at end of file diff --git a/modules/power/templates/mesh-power/moments/on-battery b/modules/power/templates/mesh-power/moments/on-battery new file mode 100644 index 00000000..58e41a97 --- /dev/null +++ b/modules/power/templates/mesh-power/moments/on-battery @@ -0,0 +1,4 @@ +# What every module on this machine runs at the moment on-battery, as the mesh placed it (module power, +# novox/hq ADR 0211). Replaced on every push: code is added by the module it belongs to, as a +# contribution for this moment. Each module's piece runs on its own, as root, bounded in time. +${shell:on-battery:first}${shell:on-battery:normal}${shell:on-battery:last} \ No newline at end of file diff --git a/modules/power/templates/mesh-power/moments/on-mains b/modules/power/templates/mesh-power/moments/on-mains new file mode 100644 index 00000000..b4a65866 --- /dev/null +++ b/modules/power/templates/mesh-power/moments/on-mains @@ -0,0 +1,4 @@ +# What every module on this machine runs at the moment on-mains, as the mesh placed it (module power, +# novox/hq ADR 0211). Replaced on every push: code is added by the module it belongs to, as a +# contribution for this moment. Each module's piece runs on its own, as root, bounded in time. +${shell:on-mains:first}${shell:on-mains:normal}${shell:on-mains:last} \ No newline at end of file diff --git a/modules/power/templates/systemd/logind.conf.d/power.conf b/modules/power/templates/systemd/logind.conf.d/power.conf new file mode 100644 index 00000000..9cf93bdc --- /dev/null +++ b/modules/power/templates/systemd/logind.conf.d/power.conf @@ -0,0 +1,8 @@ +# Written by the mesh (module power, novox/hq ADR 0211). Replaced on every push. +# +# The power key and the lid, from this module's settings: the mesh's, then this machine's. +[Login] +HandlePowerKey=${config:handle-power-key} +HandleLidSwitch=${config:handle-lid-switch} +HandleLidSwitchExternalPower=${config:handle-lid-switch-external-power} +HandleLidSwitchDocked=${config:handle-lid-switch-docked} diff --git a/modules/power/templates/systemd/system/hibernate.target.d/mesh-power.conf b/modules/power/templates/systemd/system/hibernate.target.d/mesh-power.conf new file mode 100644 index 00000000..81223d3c --- /dev/null +++ b/modules/power/templates/systemd/system/hibernate.target.d/mesh-power.conf @@ -0,0 +1,3 @@ +# Written by the mesh (module power, novox/hq ADR 0211). Replaced on every push. +[Unit] +Wants=mesh-power-after-wake.service diff --git a/modules/power/templates/systemd/system/hybrid-sleep.target.d/mesh-power.conf b/modules/power/templates/systemd/system/hybrid-sleep.target.d/mesh-power.conf new file mode 100644 index 00000000..81223d3c --- /dev/null +++ b/modules/power/templates/systemd/system/hybrid-sleep.target.d/mesh-power.conf @@ -0,0 +1,3 @@ +# Written by the mesh (module power, novox/hq ADR 0211). Replaced on every push. +[Unit] +Wants=mesh-power-after-wake.service diff --git a/modules/power/templates/systemd/system/mesh-power-after-boot.service b/modules/power/templates/systemd/system/mesh-power-after-boot.service new file mode 100644 index 00000000..b7045774 --- /dev/null +++ b/modules/power/templates/systemd/system/mesh-power-after-boot.service @@ -0,0 +1,13 @@ +# Written by the mesh (module power, novox/hq ADR 0211). Replaced on every push. +[Unit] +Description=Every module's code after boot (mesh power) +Wants=network-online.target +After=network-online.target + +[Service] +Type=oneshot +RemainAfterExit=yes +ExecStart=/usr/local/lib/mesh-power/bin/power-moment after-boot + +[Install] +WantedBy=multi-user.target diff --git a/modules/power/templates/systemd/system/mesh-power-after-wake.service b/modules/power/templates/systemd/system/mesh-power-after-wake.service new file mode 100644 index 00000000..773b2284 --- /dev/null +++ b/modules/power/templates/systemd/system/mesh-power-after-wake.service @@ -0,0 +1,11 @@ +# Written by the mesh (module power, novox/hq ADR 0211). Replaced on every push. +# +# Pulled in by the four sleep targets through this module's drop-ins on them, and ordered after +# them, so it runs once the machine is awake again. Never enabled. +[Unit] +Description=Every module's code after waking (mesh power) +After=suspend.target hibernate.target hybrid-sleep.target suspend-then-hibernate.target + +[Service] +Type=oneshot +ExecStart=/usr/local/lib/mesh-power/bin/power-moment after-wake diff --git a/modules/power/templates/systemd/system/mesh-power-before-shutdown.service b/modules/power/templates/systemd/system/mesh-power-before-shutdown.service new file mode 100644 index 00000000..52db0a4c --- /dev/null +++ b/modules/power/templates/systemd/system/mesh-power-before-shutdown.service @@ -0,0 +1,18 @@ +# Written by the mesh (module power, novox/hq ADR 0211). Replaced on every push. +# +# Started at boot and left active; stopping it is the shutdown. Ordered after the network, so at +# shutdown it stops, and runs every module's code, while the network is still up. +[Unit] +Description=Every module's code before shutdown (mesh power) +Wants=network-online.target +After=network-online.target + +[Service] +Type=oneshot +RemainAfterExit=yes +ExecStart=/bin/true +ExecStop=/usr/local/lib/mesh-power/bin/power-moment before-shutdown +TimeoutStopSec=120 + +[Install] +WantedBy=multi-user.target diff --git a/modules/power/templates/systemd/system/mesh-power-before-sleep.service b/modules/power/templates/systemd/system/mesh-power-before-sleep.service new file mode 100644 index 00000000..727587e3 --- /dev/null +++ b/modules/power/templates/systemd/system/mesh-power-before-sleep.service @@ -0,0 +1,11 @@ +# Written by the mesh (module power, novox/hq ADR 0211). Replaced on every push. +# +# Pulled in by sleep.target through this module's drop-in on it, never enabled: a one-shot waiting +# for a sleep is not a service whose state the mesh can declare. +[Unit] +Description=Every module's code before sleep (mesh power) +Before=sleep.target + +[Service] +Type=oneshot +ExecStart=/usr/local/lib/mesh-power/bin/power-moment before-sleep diff --git a/modules/power/templates/systemd/system/mesh-power-supply.service b/modules/power/templates/systemd/system/mesh-power-supply.service new file mode 100644 index 00000000..5096a00e --- /dev/null +++ b/modules/power/templates/systemd/system/mesh-power-supply.service @@ -0,0 +1,10 @@ +# Written by the mesh (module power, novox/hq ADR 0211). Replaced on every push. +# +# Started by the module's udev rule when a power supply changes; runs on-mains or on-battery once per +# change of source. +[Unit] +Description=Every module's code for the power source (mesh power) + +[Service] +Type=oneshot +ExecStart=/usr/local/lib/mesh-power/bin/power-moment supply diff --git a/modules/power/templates/systemd/system/sleep.target.d/mesh-power.conf b/modules/power/templates/systemd/system/sleep.target.d/mesh-power.conf new file mode 100644 index 00000000..096f7935 --- /dev/null +++ b/modules/power/templates/systemd/system/sleep.target.d/mesh-power.conf @@ -0,0 +1,3 @@ +# Written by the mesh (module power, novox/hq ADR 0211). Replaced on every push. +[Unit] +Wants=mesh-power-before-sleep.service diff --git a/modules/power/templates/systemd/system/suspend-then-hibernate.target.d/mesh-power.conf b/modules/power/templates/systemd/system/suspend-then-hibernate.target.d/mesh-power.conf new file mode 100644 index 00000000..81223d3c --- /dev/null +++ b/modules/power/templates/systemd/system/suspend-then-hibernate.target.d/mesh-power.conf @@ -0,0 +1,3 @@ +# Written by the mesh (module power, novox/hq ADR 0211). Replaced on every push. +[Unit] +Wants=mesh-power-after-wake.service diff --git a/modules/power/templates/systemd/system/suspend.target.d/mesh-power.conf b/modules/power/templates/systemd/system/suspend.target.d/mesh-power.conf new file mode 100644 index 00000000..81223d3c --- /dev/null +++ b/modules/power/templates/systemd/system/suspend.target.d/mesh-power.conf @@ -0,0 +1,3 @@ +# Written by the mesh (module power, novox/hq ADR 0211). Replaced on every push. +[Unit] +Wants=mesh-power-after-wake.service diff --git a/modules/power/templates/udev/rules.d/90-mesh-power.rules b/modules/power/templates/udev/rules.d/90-mesh-power.rules new file mode 100644 index 00000000..d0a75bbf --- /dev/null +++ b/modules/power/templates/udev/rules.d/90-mesh-power.rules @@ -0,0 +1,5 @@ +# Written by the mesh (module power, novox/hq ADR 0211). Replaced on every push. +# Only the charger's change can switch the machine between mains and battery. A battery reports its +# level many times a minute, and USB-C supplies come and go; matching them started the supply unit +# every second or two on the laptop. +SUBSYSTEM=="power_supply", ACTION=="change", ATTR{type}=="Mains", RUN+="/usr/bin/systemctl --no-block start mesh-power-supply.service" diff --git a/modules/powerlevel10k/files/zsh-normal b/modules/powerlevel10k/files/zsh-normal new file mode 100644 index 00000000..ba4fc1d6 --- /dev/null +++ b/modules/powerlevel10k/files/zsh-normal @@ -0,0 +1,3 @@ +# The prompt: powerlevel10k v1.20.0, pinned in this module (novox/hq ADR 0205), and its configuration. +[[ ! -f ~/.local/share/powerlevel10k/powerlevel10k.zsh-theme ]] || source ~/.local/share/powerlevel10k/powerlevel10k.zsh-theme +[[ ! -f ~/.config/powerlevel10k/p10k.zsh ]] || source ~/.config/powerlevel10k/p10k.zsh diff --git a/modules/powerlevel10k/module.json b/modules/powerlevel10k/module.json index 574a01ba..39d2810d 100644 --- a/modules/powerlevel10k/module.json +++ b/modules/powerlevel10k/module.json @@ -1,13 +1,7 @@ { "module": "powerlevel10k", "version": "1", - "shell": [ - { - "for": "zsh", - "slot": "normal", - "code": "# The prompt: powerlevel10k v1.20.0, pinned in this module (novox/hq ADR 0205), and its configuration.\n[[ ! -f ~/.local/share/powerlevel10k/powerlevel10k.zsh-theme ]] || source ~/.local/share/powerlevel10k/powerlevel10k.zsh-theme\n[[ ! -f ~/.config/powerlevel10k/p10k.zsh ]] || source ~/.config/powerlevel10k/p10k.zsh\n" - } - ], + "moved": true, "resources": [ { "id": "theme", @@ -24,6 +18,12 @@ "artifact": "configuration" } ], + "items": [ + { + "collection": "shell:zsh:normal", + "template": "files/zsh-normal" + } + ], "build": { "artifacts": [ { @@ -35,6 +35,11 @@ "name": "configuration", "kind": "archive", "from": "config" + }, + { + "name": "files", + "kind": "archive", + "from": "files" } ] } diff --git a/modules/powerlevel10k/test/module.test.ts b/modules/powerlevel10k/test/module.test.ts index 0fab4b1a..e9bc8522 100644 --- a/modules/powerlevel10k/test/module.test.ts +++ b/modules/powerlevel10k/test/module.test.ts @@ -26,14 +26,20 @@ test("both are unpacked under the account's home, owned by the account", () => { assert.deepEqual(byId.configuration, { id: "configuration", type: "archive", path: "${machine:account-home}/.config/powerlevel10k", owner: "${machine:account}", artifact: "configuration" }); }); +// The zsh code is an item of the collection shell:zsh:normal, its text a template in files/ (ADR 0348, 0349). +const shellCode = (): string => { + const items = m.items.filter((i: { collection: string }) => i.collection.startsWith("shell:")); + assert.equal(items.length, 1); + assert.equal(items[0].collection, "shell:zsh:normal", "for zsh, in the normal slot"); + return readFileSync(at(items[0].template), "utf8"); +}; + test("the zsh code in the normal slot sources the theme, then the configuration, and turns on no instant prompt", () => { - assert.equal(m.shell.length, 1); - const [c] = m.shell; - assert.equal(c.for, "zsh"); - assert.equal(c.slot, "normal"); - const sourced = [...(c.code as string).matchAll(/\|\| source (\S+)/g)].map((x) => x[1]); + assert.equal(m.shell, undefined); + const code = shellCode(); + const sourced = [...code.matchAll(/\|\| source (\S+)/g)].map((x) => x[1]); assert.deepEqual(sourced, ["~/.local/share/powerlevel10k/powerlevel10k.zsh-theme", "~/.config/powerlevel10k/p10k.zsh"]); - assert.doesNotMatch(c.code, /instant/); + assert.doesNotMatch(code, /instant/); }); test("the README names the upstream, the version and the licence", () => { @@ -41,5 +47,5 @@ test("the README names the upstream, the version and the licence", () => { assert.match(readme, /github\.com\/romkatv\/powerlevel10k/); assert.match(readme, /v1\.20\.0/); assert.match(readme, /MIT/); - assert.match(m.shell[0].code, /v1\.20\.0/, "the version in the shell code's comment matches"); + assert.match(shellCode(), /v1\.20\.0/, "the version in the shell code's comment matches"); }); diff --git a/modules/records/cmd/records/templates_test.go b/modules/records/cmd/records/templates_test.go new file mode 100644 index 00000000..7d0718b5 --- /dev/null +++ b/modules/records/cmd/records/templates_test.go @@ -0,0 +1,64 @@ +package main + +import ( + "encoding/json" + "os" + "path/filepath" + "testing" +) + +// The module's directory, where its manifest and its files/ are. +const moduleDir = "../.." + +// templateText is the text a file resource places on the machine: its template's, in the module's files/ +// (novox/hq ADR 0349 §2). A resource that names no template, or a template that is not there, fails the test. +func templateText(t *testing.T, r map[string]any) string { + t.Helper() + name, _ := r["template"].(string) + if name == "" { + t.Fatalf("resource %v names no template", r["id"]) + } + raw, err := os.ReadFile(filepath.Join(moduleDir, filepath.FromSlash(name))) + if err != nil { + t.Fatalf("resource %v names %s: %v", r["id"], name, err) + } + return string(raw) +} + +// No file resource carries its text inline: each names its template in files/, and the build packs files/ +// into the one archive the controller reads the templates back from (novox/hq ADR 0349 §2). +func TestEveryFileTakesItsTextFromATemplate(t *testing.T) { + raw, err := os.ReadFile(filepath.Join(moduleDir, "module.json")) + if err != nil { + t.Fatal(err) + } + var m struct { + Resources []map[string]any `json:"resources"` + Build struct { + Artifacts []map[string]any `json:"artifacts"` + } `json:"build"` + } + if err := json.Unmarshal(raw, &m); err != nil { + t.Fatal(err) + } + for _, r := range m.Resources { + if _, inline := r["content"]; inline { + t.Errorf("resource %v keeps its text inline: keep it in files/ and name it as the template", r["id"]) + } + if r["type"] == "file" { + templateText(t, r) + } + } + packs := 0 + for _, a := range m.Build.Artifacts { + if a["kind"] == "archive" && a["from"] == "files" { + packs++ + if a["name"] != "files" { + t.Errorf("the templates archive is named %v, not files", a["name"]) + } + } + } + if packs != 1 { + t.Errorf("the build packs files/ into %d archives, not one", packs) + } +} diff --git a/modules/records/files/config.json b/modules/records/files/config.json new file mode 100644 index 00000000..0967ef42 --- /dev/null +++ b/modules/records/files/config.json @@ -0,0 +1 @@ +{} diff --git a/modules/records/files/origin b/modules/records/files/origin new file mode 100644 index 00000000..8c829dc9 --- /dev/null +++ b/modules/records/files/origin @@ -0,0 +1 @@ +${bound:git:scheme}://${bound:git:at}:${bound:git:port} diff --git a/modules/records/module.json b/modules/records/module.json index 1ba903da..9fcf22f5 100644 --- a/modules/records/module.json +++ b/modules/records/module.json @@ -29,6 +29,15 @@ } ] }, + "config": { + "repository": { + "type": "string", + "description": "The repository the records module keeps a checkout of and serves the record from, as owner/name on the mesh's forge; the operator's, with no default (novox/hq ADR 0112).", + "pattern": "[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+", + "trusted": true, + "required": true + } + }, "resources": [ { "id": "mesh-state", @@ -47,7 +56,8 @@ "type": "file", "path": "${dir:mesh-state}/config.json", "mode": "0600", - "content": "{}\n", + "template": "files/config.json", + "takes-config": ["repository"], "merge": "json" }, { @@ -55,7 +65,7 @@ "type": "file", "path": "${dir:mesh-state}/origin", "mode": "0600", - "content": "${bound:git:scheme}://${bound:git:at}:${bound:git:port}\n" + "template": "files/origin" }, { "id": "git", @@ -80,6 +90,11 @@ "MESH_RECORDS_ORIGIN_FILE": "${dir:mesh-state}/origin", "MESH_RECORDS_DIR": "${dir:checkout}" } + }, + { + "name": "files", + "kind": "archive", + "from": "files" } ] } diff --git a/modules/redis/files/redis.conf b/modules/redis/files/redis.conf new file mode 100644 index 00000000..ac27331f --- /dev/null +++ b/modules/redis/files/redis.conf @@ -0,0 +1,3 @@ +requirepass ${secret:secret} +appendonly yes +dir /data diff --git a/modules/redis/module.json b/modules/redis/module.json index bdb344a4..fc67a289 100644 --- a/modules/redis/module.json +++ b/modules/redis/module.json @@ -86,7 +86,7 @@ "type": "file", "path": "${dir:state}/redis.conf", "mode": "0600", - "content": "requirepass ${secret:secret}\nappendonly yes\ndir /data\n", + "template": "files/redis.conf", "owner": "999:1000" }, { @@ -140,6 +140,11 @@ "MESH_PROVISION_REDIS": "127.0.0.1:${port:6379}", "MESH_PROVISION_PASSWORD_FILE": "${dir:state}/default.secret" } + }, + { + "name": "files", + "kind": "archive", + "from": "files" } ] } diff --git a/modules/restic/cmd/restic-backups/retention_test.go b/modules/restic/cmd/restic-backups/retention_test.go index b06ad26e..2e0ce490 100644 --- a/modules/restic/cmd/restic-backups/retention_test.go +++ b/modules/restic/cmd/restic-backups/retention_test.go @@ -159,25 +159,28 @@ func TestARetentionThatIsNotACountThinsNothingAndSaysWhich(t *testing.T) { } } -// The manifest declares the three preferences with today's values as their defaults, and its file, +// The manifest declares the retention's config values with today's values as their defaults, and its file, // filled with those defaults, reads as today's retention: nothing changes on a machine that sets none. func TestTheManifestsDefaultsAreTodaysRetention(t *testing.T) { raw, err := os.ReadFile("../../module.json") must(t, err) var m struct { - Settings map[string]struct { - Kind string `json:"kind"` - Default any `json:"default"` - Why string `json:"why"` - } `json:"settings"` + Config map[string]struct { + Type string `json:"type"` + Description string `json:"description"` + Minimum *float64 `json:"minimum"` + Maximum *float64 `json:"maximum"` + Default any `json:"default"` + } `json:"config"` Resources []map[string]any `json:"resources"` } must(t, json.Unmarshal(raw, &m)) want := map[string]float64{"keep-daily": 14, "keep-weekly": 8, "keep-monthly": 6, "keep-hand-taken-days": 7} for k, v := range want { - s, ok := m.Settings[k] - if !ok || s.Kind != "preference" || s.Default != v || s.Why == "" { - t.Errorf("setting %s: %+v, want a preference defaulting to %v with why", k, s, v) + s, ok := m.Config[k] + if !ok || s.Type != "number" || s.Default != v || s.Description == "" || s.Minimum == nil || *s.Minimum != 1 || + s.Maximum == nil || *s.Maximum != 9999 { + t.Errorf("config value %s: %+v, want a number from 1 to 9999 defaulting to %v with a description", k, s, v) } } content := "" @@ -186,15 +189,15 @@ func TestTheManifestsDefaultsAreTodaysRetention(t *testing.T) { if r["path"] != "${dir:state}/"+RetentionFile { t.Errorf("the retention file is placed at %v, where this program does not read it", r["path"]) } - content, _ = r["content"].(string) + content = templateText(t, r) } } if content == "" { t.Fatal("no retention resource in the manifest") } - filled := regexp.MustCompile(`\$\{setting:([a-z-]+)\}`).ReplaceAllStringFunc(content, func(p string) string { - k := p[len("${setting:") : len(p)-1] - return fmt.Sprint(m.Settings[k].Default) + filled := regexp.MustCompile(`\$\{config:([a-z-]+)\}`).ReplaceAllStringFunc(content, func(p string) string { + k := p[len("${config:") : len(p)-1] + return fmt.Sprint(m.Config[k].Default) }) got, err := ParseRetention(filled) must(t, err) diff --git a/modules/restic/cmd/restic-backups/templates_test.go b/modules/restic/cmd/restic-backups/templates_test.go new file mode 100644 index 00000000..7d0718b5 --- /dev/null +++ b/modules/restic/cmd/restic-backups/templates_test.go @@ -0,0 +1,64 @@ +package main + +import ( + "encoding/json" + "os" + "path/filepath" + "testing" +) + +// The module's directory, where its manifest and its files/ are. +const moduleDir = "../.." + +// templateText is the text a file resource places on the machine: its template's, in the module's files/ +// (novox/hq ADR 0349 §2). A resource that names no template, or a template that is not there, fails the test. +func templateText(t *testing.T, r map[string]any) string { + t.Helper() + name, _ := r["template"].(string) + if name == "" { + t.Fatalf("resource %v names no template", r["id"]) + } + raw, err := os.ReadFile(filepath.Join(moduleDir, filepath.FromSlash(name))) + if err != nil { + t.Fatalf("resource %v names %s: %v", r["id"], name, err) + } + return string(raw) +} + +// No file resource carries its text inline: each names its template in files/, and the build packs files/ +// into the one archive the controller reads the templates back from (novox/hq ADR 0349 §2). +func TestEveryFileTakesItsTextFromATemplate(t *testing.T) { + raw, err := os.ReadFile(filepath.Join(moduleDir, "module.json")) + if err != nil { + t.Fatal(err) + } + var m struct { + Resources []map[string]any `json:"resources"` + Build struct { + Artifacts []map[string]any `json:"artifacts"` + } `json:"build"` + } + if err := json.Unmarshal(raw, &m); err != nil { + t.Fatal(err) + } + for _, r := range m.Resources { + if _, inline := r["content"]; inline { + t.Errorf("resource %v keeps its text inline: keep it in files/ and name it as the template", r["id"]) + } + if r["type"] == "file" { + templateText(t, r) + } + } + packs := 0 + for _, a := range m.Build.Artifacts { + if a["kind"] == "archive" && a["from"] == "files" { + packs++ + if a["name"] != "files" { + t.Errorf("the templates archive is named %v, not files", a["name"]) + } + } + } + if packs != 1 { + t.Errorf("the build packs files/ into %d archives, not one", packs) + } +} diff --git a/modules/restic/files/backups.conf b/modules/restic/files/backups.conf new file mode 100644 index 00000000..2f23f90f --- /dev/null +++ b/modules/restic/files/backups.conf @@ -0,0 +1,2 @@ +# What the modules on this machine back up, composed by the mesh (novox/hq to-be 43). Do not edit. +${backup:lines} \ No newline at end of file diff --git a/modules/restic/files/data.conf b/modules/restic/files/data.conf new file mode 100644 index 00000000..f19e8499 --- /dev/null +++ b/modules/restic/files/data.conf @@ -0,0 +1,2 @@ +# The data the modules on this machine declare, composed by the mesh (novox/hq ADR 0233). Do not edit. +${backup:data} \ No newline at end of file diff --git a/modules/restic/files/retention.conf b/modules/restic/files/retention.conf new file mode 100644 index 00000000..cd7954c6 --- /dev/null +++ b/modules/restic/files/retention.conf @@ -0,0 +1,6 @@ +# How many restore points the night keeps, written by the mesh from this machine's settings keep-daily, +# keep-weekly, keep-monthly (novox/hq issue 416) and keep-hand-taken-days (novox/hq issue 453). Replaced on every push; change the settings, never this file. +daily=${config:keep-daily} +weekly=${config:keep-weekly} +monthly=${config:keep-monthly} +hand=${config:keep-hand-taken-days} diff --git a/modules/restic/module.json b/modules/restic/module.json index 604ce0c7..e7089278 100644 --- a/modules/restic/module.json +++ b/modules/restic/module.json @@ -1,6 +1,7 @@ { "module": "restic", "version": "1", + "moved": true, "claims": [ { "name": "node-backup", @@ -37,28 +38,40 @@ } ] }, - "settings": { - "keep-daily": { - "kind": "preference", - "default": 14, - "why": "two weeks of nightly restore points undo a mistake noticed within a fortnight; a count from 1 to 9999, read by the night, which thins nothing while it does not read (novox/hq issue 416); lowering it deletes the restore points beyond the new count at the next night, and that cannot be undone" + "config": { + "keep-daily": { + "type": "number", + "description": "How many nightly restore points are kept, a count from 1 to 9999: two weeks undo a mistake noticed within a fortnight. Read by the night, which thins nothing while it does not read (novox/hq issue 416); lowering it deletes the restore points beyond the new count at the next night, and that cannot be undone.", + "minimum": 1, + "maximum": 9999, + "trusted": true, + "default": 14 + }, + "keep-weekly": { + "type": "number", + "description": "How many weekly restore points are kept behind the dailies, a count from 1 to 9999, refused by the night like keep-daily; lowering it deletes the restore points beyond the new count at the next night, and that cannot be undone.", + "minimum": 1, + "maximum": 9999, + "trusted": true, + "default": 8 + }, + "keep-monthly": { + "type": "number", + "description": "How many monthly restore points are kept behind the weeklies, a count from 1 to 9999, refused by the night like keep-daily; lowering it deletes the restore points beyond the new count at the next night, and that cannot be undone.", + "minimum": 1, + "maximum": 9999, + "trusted": true, + "default": 6 + }, + "keep-hand-taken-days": { + "type": "number", + "description": "How many days a restore point taken by hand (node-backup.now, or the mesh's before a module moves) is kept at least, a count from 1 to 9999; it is kept until a later good night of its module exists, and this many days in any case, so the next thinning never takes the one taken against a risky change (novox/hq issue 453). Lowering it lets go of the older ones taken by hand at the next thinning, and that cannot be undone.", + "minimum": 1, + "maximum": 9999, + "trusted": true, + "default": 7 + } }, - "keep-weekly": { - "kind": "preference", - "default": 8, - "why": "two months of weekly restore points behind the dailies; a count from 1 to 9999, refused by the night like keep-daily; lowering it deletes the restore points beyond the new count at the next night, and that cannot be undone" - }, - "keep-monthly": { - "kind": "preference", - "default": 6, - "why": "half a year of monthly restore points behind the weeklies; a count from 1 to 9999, refused by the night like keep-daily; lowering it deletes the restore points beyond the new count at the next night, and that cannot be undone" - }, - "keep-hand-taken-days": { - "kind": "preference", - "default": 7, - "why": "a restore point taken by hand (node-backup.now, or the mesh's before a module moves) is kept until a later good night of its module exists, and this many days in any case, so the next thinning never takes the one taken against a risky change (novox/hq issue 453); a count from 1 to 9999, refused by the night like keep-daily; lowering it lets go of the restore points taken by hand that are older than the new count, past a later night, at the next thinning, and that cannot be undone" - } - }, "resources": [ { "id": "state", @@ -76,14 +89,14 @@ "type": "file", "path": "${dir:state}/backups.conf", "mode": "0600", - "content": "# What the modules on this machine back up, composed by the mesh (novox/hq to-be 43). Do not edit.\n${contribution:node-backup:backup}" + "template": "files/backups.conf" }, { "id": "data-declared", "type": "file", "path": "${dir:state}/data.conf", "mode": "0600", - "content": "# The data the modules on this machine declare, composed by the mesh (novox/hq ADR 0233). Do not edit.\n${contribution:node-backup:data}" + "template": "files/data.conf" }, { "id": "retention", @@ -91,7 +104,7 @@ "path": "${dir:state}/retention.conf", "mode": "0644", "trusted": true, - "content": "# How many restore points the night keeps, written by the mesh from this machine's settings keep-daily,\n# keep-weekly, keep-monthly (novox/hq issue 416) and keep-hand-taken-days (novox/hq issue 453). Replaced on every push; change the settings, never this file.\ndaily=${setting:keep-daily}\nweekly=${setting:keep-weekly}\nmonthly=${setting:keep-monthly}\nhand=${setting:keep-hand-taken-days}\n" + "template": "files/retention.conf" }, { "id": "tool", @@ -170,6 +183,11 @@ "MESH_BACKUP_STATE": "${dir:state}", "MESH_BACKUP_DATA": "${dir:state}/data.conf" } + }, + { + "name": "files", + "kind": "archive", + "from": "files" } ] } diff --git a/modules/rofi/cmd/rofi-tools/main.go b/modules/rofi/cmd/rofi-tools/main.go index 3137e9e7..a79d2980 100644 --- a/modules/rofi/cmd/rofi-tools/main.go +++ b/modules/rofi/cmd/rofi-tools/main.go @@ -48,7 +48,7 @@ func tools() []stdio.Tool { Description: fmt.Sprintf("Ask the operator for a value in a prompt that shows nothing of what is typed, "+ "and answer it sealed to the key seal_to names — never in the clear (novox/hq ADR 0259 §10): how the "+ "mesh takes a module's own secret at the desk. The prompt is written here from the names, and says the "+ - "mesh's controller asks: the bus lets nobody else ask it. Answers sealed, or cancelled when the prompt was "+ + "mesh's controller or vault asks: the bus lets nobody else ask it. Answers sealed, or cancelled when the prompt was "+ "dismissed, answered empty or not answered within timeout_seconds (default and at most %d).", menuMost), Input: map[string]any{ "type": "object", @@ -62,7 +62,7 @@ func tools() []stdio.Tool { "required": []string{"module", "secret", "node", "seal_to"}, }, Run: func(args map[string]any) (any, error) { - q, err := secretQuestion(args) + q, err := secretQuestion(stdio.CallerOf(args), args) if err != nil { return nil, err } diff --git a/modules/rofi/cmd/rofi-tools/manifest_helpers_test.go b/modules/rofi/cmd/rofi-tools/manifest_helpers_test.go index 441415f9..dff8ee96 100644 --- a/modules/rofi/cmd/rofi-tools/manifest_helpers_test.go +++ b/modules/rofi/cmd/rofi-tools/manifest_helpers_test.go @@ -13,6 +13,7 @@ import ( type manifest struct { Module string `json:"module"` + Moved bool `json:"moved"` Version string `json:"version"` Capabilities []string `json:"capabilities"` Requires []string `json:"requires"` @@ -23,7 +24,8 @@ type manifest struct { Shell []shellCode `json:"shell"` Resources []map[string]any `json:"resources"` // Lines for other modules' seats (novox/hq ADR 0212): the window manager's, here. - Contributions []contribution `json:"contributions"` + Items []item `json:"items"` + Contributions []any `json:"contributions"` Build struct { Artifacts []map[string]any `json:"artifacts"` } `json:"build"` @@ -85,26 +87,66 @@ func (m manifest) packages() (present, absent []string) { return present, absent } -// sameAsSource checks that a file resource's content is byte for byte the module's source file, so -// the readable file in the repository is what the machine gets. -func (m manifest) sameAsSource(t *testing.T, id, source string) { +// fromTemplate checks that a file resource takes its text from its template file in the module's +// repository (novox/hq ADR 0349 §2): it names the template, carries no text of its own beside it, and the +// template is there. The readable file in the repository is the one place the machine's text is written. +func (m manifest) fromTemplate(t *testing.T, id, template string) { t.Helper() - want, err := os.ReadFile(filepath.Join("..", "..", source)) - if err != nil { - t.Fatal(err) - } r := m.resource(t, id) if r["type"] != "file" { t.Fatalf("%s is a %v, not a file", id, r["type"]) } - if got, _ := r["content"].(string); got != string(want) { - t.Fatalf("resource %s's content is not %s: edit the source and copy it into module.json", id, source) + if r["template"] != template { + t.Fatalf("resource %s does not take its text from %s: template %v", id, template, r["template"]) + } + if _, inline := r["content"]; inline { + t.Fatalf("resource %s keeps its text inline beside its template: a file's text is in one place", id) + } + if _, err := os.Stat(filepath.Join("..", "..", template)); err != nil { + t.Fatalf("resource %s names %s: %v", id, template, err) } if r["owner"] != "${machine:account}" && !strings.HasPrefix(r["path"].(string), "/etc/") { t.Fatalf("%s under the home is the account's", id) } } +// everyFileIsATemplate checks that no file resource carries its text inline, and that the build packs +// files/ into the one archive the controller reads the templates back from (novox/hq ADR 0349 §2). +func (m manifest) everyFileIsATemplate(t *testing.T) { + t.Helper() + for _, r := range m.Resources { + if _, inline := r["content"]; inline { + t.Errorf("resource %v keeps its text inline: keep it in files/ and name it as the template", r["id"]) + } + } + packs := 0 + for _, a := range m.Build.Artifacts { + if a["kind"] == "archive" && a["from"] == "files" { + packs++ + if a["name"] != "files" { + t.Errorf("the templates archive is named %v, not files", a["name"]) + } + } + } + if packs != 1 { + t.Errorf("the build packs files/ into %d archives, not one", packs) + } +} + +// text is the text a file resource places on the machine: its template file's. +func (m manifest) text(t *testing.T, id string) string { + t.Helper() + name, _ := m.resource(t, id)["template"].(string) + if name == "" { + t.Fatalf("resource %s names no template", id) + } + raw, err := os.ReadFile(filepath.Join("..", "..", name)) + if err != nil { + t.Fatal(err) + } + return string(raw) +} + // checkTheToolsAgree checks that the manifest lists the module's own tools exactly, that the bundle // serves each seat verb the claims promise as ., and that the Go bundle is declared. func checkTheToolsAgree(t *testing.T, m manifest) { @@ -176,31 +218,49 @@ func checkNoSecretsOrInstallationNames(t *testing.T) { } } -type contribution struct { - Seat string `json:"seat"` - Kind string `json:"kind"` - Content string `json:"content"` -} +// displaySession is the collection the module's lines for the window manager are an item of. +const displaySession = "contribution:node-display-session:config" -// i3Lines is what the module contributes to the window manager. +// i3Lines is what the module contributes to the window manager: the text of its item of the collection +// contribution:node-display-session:config (design 57 L3, issue 848). func (m manifest) i3Lines() string { var out string - for _, c := range m.Contributions { - if c.Seat == "node-display-session" && c.Kind == "config" { - out += c.Content + for _, it := range m.Items { + if it.Collection == displaySession { + text, err := os.ReadFile(filepath.Join("..", "..", it.Template)) + if err != nil { + return "" + } + out += string(text) } } return out } -// i3LinesAreSource checks the window-manager contribution is the source file it is written from. +// i3LinesAreSource checks the window-manager lines are one item whose template is the source file, and that no +// old-form contribution to node-display-session is left. func (m manifest) i3LinesAreSource(t *testing.T, source string) { t.Helper() - want, err := os.ReadFile(filepath.Join("..", "..", source)) - if err != nil { - t.Fatal(err) + if len(m.Contributions) != 0 { + t.Fatalf("the module still holds contributions: the lines are an item of %s", displaySession) } - if m.i3Lines() != string(want) { - t.Fatalf("the contribution to node-display-session is not %s: edit the source and copy it into module.json", source) + var templates []string + for _, it := range m.Items { + if it.Collection == displaySession { + templates = append(templates, it.Template) + } + } + if len(templates) != 1 || templates[0] != source { + t.Fatalf("the items of %s are %v, want the one template %s", displaySession, templates, source) + } + if m.i3Lines() == "" { + t.Fatalf("the template %s holds no line", source) } } + +type item struct { + Collection string `json:"collection"` + Name string `json:"name"` + Attributes map[string]string `json:"attributes"` + Template string `json:"template"` +} diff --git a/modules/rofi/cmd/rofi-tools/manifest_test.go b/modules/rofi/cmd/rofi-tools/manifest_test.go index bc5eff4f..36f12ec3 100644 --- a/modules/rofi/cmd/rofi-tools/manifest_test.go +++ b/modules/rofi/cmd/rofi-tools/manifest_test.go @@ -37,7 +37,7 @@ func TestItOwnsItsFilesAsWrittenInTheModule(t *testing.T) { "launch": "files/bin/rofi-launch", "powermenu": "files/bin/rofi-powermenu", } { - m.sameAsSource(t, id, source) + m.fromTemplate(t, id, source) } for _, id := range []string{"dmenu", "launch", "powermenu"} { if m.resource(t, id)["mode"] != "0755" { @@ -51,25 +51,25 @@ func TestItOwnsItsFilesAsWrittenInTheModule(t *testing.T) { func TestTheThemesAreTheMeshsFacesAndTheConfigurationNamesOne(t *testing.T) { m := readManifest(t) - conf := m.resource(t, "configuration")["content"].(string) + conf := m.text(t, "configuration") if !strings.Contains(conf, `@theme "mesh"`) || strings.Contains(conf, "@import") { t.Fatal("the configuration names the theme mesh and imports nothing") } for _, id := range []string{"theme", "theme-powermenu"} { - c := m.resource(t, id)["content"].(string) + c := m.text(t, id) if !strings.Contains(c, `"JetBrainsMono Nerd Font`) || strings.Contains(c, "@import") || strings.Contains(c, "Hack") || strings.Contains(c, "Iosevka") { t.Errorf("%s: the monospace face, and no imported file", id) } } - if !strings.Contains(m.resource(t, "theme")["content"].(string), `"Inter 11"`) { + if !strings.Contains(m.text(t, "theme"), `"Inter 11"`) { t.Error("the theme's prose is in the interface face") } } func TestThePowerMenuNamesNoWindowManagerAndNoLocker(t *testing.T) { m := readManifest(t) - menu := m.resource(t, "powermenu")["content"].(string) + menu := m.text(t, "powermenu") for _, never := range []string{"i3-msg", "i3lock", "betterlockscreen", "openbox", "bspc", "qdbus", "mpc"} { if strings.Contains(menu, never) { t.Errorf("the power menu names %s", never) @@ -109,3 +109,8 @@ func TestTheToolsAgreeWithTheManifest(t *testing.T) { checkTheToolsAgree(t, m) checkNoSecretsOrInstallationNames(t) } + +// No file resource carries its text inline (novox/hq ADR 0349 §2). +func TestEveryFileTakesItsTextFromATemplate(t *testing.T) { + readManifest(t).everyFileIsATemplate(t) +} diff --git a/modules/rofi/cmd/rofi-tools/secret.go b/modules/rofi/cmd/rofi-tools/secret.go index ce95e678..80377868 100644 --- a/modules/rofi/cmd/rofi-tools/secret.go +++ b/modules/rofi/cmd/rofi-tools/secret.go @@ -14,12 +14,13 @@ import ( // node-launcher's verb `secret` (novox/hq ADR 0259 §10): a prompt that does not show what is typed, whose // answer is sealed to the key the asker gives before it leaves this process — never in the clear on the -// bus, in the runtime's log or in any call's record. The controller asks it to take a module's own secret, -// such as a bot token, from the operator at their desk. +// bus, in the runtime's log or in any call's record. The vault asks it, in its own name, to take a module's own +// secret, such as a bot token, from the operator at their desk; the controller asks it for a module still filled +// the old way, until R1 (issue 682). // // **What it does not do**: on an X11 desk any program of the operator's account can read the keys as they // are typed, and a program that asks this verb with a key of its own is answered with what the operator -// typed into a prompt they did not ask for. The bus lets only the controller ask this verb, and the prompt, +// typed into a prompt they did not ask for. The bus lets only the controller and the vault ask this verb, and the prompt, // written here from names, says so and for what, so the operator types only into one they started with give // (ADR 0234's accepted residual risk is what remains: a program drawing a window of its own). @@ -46,6 +47,10 @@ var aName = regexp.MustCompile(`^[a-z0-9][a-z0-9-]{0,62}$`) // break, no markup, no punctuation a sentence is written with — so the prompt's words stay this holder's. var aCaller = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._/@ -]{0,79}$`) +// theVault says the caller the runtime proved is the vault's own account (`.mesh-vault`, issue 723): the one +// account beside the controller the bus lets ask this verb, and the only one that asks it in its own name. +func theVault(caller string) bool { return strings.HasSuffix(caller, ".mesh-vault") } + // escapeMarkup is `what` shown as text, never markup (novox/hq issue 789): the window's message is read with // pango markup on, so a catalogue's words reach it with the characters pango gives meaning to turned to their // entities first — never interpreted, whatever a module's manifest holds. @@ -54,10 +59,10 @@ var escapeMarkupReplacer = strings.NewReplacer("&", "&", "<", "<", ">", " func escapeMarkup(s string) string { return escapeMarkupReplacer.Replace(s) } // secretQuestion reads the call. The prompt is this holder's words, written from the module, the secret and the -// machine (the review of 2026-10-09, M4): it says the mesh's controller asks, which the bus makes true — it -// denies these subjects to every principal but the controller — and a caller's own words are refused, so -// nobody puts a prompt of its choosing before the operator. -func secretQuestion(args map[string]any) (SecretQuestion, error) { +// machine (the review of 2026-10-09, M4): it says the mesh's controller or the mesh's vault asks, which the bus +// makes true — it denies these subjects to every principal but those two, and lets the vault alone call them in +// its name — and a caller's own words are refused, so nobody puts a prompt of its choosing before the operator. +func secretQuestion(caller string, args map[string]any) (SecretQuestion, error) { var q SecretQuestion for _, free := range []string{"prompt", "message"} { if _, there := args[free]; there { @@ -103,7 +108,11 @@ func secretQuestion(args map[string]any) (SecretQuestion, error) { what = v } q.Prompt = names["secret"] + " for " + names["module"] - q.Message = fmt.Sprintf("The mesh's controller asks for %s, the own secret of %s on %s.", names["secret"], + asker := "controller" + if theVault(caller) { + asker = "vault" + } + q.Message = fmt.Sprintf("The mesh's %s asks for %s, the own secret of %s on %s.", asker, names["secret"], names["module"], names["node"]) if what != "" { q.Message += "\nFor: " + escapeMarkup(what) diff --git a/modules/rofi/cmd/rofi-tools/secret_test.go b/modules/rofi/cmd/rofi-tools/secret_test.go index d9db64c6..454a97fc 100644 --- a/modules/rofi/cmd/rofi-tools/secret_test.go +++ b/modules/rofi/cmd/rofi-tools/secret_test.go @@ -30,7 +30,7 @@ func TestWhatIsTypedIsAnsweredSealedToTheAskersKeyAndNeverInTheClear(t *testing. bin := fakeBinaries(t, map[string]string{"rofi": `echo "$*" > "$LOG"; echo "` + typedValue + `"`}) t.Setenv("LOG", filepath.Join(bin, "log")) key, pub, priv := aKey(t) - q, err := secretQuestion(map[string]any{"module": "telegram", "secret": "telegram-token", "node": "anchor", "seal_to": key}) + q, err := secretQuestion("unproven", map[string]any{"module": "telegram", "secret": "telegram-token", "node": "anchor", "seal_to": key}) if err != nil { t.Fatal(err) } @@ -62,7 +62,7 @@ func TestWhatIsTypedIsAnsweredSealedToTheAskersKeyAndNeverInTheClear(t *testing. func TestADismissedEmptyOrLatePromptIsACancelAndAFailureNeverSaysTheValue(t *testing.T) { withSession(t) key, _, _ := aKey(t) - q, _ := secretQuestion(map[string]any{"module": "telegram", "secret": "telegram-token", "node": "anchor", "seal_to": key, "timeout_seconds": 1}) + q, _ := secretQuestion("unproven", map[string]any{"module": "telegram", "secret": "telegram-token", "node": "anchor", "seal_to": key, "timeout_seconds": 1}) for script, timedOut := range map[string]bool{`exit 1`: false, `echo ""`: false, `sleep 30`: true} { fakeBinaries(t, map[string]string{"rofi": script}) got, err := Secret(q) @@ -92,7 +92,7 @@ func TestASecretIsAskedOnlyWithAKeyToSealItTo(t *testing.T) { named(map[string]any{"seal_to": key, "timeout_seconds": 60}), {"seal_to": key}, } { - if _, err := secretQuestion(args); err == nil { + if _, err := secretQuestion("unproven", args); err == nil { t.Errorf("%v was taken", args) } } @@ -104,7 +104,7 @@ func TestASecretIsAskedOnlyWithAKeyToSealItTo(t *testing.T) { func TestThePromptIsWrittenHereFromNamesAndSaysTheControllerAsks(t *testing.T) { key := base64.StdEncoding.EncodeToString(make([]byte, 32)) base := map[string]any{"module": "telegram", "secret": "telegram-token", "node": "anchor", "seal_to": key} - q, err := secretQuestion(base) + q, err := secretQuestion("unproven", base) if err != nil { t.Fatal(err) } @@ -112,18 +112,26 @@ func TestThePromptIsWrittenHereFromNamesAndSaysTheControllerAsks(t *testing.T) { strings.Contains(q.Message, "asked for by") { t.Errorf("prompt %q, message %q", q.Prompt, q.Message) } + // Asked by the vault's own account, as the runtime proved it (issue 723): the prompt says the vault asks. + if q, err := secretQuestion("novox.mesh-vault", base); err != nil || + !strings.HasPrefix(q.Message, "The mesh's vault asks for telegram-token") { + t.Errorf("asked by the vault: %q %v", q.Message, err) + } + if q, _ := secretQuestion("novox.mesh-vault-not", base); !strings.HasPrefix(q.Message, "The mesh's controller") { + t.Errorf("a caller that is not the vault's account is said to be the vault: %q", q.Message) + } // And who asked the controller, when it says so (novox/hq ADR 0277): a caller's name, in a name's characters. withAsker := map[string]any{} for k, v := range base { withAsker[k] = v } withAsker["asked_by"] = "g14/claude-code" - q, err = secretQuestion(withAsker) + q, err = secretQuestion("unproven", withAsker) if err != nil || !strings.Contains(q.Message, "It was asked for by g14/claude-code.") { t.Errorf("asked_by: %q %v", q.Message, err) } withAsker["asked_by"] = "jochen at a shell on novox" - if q, err = secretQuestion(withAsker); err != nil || !strings.Contains(q.Message, "It was asked for by jochen at a shell on novox.") { + if q, err = secretQuestion("unproven", withAsker); err != nil || !strings.Contains(q.Message, "It was asked for by jochen at a shell on novox.") { t.Errorf("asked_by at a shell: %q %v", q.Message, err) } for key, value := range map[string]any{ @@ -139,13 +147,13 @@ func TestThePromptIsWrittenHereFromNamesAndSaysTheControllerAsks(t *testing.T) { args[k] = v } args[key] = value - if _, err := secretQuestion(args); err == nil { + if _, err := secretQuestion("unproven", args); err == nil { t.Errorf("%s=%q was taken", key, value) } } for _, bad := range []string{"", "a\nb", "ac", "a,b", strings.Repeat("a", 81), "a b", "a:b", "a'b"} { withAsker["asked_by"] = bad - if _, err := secretQuestion(withAsker); err == nil { + if _, err := secretQuestion("unproven", withAsker); err == nil { t.Errorf("asked_by=%q was taken", bad) } } @@ -157,7 +165,7 @@ func TestThePromptIsWrittenHereFromNamesAndSaysTheControllerAsks(t *testing.T) { func TestWhatShowsAsItsOwnLineUnderTheNamesWhenDeclaredAndNothingWhenNot(t *testing.T) { key := base64.StdEncoding.EncodeToString(make([]byte, 32)) base := map[string]any{"module": "mesh-mail", "secret": "mail-credential-gmail", "node": "anchor", "seal_to": key} - q, err := secretQuestion(base) + q, err := secretQuestion("unproven", base) if err != nil { t.Fatal(err) } @@ -170,7 +178,7 @@ func TestWhatShowsAsItsOwnLineUnderTheNamesWhenDeclaredAndNothingWhenNot(t *test withWhat[k] = v } withWhat["what"] = "the password of the mail account gmail" - q, err = secretQuestion(withWhat) + q, err = secretQuestion("unproven", withWhat) if err != nil { t.Fatal(err) } @@ -189,7 +197,7 @@ func TestWhatIsShownAsTextNeverMarkupAndIsOneLine(t *testing.T) { key := base64.StdEncoding.EncodeToString(make([]byte, 32)) base := map[string]any{"module": "mesh-mail", "secret": "mail-credential-gmail", "node": "anchor", "seal_to": key, "what": "the password of the mail & account"} - q, err := secretQuestion(base) + q, err := secretQuestion("unproven", base) if err != nil { t.Fatal(err) } @@ -203,7 +211,7 @@ func TestWhatIsShownAsTextNeverMarkupAndIsOneLine(t *testing.T) { multi[k] = v } multi["what"] = bad - if _, err := secretQuestion(multi); err == nil { + if _, err := secretQuestion("unproven", multi); err == nil { t.Errorf("what=%q holding more than one line was taken", bad) } } diff --git a/modules/rofi/go.mod b/modules/rofi/go.mod index ca15f8ce..9b8d8ad1 100644 --- a/modules/rofi/go.mod +++ b/modules/rofi/go.mod @@ -2,7 +2,7 @@ module rofi go 1.22 -require git.novox.be/novox/mesh-sdk/go v0.1.7 +require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261013235212-c8b07242a009 require ( golang.org/x/crypto v0.31.0 diff --git a/modules/rofi/go.sum b/modules/rofi/go.sum index dff94e3d..47386f64 100644 --- a/modules/rofi/go.sum +++ b/modules/rofi/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.7 h1:C0sTQmtTiyYH7bnqZb7PusXnqA37gKuT7Nqjn9gG47w= -git.novox.be/novox/mesh-sdk/go v0.1.7/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261013235212-c8b07242a009 h1:I9rm8HhYRJ0+mzQQz+VAH0kgv2YJWkvctEc67Yd0PX8= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261013235212-c8b07242a009/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY= golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U= golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= diff --git a/modules/rofi/module.json b/modules/rofi/module.json index d603a199..1cc9b2ff 100644 --- a/modules/rofi/module.json +++ b/modules/rofi/module.json @@ -1,6 +1,7 @@ { "module": "rofi", "version": "1", + "moved": true, "capabilities": [ "package-manager" ], @@ -48,7 +49,7 @@ "path": "${machine:account-home}/.config/rofi/config.rasi", "owner": "${machine:account}", "mode": "0644", - "content": "/* rofi, the launcher (module rofi, novox/hq ADR 0208). Owned by the mesh: this file is replaced at\n * every push. Adopted from the two workstations' file of 2026-10-04: the settings that differ from\n * rofi's defaults, nothing else. Its look is the theme \"mesh\", in themes/ beside this file. */\n\nconfiguration {\n\tmodi: \"drun,run,window,filebrowser\";\n\tshow-icons: true;\n\tcase-sensitive: false;\n\tnormalize-match: true;\n\tmatching: \"normal\";\n\ttokenize: true;\n\tcycle: true;\n\tsteal-focus: false;\n\tclick-to-exit: true;\n\tsort: false;\n\n\t/* Applications: the user's and the system's desktop entries, read fresh each time so a\n\t * program installed a minute ago is there. */\n\tdrun-match-fields: \"name,generic,exec,categories,keywords\";\n\tdrun-display-format: \"{name} [({generic})]\";\n\tdrun-show-actions: false;\n\tdrun-url-launcher: \"xdg-open\";\n\tdrun-use-desktop-cache: false;\n\tdrun-reload-desktop-cache: false;\n\trun,drun {\n\t\tfallback-icon: \"application-x-addon\";\n\t}\n\n\t/* A command run from `run` opens in the node's terminal when it asks for one. */\n\tterminal: \"rofi-sensible-terminal\";\n\trun-shell-command: \"{terminal} -e {cmd}\";\n\tssh-command: \"{terminal} -e {ssh-client} {host} [-p {port}]\";\n\tparse-hosts: true;\n\tparse-known-hosts: true;\n\n\twindow-match-fields: \"title,class,role,name,desktop\";\n\twindow-format: \"{w} - {c} - {t:0}\";\n\n\tdisable-history: false;\n\tmax-history-size: 25;\n\n\tfilebrowser {\n\t\tdirectories-first: true;\n\t\tsorting-method: \"name\";\n\t}\n\n\tdisplay-window: \"Windows\";\n\tdisplay-run: \"Run\";\n\tdisplay-ssh: \"SSH\";\n\tdisplay-drun: \"Apps\";\n\tdisplay-filebrowser: \"Files\";\n}\n\n@theme \"mesh\"\n" + "template": "files/config.rasi" }, { "id": "theme", @@ -56,7 +57,7 @@ "path": "${machine:account-home}/.config/rofi/themes/mesh.rasi", "owner": "${machine:account}", "mode": "0644", - "content": "/**\n * The theme \"mesh\" (module rofi, novox/hq ADR 0208): every rofi window unless a caller names\n * another. Owned by the mesh; replaced at every push. Adopted from the workstations' theme of\n * 2026-10-04 (after adi1090x's launcher type-4, style-1), its colour file inlined so the theme is\n * one file, and its face the monospace one every desktop module names.\n **/\n\n/*****----- Global Properties -----*****/\n* {\n background: #000000FF;\n background-alt: #282B31FF;\n foreground: #FFFFFFFF;\n selected: #DE5200FF;\n active: #DE5200FF;\n urgent: #CC0000FF;\n\n font: \"JetBrainsMono Nerd Font 11\";\n\n border-colour: var(selected);\n handle-colour: var(selected);\n background-colour: var(background);\n foreground-colour: var(foreground);\n alternate-background: var(background-alt);\n normal-background: var(background);\n normal-foreground: var(foreground);\n urgent-background: var(urgent);\n urgent-foreground: var(background);\n active-background: var(active);\n active-foreground: var(background);\n selected-normal-background: var(selected);\n selected-normal-foreground: var(background);\n selected-urgent-background: var(active);\n selected-urgent-foreground: var(background);\n selected-active-background: var(urgent);\n selected-active-foreground: var(background);\n alternate-normal-background: var(background);\n alternate-normal-foreground: var(foreground);\n alternate-urgent-background: var(urgent);\n alternate-urgent-foreground: var(background);\n alternate-active-background: var(active);\n alternate-active-foreground: var(background);\n}\n\n/*****----- Main Window -----*****/\nwindow {\n transparency: \"real\";\n location: center;\n anchor: center;\n fullscreen: false;\n width: 600px;\n x-offset: 0px;\n y-offset: 0px;\n enabled: true;\n margin: 0px;\n padding: 0px;\n border: 1px solid;\n border-radius: 0px;\n border-color: @border-colour;\n cursor: \"default\";\n background-color: @background-colour;\n}\n\n/*****----- Main Box -----*****/\nmainbox {\n enabled: true;\n spacing: 10px;\n margin: 0px;\n padding: 10px;\n border: 0px solid;\n border-radius: 0px;\n border-color: @border-colour;\n background-color: transparent;\n children: [ \"inputbar\", \"message\", \"listview\", \"mode-switcher\" ];\n}\n\n/*****----- Inputbar -----*****/\ninputbar {\n enabled: true;\n spacing: 10px;\n margin: 0px;\n padding: 10px;\n border: 0px 0px 1px 0px;\n border-radius: 0px;\n border-color: @border-colour;\n background-color: @alternate-background;\n text-color: @foreground-colour;\n children: [ \"prompt\", \"entry\" ];\n}\n\nprompt {\n enabled: true;\n background-color: inherit;\n text-color: inherit;\n}\ntextbox-prompt-colon {\n enabled: true;\n expand: false;\n str: \"::\";\n background-color: inherit;\n text-color: inherit;\n}\nentry {\n enabled: true;\n background-color: inherit;\n text-color: inherit;\n cursor: text;\n placeholder: \"Search...\";\n placeholder-color: inherit;\n}\n\n/*****----- Listview -----*****/\nlistview {\n enabled: true;\n columns: 1;\n lines: 8;\n cycle: true;\n dynamic: true;\n scrollbar: false;\n layout: vertical;\n reverse: false;\n fixed-height: true;\n fixed-columns: true;\n spacing: 0px;\n margin: 0px;\n padding: 0px;\n border: 0px solid;\n border-radius: 0px;\n border-color: @border-colour;\n background-color: transparent;\n text-color: @foreground-colour;\n cursor: \"default\";\n}\nscrollbar {\n handle-width: 5px;\n handle-color: @handle-colour;\n border-radius: 0px;\n background-color: @alternate-background;\n}\n\n/*****----- Elements -----*****/\nelement {\n enabled: true;\n spacing: 10px;\n margin: 0px;\n padding: 10px;\n border: 0px solid;\n border-radius: 0px;\n border-color: @border-colour;\n background-color: transparent;\n text-color: @foreground-colour;\n cursor: pointer;\n}\nelement normal.normal {\n background-color: var(normal-background);\n text-color: var(normal-foreground);\n}\nelement normal.urgent {\n background-color: var(urgent-background);\n text-color: var(urgent-foreground);\n}\nelement normal.active {\n background-color: var(active-background);\n text-color: var(active-foreground);\n}\nelement selected.normal {\n background-color: var(selected-normal-background);\n text-color: var(selected-normal-foreground);\n}\nelement selected.urgent {\n background-color: var(selected-urgent-background);\n text-color: var(selected-urgent-foreground);\n}\nelement selected.active {\n background-color: var(selected-active-background);\n text-color: var(selected-active-foreground);\n}\nelement alternate.normal {\n background-color: var(alternate-normal-background);\n text-color: var(alternate-normal-foreground);\n}\nelement alternate.urgent {\n background-color: var(alternate-urgent-background);\n text-color: var(alternate-urgent-foreground);\n}\nelement alternate.active {\n background-color: var(alternate-active-background);\n text-color: var(alternate-active-foreground);\n}\nelement-icon {\n background-color: transparent;\n text-color: inherit;\n size: 24px;\n cursor: inherit;\n}\nelement-text {\n background-color: transparent;\n text-color: inherit;\n highlight: inherit;\n cursor: inherit;\n vertical-align: 0.5;\n horizontal-align: 0.0;\n}\n\n/*****----- Mode Switcher -----*****/\nmode-switcher {\n enabled: true;\n spacing: 10px;\n margin: 0px;\n padding: 0px;\n border: 0px solid;\n border-radius: 0px;\n border-color: @border-colour;\n background-color: transparent;\n text-color: @foreground-colour;\n}\nbutton {\n padding: 10px;\n border: 0px solid;\n border-radius: 0px;\n border-color: @border-colour;\n background-color: @alternate-background;\n text-color: inherit;\n cursor: pointer;\n}\nbutton selected {\n background-color: var(selected-normal-background);\n text-color: var(selected-normal-foreground);\n}\n\n/*****----- Message -----*****/\nmessage {\n enabled: true;\n margin: 0px;\n padding: 0px;\n border: 0px solid;\n border-radius: 0px;\n border-color: @border-colour;\n background-color: transparent;\n text-color: @foreground-colour;\n}\ntextbox {\n font: \"Inter 11\";\n padding: 10px;\n border: 0px solid;\n border-radius: 0px;\n border-color: @border-colour;\n background-color: @alternate-background;\n text-color: @foreground-colour;\n vertical-align: 0.5;\n horizontal-align: 0.0;\n highlight: none;\n placeholder-color: @foreground-colour;\n blink: true;\n markup: true;\n}\nerror-message {\n padding: 10px;\n border: 0px solid;\n border-radius: 0px;\n border-color: @border-colour;\n background-color: @background-colour;\n text-color: @foreground-colour;\n}\n" + "template": "files/themes/mesh.rasi" }, { "id": "theme-powermenu", @@ -64,7 +65,7 @@ "path": "${machine:account-home}/.config/rofi/themes/mesh-powermenu.rasi", "owner": "${machine:account}", "mode": "0644", - "content": "/**\n * The theme \"mesh-powermenu\" (module rofi, novox/hq ADR 0208): the power menu's five buttons.\n * Owned by the mesh; replaced at every push. Adopted from the workstations' power menu theme of\n * 2026-10-04 (after adi1090x's powermenu type-2, style-1), its colours inlined.\n **/\n\n/*****----- Configuration -----*****/\nconfiguration {\n show-icons: false;\n}\n\n/*****----- Global Properties -----*****/\n* {\n background: #000000FF;\n background-alt: #282B31FF;\n foreground: #FFFFFFFF;\n selected: #DE5200FF;\n active: #DE5200FF;\n urgent: #CC0000FF;\n\n font: \"JetBrainsMono Nerd Font 11\";\n}\n\n\n/*****----- Main Window -----*****/\nwindow {\n /* properties for window widget */\n transparency: \"real\";\n location: center;\n anchor: center;\n fullscreen: false;\n width: 800px;\n x-offset: 0px;\n y-offset: 0px;\n\n /* properties for all widgets */\n enabled: true;\n margin: 0px;\n padding: 0px;\n border: 0px solid;\n border-radius: 0px;\n border-color: @selected;\n cursor: \"default\";\n background-color: @background;\n}\n\n/*****----- Main Box -----*****/\nmainbox {\n enabled: true;\n spacing: 15px;\n margin: 0px;\n padding: 30px;\n border: 0px solid;\n border-radius: 0px;\n border-color: @selected;\n background-color: transparent;\n children: [ \"inputbar\", \"listview\" ];\n}\n\n/*****----- Inputbar -----*****/\ninputbar {\n enabled: true;\n spacing: 15px;\n margin: 0px;\n padding: 0px;\n border: 0px;\n border-radius: 0px;\n border-color: @selected;\n background-color: transparent;\n text-color: @foreground;\n children: [ \"textbox-prompt-colon\", \"prompt\"];\n}\n\ndummy {\n background-color: transparent;\n}\n\ntextbox-prompt-colon {\n enabled: true;\n expand: false;\n str: \"\";\n padding: 12px 16px;\n border-radius: 0px;\n background-color: @urgent;\n text-color: @background;\n}\nprompt {\n enabled: true;\n padding: 12px;\n border-radius: 0px;\n background-color: @active;\n text-color: @background;\n}\n\n/*****----- Message -----*****/\nmessage {\n enabled: true;\n margin: 0px;\n padding: 12px;\n border: 0px solid;\n border-radius: 0px;\n border-color: @selected;\n background-color: @background-alt;\n text-color: @foreground;\n}\ntextbox {\n background-color: inherit;\n text-color: inherit;\n vertical-align: 0.5;\n horizontal-align: 0.5;\n placeholder-color: @foreground;\n blink: true;\n markup: true;\n}\nerror-message {\n padding: 12px;\n border: 0px solid;\n border-radius: 0px;\n border-color: @selected;\n background-color: @background;\n text-color: @foreground;\n}\n\n/*****----- Listview -----*****/\nlistview {\n enabled: true;\n columns: 5;\n lines: 1;\n cycle: true;\n dynamic: true;\n scrollbar: false;\n layout: vertical;\n reverse: false;\n fixed-height: true;\n fixed-columns: true;\n\n spacing: 15px;\n margin: 0px;\n padding: 0px;\n border: 0px solid;\n border-radius: 0px;\n border-color: @selected;\n background-color: transparent;\n text-color: @foreground;\n cursor: \"default\";\n}\n\n/*****----- Elements -----*****/\nelement {\n enabled: true;\n spacing: 0px;\n margin: 0px;\n padding: 40px 10px;\n border: 0px solid;\n border-radius: 0px;\n border-color: @selected;\n background-color: @background-alt;\n text-color: @foreground;\n cursor: pointer;\n}\nelement-text {\n font: \"JetBrainsMono Nerd Font Bold 32\";\n background-color: transparent;\n text-color: inherit;\n cursor: inherit;\n vertical-align: 0.5;\n horizontal-align: 0.5;\n}\nelement selected.normal {\n background-color: var(selected);\n text-color: var(background);\n}\n" + "template": "files/themes/mesh-powermenu.rasi" }, { "id": "dmenu", @@ -72,7 +73,7 @@ "path": "${machine:account-home}/.local/bin/dmenu", "owner": "${machine:account}", "mode": "0755", - "content": "#!/bin/sh\n# dmenu, as this node's launcher answers it (module rofi, novox/hq ADR 0208). The node-launcher\n# seat's dmenu-compatible command: choices on standard input, the chosen one on standard output,\n# exit 1 when nothing was chosen. A script calls `dmenu` and works whichever module holds the seat.\n#\n# rofi understands dmenu's -p, -l and -i. dmenu's look options (-fn, -nb, -nf, -sb, -sf, -m, -w)\n# take a value rofi would misread, so they are dropped with it; -b and -f are dropped alone. The\n# look is the launcher's theme.\nset -u\nn=$#\nwhile [ \"$n\" -gt 0 ]; do\n\targ=$1\n\tshift\n\tn=$((n - 1))\n\tcase $arg in\n\t-fn | -nb | -nf | -sb | -sf | -m | -w)\n\t\tif [ \"$n\" -gt 0 ]; then\n\t\t\tshift\n\t\t\tn=$((n - 1))\n\t\tfi\n\t\t;;\n\t-b | -f) ;;\n\t*) set -- \"$@\" \"$arg\" ;;\n\tesac\ndone\nexec rofi -dmenu \"$@\"\n" + "template": "files/bin/dmenu" }, { "id": "launch", @@ -80,7 +81,7 @@ "path": "${machine:account-home}/.local/bin/rofi-launch", "owner": "${machine:account}", "mode": "0755", - "content": "#!/bin/sh\n# rofi-launch drun|run|window|filebrowser|sudo (module rofi, novox/hq ADR 0208): what the launcher's\n# key bindings run.\n#\n# After a resume the keyboard grab can fail for a moment, and rofi then exits at once: a key press\n# that opens nothing. So a failure within half a second is tried again, up to five times. A person\n# closing the menu takes longer than that, and is never shown it a second time.\nset -u\nmode=${1:-drun}\n\nnow_ms() { date +%s%3N; }\n\nattempt() {\n\ti=0\n\twhile [ \"$i\" -lt 5 ]; do\n\t\tstart=$(now_ms)\n\t\t\"$@\" && return 0\n\t\t[ $(($(now_ms) - start)) -ge 500 ] && return 1\n\t\ti=$((i + 1))\n\t\tsleep 0.1\n\tdone\n\treturn 1\n}\n\ncase $mode in\ndrun | run | window | filebrowser)\n\tattempt rofi -show \"$mode\"\n\t;;\nsudo)\n\t# A command line, run with sudo in the node's terminal.\n\tcommand=$(attempt rofi -dmenu -p sudo &2\n\texit 2\n\t;;\nesac\n" + "template": "files/bin/rofi-launch" }, { "id": "powermenu", @@ -88,7 +89,7 @@ "path": "${machine:account-home}/.local/bin/rofi-powermenu", "owner": "${machine:account}", "mode": "0755", - "content": "#!/usr/bin/env bash\n# rofi-powermenu (module rofi, novox/hq ADR 0208): lock, suspend, log out, reboot, shut down.\n#\n# It belongs to the session, and it is the launcher's: its whole face is rofi's. Every action is a\n# verb of logind or the service manager, so nothing here names a window manager or a locker:\n# - lock asks logind to lock the session, which the holder of node-lock-screen answers;\n# - log out ends this login session, whichever window manager runs in it.\n# Adopted from the workstations' power menu of 2026-10-04 (after adi1090x's powermenu type-2).\nset -u\n\ntheme=mesh-powermenu\nuptime=\"$(uptime -p | sed -e 's/^up //')\"\n\nshutdown='󰤂'\nreboot='󰜉'\nlock='󰌾'\nsuspend='󰤄'\nlogout='󰍃'\nyes='󰄲'\nno='󰅖'\n\nmenu() {\n\trofi -dmenu -p \"Uptime: $uptime\" -mesg \"Uptime: $uptime\" -theme \"$theme\"\n}\n\nconfirm() {\n\tprintf '%s\\n%s\\n' \"$yes\" \"$no\" | rofi -dmenu -p 'Confirmation' -mesg 'Are you sure?' -theme \"$theme\" \\\n\t\t-theme-str 'window {location: center; anchor: center; fullscreen: false; width: 350px;}' \\\n\t\t-theme-str 'mainbox {children: [ \"message\", \"listview\" ];}' \\\n\t\t-theme-str 'listview {columns: 2; lines: 1;}' \\\n\t\t-theme-str 'element-text {horizontal-align: 0.5;}' \\\n\t\t-theme-str 'textbox {horizontal-align: 0.5;}'\n}\n\nsession() {\n\t# The login session this menu runs in: the session's own id, else logind's answer for this process.\n\tif [ -n \"${XDG_SESSION_ID:-}\" ]; then\n\t\techo \"$XDG_SESSION_ID\"\n\telse\n\t\tloginctl show-session auto -p Id --value 2>/dev/null\n\tfi\n}\n\nconfirmed() { [ \"$(confirm)\" = \"$yes\" ]; }\n\nchosen=\"$(printf '%s\\n' \"$lock\" \"$suspend\" \"$logout\" \"$reboot\" \"$shutdown\" | menu)\" || exit 0\ncase $chosen in\n\"$lock\") loginctl lock-session \"$(session)\" ;;\n\"$suspend\") confirmed && systemctl suspend ;;\n\"$logout\") confirmed && loginctl terminate-session \"$(session)\" ;;\n\"$reboot\") confirmed && systemctl reboot ;;\n\"$shutdown\") confirmed && systemctl poweroff ;;\nesac\nexit 0\n" + "template": "files/bin/rofi-powermenu" } ], "build": { @@ -103,14 +104,19 @@ "loads": [ "rofi-tools" ] + }, + { + "name": "files", + "kind": "archive", + "from": "files" } ] }, - "contributions": [ + "items": [ { - "seat": "node-display-session", - "kind": "config", - "content": "# The launcher's key bindings (module rofi, novox/hq ADR 0208). Owned by the mesh: replaced at every\n# push. i3 reads this file through its configuration's `include ~/.config/i3/config.d/*.conf`, after\n# the variables it sets, so $mod is i3's.\nbindsym $mod+d exec --no-startup-id ~/.local/bin/rofi-launch drun\nbindsym $mod+t exec --no-startup-id ~/.local/bin/rofi-launch run\nbindsym $mod+Shift+t exec --no-startup-id ~/.local/bin/rofi-launch sudo\nbindsym $mod+Shift+w exec --no-startup-id ~/.local/bin/rofi-launch window\nbindsym $mod+Escape exec --no-startup-id ~/.local/bin/rofi-powermenu\n" + "collection": "contribution:node-display-session:config", + "name": "display-session", + "template": "files/i3/50-rofi.conf" } ] } diff --git a/modules/route-adapter/README.md b/modules/route-adapter/README.md index 2e92b98a..d71d3975 100644 --- a/modules/route-adapter/README.md +++ b/modules/route-adapter/README.md @@ -171,3 +171,11 @@ rather than skipped: the predecessor already served its own registry name this w not a whole positive number of bytes takes the route with it — written without the limit, the predecessor would carry exactly what the module said not to carry, and this module would report success doing it. + +## Handed to the provisioner (issue 792) + +The module is marked `moved`, and a small Go program beside the TypeScript adapter (`cmd/route-provider`) runs the SDK's +provider helper for `route`, with no credential. A consumer of a route whose templates name `${bound:route:name-web}` +(a handed gitea's origins file) is filled by its machine's client from this module's entry for it. The entry's values are +the names the controller composed for that consumer, carried in its plan and copied unchanged: the public domain is +placement, so only the controller can join a label to it. The adapter still writes the predecessor proxy's files. diff --git a/modules/route-adapter/cmd/route-provider/main.go b/modules/route-adapter/cmd/route-provider/main.go new file mode 100644 index 00000000..013c80ae --- /dev/null +++ b/modules/route-adapter/cmd/route-provider/main.go @@ -0,0 +1,37 @@ +// route-provider: route-adapter's Go half, one binary the node's runtime launches and speaks MCP to over stdio +// through the Go SDK. It serves no tools; it runs the SDK's provider helper that answers the consumers of `route` +// (issue 792). The TypeScript adapter beside it keeps writing the proxy's files. +// +// stdout is the MCP channel; everything this module says, it says on stderr. +package main + +import ( + "context" + "fmt" + "os" + + stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/provider" +) + +func say(format string, args ...any) { + fmt.Fprintf(os.Stderr, "[route-adapter] "+format+"\n", args...) +} + +func main() { + node := os.Getenv("MESH_NODE") + if node == "" { + say("MESH_NODE is not set; answering no consumers") + } else { + p := newProvider(node, provider.StateBus(ModuleName), say) + go func() { + if err := p.Run(context.Background()); err != nil { + say("the provider helper stopped: %v", err) + } + }() + } + if err := stdio.Serve("", nil); err != nil { + say("%v", err) + os.Exit(1) + } +} diff --git a/modules/route-adapter/cmd/route-provider/provider.go b/modules/route-adapter/cmd/route-provider/provider.go new file mode 100644 index 00000000..5061a668 --- /dev/null +++ b/modules/route-adapter/cmd/route-provider/provider.go @@ -0,0 +1,36 @@ +package main + +// route-adapter handed to the provisioner (novox/hq ADR 0349 clause 12, issue 792): the module runs the SDK's provider +// helper for `route`, so a consumer of a route — a handed gitea reading `${bound:route:name-web}` — is told its public +// name from the entry in this module's bucket, the one its machine's client fills its files from. +// +// **It composes nothing.** A route's public name is the label its consumer chose joined to the node's public domain, +// which only the controller knows (the domain is placement), so the controller composes it and carries it in the +// consumer's plan entry (`names`); the helper copies it into the entry's values unchanged. There is no credential for +// a route and nothing to make: the traefik files stay the TypeScript adapter's, written from the contributions the +// mesh still gives it. + +import ( + "git.novox.be/novox/mesh-sdk/go/provider" +) + +// ProvisionName is what route-adapter provides, as its manifest says. +const ProvisionName = "route" + +// ModuleName is this module. +const ModuleName = "route-adapter" + +// newProvider is route-adapter answering its consumers over bus on node: the route, served as the manifest serves it +// (nothing of its own), with no credential, and nothing to make or retire for a consumer. +func newProvider(node string, bus provider.Bus, logf func(string, ...any)) *provider.Provider { + return provider.New(provider.Options{ + Node: node, + Module: ModuleName, + Offers: map[string]provider.Offer{ProvisionName: { + Credential: provider.NoCredential, + Serves: map[string]any{}, + }}, + Bus: bus, + Logf: logf, + }) +} diff --git a/modules/route-adapter/cmd/route-provider/provider_test.go b/modules/route-adapter/cmd/route-provider/provider_test.go new file mode 100644 index 00000000..10c30933 --- /dev/null +++ b/modules/route-adapter/cmd/route-provider/provider_test.go @@ -0,0 +1,72 @@ +package main + +import ( + "context" + "encoding/json" + "sync" + "testing" + + "git.novox.be/novox/mesh-sdk/go/provider" +) + +// bucketBus is the helper's bus over in-memory buckets. +type bucketBus struct { + mu sync.Mutex + buckets map[string]map[string][]byte +} + +func (b *bucketBus) Watch(_ context.Context, bucket string, handle func(string, []byte)) error { + b.mu.Lock() + defer b.mu.Unlock() + for k, v := range b.buckets[bucket] { + handle(k, v) + } + return nil +} + +func (b *bucketBus) Put(_ context.Context, bucket, key string, value []byte) error { + b.mu.Lock() + defer b.mu.Unlock() + if b.buckets[bucket] == nil { + b.buckets[bucket] = map[string][]byte{} + } + b.buckets[bucket][key] = append([]byte(nil), value...) + return nil +} + +func (b *bucketBus) Delete(_ context.Context, bucket, key string) error { + b.mu.Lock() + defer b.mu.Unlock() + delete(b.buckets[bucket], key) + return nil +} + +// A consumer of a route is answered with the names its plan's entry carries, and nothing of a credential (issue 792): +// the entry's values are the controller's composition, copied unchanged, and no vault is asked. +func TestAConsumerOfARouteIsToldItsNamesFromItsPlanEntry(t *testing.T) { + bus := &bucketBus{buckets: map[string]map[string][]byte{}} + plan, _ := json.Marshal(map[string]any{"node": "g14", "module": "gitea", "provisioning": map[string]any{ + "provisions": []any{map[string]any{"provision": ProvisionName, "provider": "novox", + "provider-module": ModuleName, "at": "10.7.0.1", "from": "novox", "as": "mesh_g14_gitea", + "names": map[string]any{"name": "git.example.org", "internal-name": "git.novox.internal", + "name-web": "git.example.org"}}}}}) + if err := bus.Put(context.Background(), provider.AssignmentsBucket, "g14.gitea", plan); err != nil { + t.Fatal(err) + } + p := newProvider("novox", bus, t.Logf) + out, err := p.Pass(context.Background()) + if err != nil { + t.Fatal(err) + } + if len(out) != 1 || out[0].Status != provider.Answered { + t.Fatalf("the pass did %+v", out) + } + var e provider.Entry + if err := json.Unmarshal(bus.buckets[provider.BucketOf(ModuleName)]["g14.gitea.route"], &e); err != nil { + t.Fatal(err) + } + if e.Values["name-web"] != "git.example.org" || e.Values["name"] != "git.example.org" || + e.Values["internal-name"] != "git.novox.internal" || e.Credential != "" || e.User != "" || e.Refused != "" { + t.Fatalf("the entry is %+v", e) + } +} diff --git a/modules/route-adapter/files/config.json b/modules/route-adapter/files/config.json new file mode 100644 index 00000000..1d8d8c2a --- /dev/null +++ b/modules/route-adapter/files/config.json @@ -0,0 +1,6 @@ +{ + "dynamic": "/services/traefik/dynamic", + "entrypoint": "websecure", + "certificate-resolver": "le", + "machine": "host.docker.internal" +} diff --git a/modules/route-adapter/go.mod b/modules/route-adapter/go.mod new file mode 100644 index 00000000..792f6ec3 --- /dev/null +++ b/modules/route-adapter/go.mod @@ -0,0 +1,5 @@ +module route-adapter + +go 1.25.0 + +require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261015210729-d3e182503c19 diff --git a/modules/route-adapter/go.sum b/modules/route-adapter/go.sum new file mode 100644 index 00000000..9a531b0e --- /dev/null +++ b/modules/route-adapter/go.sum @@ -0,0 +1,2 @@ +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261015210729-d3e182503c19 h1:mAMt3KFmrHzDiBv4MLL94opWcG9y73ISMrBUyisapQE= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261015210729-d3e182503c19/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY= diff --git a/modules/route-adapter/module.json b/modules/route-adapter/module.json index 10cb2d98..0f2f37d2 100644 --- a/modules/route-adapter/module.json +++ b/modules/route-adapter/module.json @@ -1,6 +1,7 @@ { "module": "route-adapter", "version": "1", + "moved": true, "slug": "radapt", "capabilities": [ "container-runtime" @@ -25,6 +26,33 @@ "mode": "read-write" } ], + "config": { + "dynamic": { + "type": "string", + "description": "The directory the predecessor proxy's file provider reads, on this machine, as an absolute path.", + "pattern": "/.*", + "trusted": true, + "default": "/services/traefik/dynamic" + }, + "entrypoint": { + "type": "string", + "description": "The predecessor proxy's entry point it serves public HTTPS on, by its own name.", + "trusted": true, + "default": "websecure" + }, + "certificate-resolver": { + "type": "string", + "description": "The predecessor proxy's certificate resolver, by its own name.", + "trusted": true, + "default": "le" + }, + "machine": { + "type": "string", + "description": "How the predecessor's proxy reaches this machine: a host name or an address.", + "trusted": true, + "default": "host.docker.internal" + } + }, "resources": [ { "id": "state", @@ -42,9 +70,10 @@ "id": "config", "type": "file", "path": "${dir:state}/config.json", + "takes-config": ["dynamic", "entrypoint", "certificate-resolver", "machine"], "merge": "json", "mode": "0644", - "content": "{\n \"dynamic\": \"/services/traefik/dynamic\",\n \"entrypoint\": \"websecure\",\n \"certificate-resolver\": \"le\",\n \"machine\": \"host.docker.internal\"\n}\n" + "template": "files/config.json" }, { "id": "adapt", @@ -75,7 +104,29 @@ "entrypoints": [ "index.js" ] + }, + { + "name": "provider", + "kind": "bundle", + "language": "go", + "system": "arch", + "from": "cmd/route-provider", + "binary": "route-provider", + "loads": [ + "route-provider" + ] + }, + { + "name": "files", + "kind": "archive", + "from": "files" } ] - } + }, + "state": [ + "bound" + ], + "reads": [ + "mesh-controller.assignments" + ] } diff --git a/modules/route-proxy/README.md b/modules/route-proxy/README.md index bc0e0d7a..5b167dfe 100644 --- a/modules/route-proxy/README.md +++ b/modules/route-proxy/README.md @@ -84,3 +84,10 @@ traffic. | `ACME_CACHE` | `/acme` | where issued certificates persist; required when `TLS_LISTEN` is set, so a restart does not re-order | | `ACME_DIRECTORY` | Let's Encrypt **staging** in the binary; production in the module's `acme.env` | the issuer | | `ACME_CA_BUNDLE` | *(unset)* | a file of roots to trust for the issuer's own API — set only for a private/lab authority whose API certificate the world does not yet trust | + +## Answered by the provider helper (issue 792) + +A small Go program beside the proxy (`cmd/route-provider`) runs the SDK's provider helper for `route`, with no credential, +and the offer says `"answered-by": "helper"`: a handed consumer (gitea's origins file) is filled by its machine's client +from this module's entry, which carries the public names the controller composed for it. The module is **not** marked +moved: its own files, `acme.env` and the broker secret, are composed as before, and the proxy itself is unchanged. diff --git a/modules/route-proxy/cmd/route-provider/main.go b/modules/route-proxy/cmd/route-provider/main.go new file mode 100644 index 00000000..e70a6892 --- /dev/null +++ b/modules/route-proxy/cmd/route-provider/main.go @@ -0,0 +1,37 @@ +// route-provider: route-proxy's Go half, one binary the node's runtime launches and speaks MCP to over stdio +// through the Go SDK. It serves no tools; it runs the SDK's provider helper that answers the consumers of `route` +// (issue 792). The TypeScript adapter beside it keeps writing the proxy's files. +// +// stdout is the MCP channel; everything this module says, it says on stderr. +package main + +import ( + "context" + "fmt" + "os" + + stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/provider" +) + +func say(format string, args ...any) { + fmt.Fprintf(os.Stderr, "[route-proxy] "+format+"\n", args...) +} + +func main() { + node := os.Getenv("MESH_NODE") + if node == "" { + say("MESH_NODE is not set; answering no consumers") + } else { + p := newProvider(node, provider.StateBus(ModuleName), say) + go func() { + if err := p.Run(context.Background()); err != nil { + say("the provider helper stopped: %v", err) + } + }() + } + if err := stdio.Serve("", nil); err != nil { + say("%v", err) + os.Exit(1) + } +} diff --git a/modules/route-proxy/cmd/route-provider/provider.go b/modules/route-proxy/cmd/route-provider/provider.go new file mode 100644 index 00000000..7eee671b --- /dev/null +++ b/modules/route-proxy/cmd/route-provider/provider.go @@ -0,0 +1,36 @@ +package main + +// route-proxy handed to the provisioner (novox/hq ADR 0349 clause 12, issue 792): the module runs the SDK's provider +// helper for `route`, so a consumer of a route — a handed gitea reading `${bound:route:name-web}` — is told its public +// name from the entry in this module's bucket, the one its machine's client fills its files from. +// +// **It composes nothing.** A route's public name is the label its consumer chose joined to the node's public domain, +// which only the controller knows (the domain is placement), so the controller composes it and carries it in the +// consumer's plan entry (`names`); the helper copies it into the entry's values unchanged. There is no credential for +// a route and nothing to make: the traefik files stay the TypeScript adapter's, written from the contributions the +// mesh still gives it. + +import ( + "git.novox.be/novox/mesh-sdk/go/provider" +) + +// ProvisionName is what route-proxy provides, as its manifest says. +const ProvisionName = "route" + +// ModuleName is this module. +const ModuleName = "route-proxy" + +// newProvider is route-proxy answering its consumers over bus on node: the route, served as the manifest serves it +// (nothing of its own), with no credential, and nothing to make or retire for a consumer. +func newProvider(node string, bus provider.Bus, logf func(string, ...any)) *provider.Provider { + return provider.New(provider.Options{ + Node: node, + Module: ModuleName, + Offers: map[string]provider.Offer{ProvisionName: { + Credential: provider.NoCredential, + Serves: map[string]any{}, + }}, + Bus: bus, + Logf: logf, + }) +} diff --git a/modules/route-proxy/cmd/route-provider/provider_test.go b/modules/route-proxy/cmd/route-provider/provider_test.go new file mode 100644 index 00000000..f5c4b7c2 --- /dev/null +++ b/modules/route-proxy/cmd/route-provider/provider_test.go @@ -0,0 +1,108 @@ +package main + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "os" + "sync" + "testing" + + "git.novox.be/novox/mesh-sdk/go/provider" +) + +// bucketBus is the helper's bus over in-memory buckets. +type bucketBus struct { + mu sync.Mutex + buckets map[string]map[string][]byte +} + +func (b *bucketBus) Watch(_ context.Context, bucket string, handle func(string, []byte)) error { + b.mu.Lock() + defer b.mu.Unlock() + for k, v := range b.buckets[bucket] { + handle(k, v) + } + return nil +} + +func (b *bucketBus) Put(_ context.Context, bucket, key string, value []byte) error { + b.mu.Lock() + defer b.mu.Unlock() + if b.buckets[bucket] == nil { + b.buckets[bucket] = map[string][]byte{} + } + b.buckets[bucket][key] = append([]byte(nil), value...) + return nil +} + +func (b *bucketBus) Delete(_ context.Context, bucket, key string) error { + b.mu.Lock() + defer b.mu.Unlock() + delete(b.buckets[bucket], key) + return nil +} + +// A consumer of a route is answered with the names its plan's entry carries, and nothing of a credential (issue 792): +// the entry's values are the controller's composition, copied unchanged, and no vault is asked. +func TestAConsumerOfARouteIsToldItsNamesFromItsPlanEntry(t *testing.T) { + bus := &bucketBus{buckets: map[string]map[string][]byte{}} + plan, _ := json.Marshal(map[string]any{"node": "g14", "module": "gitea", "provisioning": map[string]any{ + "provisions": []any{map[string]any{"provision": ProvisionName, "provider": "novox", + "provider-module": ModuleName, "at": "10.7.0.1", "from": "novox", "as": "mesh_g14_gitea", + "names": map[string]any{"name": "git.example.org", "internal-name": "git.novox.internal", + "name-web": "git.example.org"}}}}}) + if err := bus.Put(context.Background(), provider.AssignmentsBucket, "g14.gitea", plan); err != nil { + t.Fatal(err) + } + p := newProvider("novox", bus, t.Logf) + out, err := p.Pass(context.Background()) + if err != nil { + t.Fatal(err) + } + if len(out) != 1 || out[0].Status != provider.Answered { + t.Fatalf("the pass did %+v", out) + } + var e provider.Entry + if err := json.Unmarshal(bus.buckets[provider.BucketOf(ModuleName)]["g14.gitea.route"], &e); err != nil { + t.Fatal(err) + } + if e.Values["name-web"] != "git.example.org" || e.Values["name"] != "git.example.org" || + e.Values["internal-name"] != "git.novox.internal" || e.Credential != "" || e.User != "" || e.Refused != "" { + t.Fatalf("the entry is %+v", e) + } +} + +// route-proxy's own files and resources stay as they were (issue 792): the module answers its route through the +// helper but is not marked moved, so its acme.env, its broker secret and its containers are composed as today and +// the public ingress does not change. Everything of the manifest but its build, what it provides and the state it +// keeps is pinned by this digest; changing the ingress means changing it here, on purpose. +func TestRouteProxyIsNotMovedAndKeepsItsOwnFiles(t *testing.T) { + raw, err := os.ReadFile("../../module.json") + if err != nil { + t.Fatal(err) + } + var m map[string]any + if err := json.Unmarshal(raw, &m); err != nil { + t.Fatal(err) + } + if _, moved := m["moved"]; moved { + t.Fatal("route-proxy is marked moved: its own files would be handed") + } + provides, _ := m["provides"].([]any) + if len(provides) != 1 || provides[0].(map[string]any)["answered-by"] != "helper" { + t.Fatalf("route is not said to be answered by the helper: %v", provides) + } + for _, k := range []string{"build", "provides", "state", "reads"} { + delete(m, k) + } + pinned, err := json.Marshal(m) // sorted keys, compact + if err != nil { + t.Fatal(err) + } + sum := sha256.Sum256(pinned) + if got := hex.EncodeToString(sum[:]); got != "57a39de0943d8b33b9db6cb998ce97f8793602cca6cdfa5849c1c392390454a3" { + t.Fatalf("route-proxy's own manifest changed: digest %s", got) + } +} diff --git a/modules/route-proxy/files/acme.env b/modules/route-proxy/files/acme.env new file mode 100644 index 00000000..428aae2b --- /dev/null +++ b/modules/route-proxy/files/acme.env @@ -0,0 +1,3 @@ +ACME_DIRECTORY=https://acme-v02.api.letsencrypt.org:443/directory +ACME_ROOTS=https://acme-v02.api.letsencrypt.org:443 +ACME_ROOTS_PATH= diff --git a/modules/route-proxy/files/internal-acme.env b/modules/route-proxy/files/internal-acme.env new file mode 100644 index 00000000..72b0f7bb --- /dev/null +++ b/modules/route-proxy/files/internal-acme.env @@ -0,0 +1,3 @@ +INTERNAL_ACME_DIRECTORY=https://${bound:internal-acme-ca:at}:${bound:internal-acme-ca:port}${bound:internal-acme-ca:path} +INTERNAL_ACME_ROOTS=https://${bound:internal-acme-ca:at}:${bound:internal-acme-ca:port}${bound:internal-acme-ca:roots} +INTERNAL_ACME_ROOTS_PATH=${bound:internal-acme-ca:roots} diff --git a/modules/route-proxy/go.mod b/modules/route-proxy/go.mod new file mode 100644 index 00000000..ade1dc0a --- /dev/null +++ b/modules/route-proxy/go.mod @@ -0,0 +1,5 @@ +module route-proxy + +go 1.25.0 + +require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261015210729-d3e182503c19 diff --git a/modules/route-proxy/go.sum b/modules/route-proxy/go.sum new file mode 100644 index 00000000..9a531b0e --- /dev/null +++ b/modules/route-proxy/go.sum @@ -0,0 +1,2 @@ +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261015210729-d3e182503c19 h1:mAMt3KFmrHzDiBv4MLL94opWcG9y73ISMrBUyisapQE= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261015210729-d3e182503c19/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY= diff --git a/modules/route-proxy/module.json b/modules/route-proxy/module.json index 8b95306b..a2734f17 100644 --- a/modules/route-proxy/module.json +++ b/modules/route-proxy/module.json @@ -9,7 +9,8 @@ { "name": "route", "scope": "mesh", - "identity": false + "identity": false, + "answered-by": "helper" } ], "serves": { @@ -95,14 +96,14 @@ "type": "file", "path": "${dir:state}/acme.env", "mode": "0600", - "content": "ACME_DIRECTORY=https://acme-v02.api.letsencrypt.org:443/directory\nACME_ROOTS=https://acme-v02.api.letsencrypt.org:443\nACME_ROOTS_PATH=\n" + "template": "files/acme.env" }, { "id": "internal-acme-env", "type": "file", "path": "${dir:state}/internal-acme.env", "mode": "0600", - "content": "INTERNAL_ACME_DIRECTORY=https://${bound:internal-acme-ca:at}:${bound:internal-acme-ca:port}${bound:internal-acme-ca:path}\nINTERNAL_ACME_ROOTS=https://${bound:internal-acme-ca:at}:${bound:internal-acme-ca:port}${bound:internal-acme-ca:roots}\nINTERNAL_ACME_ROOTS_PATH=${bound:internal-acme-ca:roots}\n" + "template": "files/internal-acme.env" }, { "id": "trust", @@ -207,6 +208,22 @@ "name": "trust", "kind": "upstream", "from": "alpine@sha256:28bd5fe8b56d1bd048e5babf5b10710ebe0bae67db86916198a6eec434943f8b" + }, + { + "name": "provider", + "kind": "bundle", + "language": "go", + "system": "arch", + "from": "cmd/route-provider", + "binary": "route-provider", + "loads": [ + "route-provider" + ] + }, + { + "name": "files", + "kind": "archive", + "from": "files" } ], "on": [ @@ -226,5 +243,11 @@ "failregex": "^.*TLS handshake error from :\\d+: (?:no public route for|acme/autocert: missing server name)\n ^.*refused: no route for .*, asked from :\\d+$", "jail": "backend = systemd\njournalmatch = CONTAINER_NAME=route-proxy\nport = http,https\nmaxretry = 10\nfindtime = 1d\nbantime = 1d" } + ], + "state": [ + "bound" + ], + "reads": [ + "mesh-controller.assignments" ] } diff --git a/modules/screen-lock/cmd/screen-lock-tools/lock_test.go b/modules/screen-lock/cmd/screen-lock-tools/lock_test.go index b6fc0420..7925c9b6 100644 --- a/modules/screen-lock/cmd/screen-lock-tools/lock_test.go +++ b/modules/screen-lock/cmd/screen-lock-tools/lock_test.go @@ -181,7 +181,7 @@ func TestAnInhibitionTurnsIdleOffAndRestoresWhatWasThereWhenItEndsOrIsStopped(t func TestTheDeclaredTimeoutsAreTheSessionStartsOwn(t *testing.T) { m := readManifest(t) - code := m.Shell[0].Code + code := m.shell(t)[0].Code if !strings.Contains(code, "xset s "+strconv.Itoa(declaredLock)+" "+strconv.Itoa(declaredLock)+"\n") || !strings.Contains(code, "xset dpms "+strconv.Itoa(declaredStandby)+" "+strconv.Itoa(declaredSuspend)+" "+strconv.Itoa(declaredOff)+"\n") { t.Fatalf("the tools' declared values and the session start's disagree:\n%s", code) diff --git a/modules/screen-lock/cmd/screen-lock-tools/manifest_helpers_test.go b/modules/screen-lock/cmd/screen-lock-tools/manifest_helpers_test.go index d4fb76d9..fb992951 100644 --- a/modules/screen-lock/cmd/screen-lock-tools/manifest_helpers_test.go +++ b/modules/screen-lock/cmd/screen-lock-tools/manifest_helpers_test.go @@ -14,6 +14,8 @@ import ( type manifest struct { Module string `json:"module"` Version string `json:"version"` + Moved bool `json:"moved"` + Items []item `json:"items"` Capabilities []string `json:"capabilities"` Requires []string `json:"requires"` Claims []claim `json:"claims"` @@ -44,6 +46,53 @@ type shellCode struct { Code string `json:"code"` } +// An item of a collection (novox/hq ADR 0349 clause 13): its text is the template file beside the manifest. +type item struct { + Collection string `json:"collection"` + Name string `json:"name"` + Attributes map[string]string `json:"attributes"` + Template string `json:"template"` +} + +// shell is the code the module adds to a shell's or the session's start, read from the templates of its items of +// the collections shell::, in the order they are declared. +func (m manifest) shell(t *testing.T) []shellCode { + t.Helper() + var out []shellCode + for _, it := range m.Items { + parts := strings.Split(it.Collection, ":") + if len(parts) != 3 || parts[0] != "shell" { + continue + } + text, err := os.ReadFile(filepath.Join("..", "..", it.Template)) + if err != nil { + t.Fatal(err) + } + out = append(out, shellCode{For: parts[1], Slot: parts[2], Code: string(text)}) + } + return out +} + +// environmentOf is the variables the module sets, read from the templates of its items of the environment. +func (m manifest) environmentOf(t *testing.T) *environment { + t.Helper() + var env *environment + for _, it := range m.Items { + if it.Collection != "environment" || it.Attributes["kind"] != "variable" { + continue + } + text, err := os.ReadFile(filepath.Join("..", "..", it.Template)) + if err != nil { + t.Fatal(err) + } + if env == nil { + env = &environment{Variables: map[string]string{}} + } + env.Variables[it.Attributes["name"]] = strings.TrimSuffix(string(text), "\n") + } + return env +} + func readManifest(t *testing.T) manifest { t.Helper() raw, err := os.ReadFile(filepath.Join("..", "..", "module.json")) @@ -56,6 +105,7 @@ func readManifest(t *testing.T) manifest { if err := dec.Decode(&m); err != nil { t.Fatalf("module.json: %v", err) } + m.Environment = m.environmentOf(t) return m } @@ -83,26 +133,66 @@ func (m manifest) packages() (present, absent []string) { return present, absent } -// sameAsSource checks that a file resource's content is byte for byte the module's source file, so -// the readable file in the repository is what the machine gets. -func (m manifest) sameAsSource(t *testing.T, id, source string) { +// fromTemplate checks that a file resource takes its text from its template file in the module's +// repository (novox/hq ADR 0349 §2): it names the template, carries no text of its own beside it, and the +// template is there. The readable file in the repository is the one place the machine's text is written. +func (m manifest) fromTemplate(t *testing.T, id, template string) { t.Helper() - want, err := os.ReadFile(filepath.Join("..", "..", source)) - if err != nil { - t.Fatal(err) - } r := m.resource(t, id) if r["type"] != "file" { t.Fatalf("%s is a %v, not a file", id, r["type"]) } - if got, _ := r["content"].(string); got != string(want) { - t.Fatalf("resource %s's content is not %s: edit the source and copy it into module.json", id, source) + if r["template"] != template { + t.Fatalf("resource %s does not take its text from %s: template %v", id, template, r["template"]) + } + if _, inline := r["content"]; inline { + t.Fatalf("resource %s keeps its text inline beside its template: a file's text is in one place", id) + } + if _, err := os.Stat(filepath.Join("..", "..", template)); err != nil { + t.Fatalf("resource %s names %s: %v", id, template, err) } if r["owner"] != "${machine:account}" && !strings.HasPrefix(r["path"].(string), "/etc/") { t.Fatalf("%s under the home is the account's", id) } } +// everyFileIsATemplate checks that no file resource carries its text inline, and that the build packs +// files/ into the one archive the controller reads the templates back from (novox/hq ADR 0349 §2). +func (m manifest) everyFileIsATemplate(t *testing.T) { + t.Helper() + for _, r := range m.Resources { + if _, inline := r["content"]; inline { + t.Errorf("resource %v keeps its text inline: keep it in files/ and name it as the template", r["id"]) + } + } + packs := 0 + for _, a := range m.Build.Artifacts { + if a["kind"] == "archive" && a["from"] == "files" { + packs++ + if a["name"] != "files" { + t.Errorf("the templates archive is named %v, not files", a["name"]) + } + } + } + if packs != 1 { + t.Errorf("the build packs files/ into %d archives, not one", packs) + } +} + +// text is the text a file resource places on the machine: its template file's. +func (m manifest) text(t *testing.T, id string) string { + t.Helper() + name, _ := m.resource(t, id)["template"].(string) + if name == "" { + t.Fatalf("resource %s names no template", id) + } + raw, err := os.ReadFile(filepath.Join("..", "..", name)) + if err != nil { + t.Fatal(err) + } + return string(raw) +} + // checkTheToolsAgree checks that the manifest lists the module's own tools exactly, that the bundle // serves each seat verb the claims promise as ., and that the Go bundle is declared. func checkTheToolsAgree(t *testing.T, m manifest) { diff --git a/modules/screen-lock/cmd/screen-lock-tools/manifest_test.go b/modules/screen-lock/cmd/screen-lock-tools/manifest_test.go index 3a585d68..3934c73b 100644 --- a/modules/screen-lock/cmd/screen-lock-tools/manifest_test.go +++ b/modules/screen-lock/cmd/screen-lock-tools/manifest_test.go @@ -30,12 +30,12 @@ func TestTheColourBuildIsKeptWithAFallbackAndXscreensaverGoes(t *testing.T) { if !reflect.DeepEqual(present, []string{"xss-lock", "i3lock"}) || !reflect.DeepEqual(absent, []string{"xscreensaver"}) { t.Fatalf("packages: %v, absent %v", present, absent) } - m.sameAsSource(t, "wrapper", "files/bin/screen-lock") + m.fromTemplate(t, "wrapper", "files/bin/screen-lock") wrapper := m.resource(t, "wrapper") if wrapper["mode"] != "0755" || wrapper["path"] != "${machine:account-home}/.local/bin/screen-lock" { t.Fatalf("the locker: %v", wrapper) } - c := wrapper["content"].(string) + c := m.text(t, "wrapper") // The colour build's options only once it has said it is the colour build: the distribution's // i3lock refuses an option it does not know, and the screen would not lock at all. check := strings.Index(c, `i3lock --version 2>&1 | grep -qE '[0-9]\.c\.[0-9]'`) @@ -53,10 +53,10 @@ func TestTheColourBuildIsKeptWithAFallbackAndXscreensaverGoes(t *testing.T) { func TestTheSessionStartSetsTheTimeoutsAndKeepsOneWatcherForTheSession(t *testing.T) { m := readManifest(t) - if len(m.Shell) != 1 || m.Shell[0].For != "xinitrc" || m.Shell[0].Slot != "normal" { - t.Fatalf("one xinitrc contribution in normal: %+v", m.Shell) + if len(m.shell(t)) != 1 || m.shell(t)[0].For != "xinitrc" || m.shell(t)[0].Slot != "normal" { + t.Fatalf("one xinitrc contribution in normal: %+v", m.shell(t)) } - code := m.Shell[0].Code + code := m.shell(t)[0].Code if strings.Count(code, "xss-lock --") != 1 || !strings.Contains(code, "--transfer-sleep-lock") || !strings.Contains(code, "while kill -0 $$") || !strings.HasSuffix(strings.TrimSpace(code), "&") { t.Fatalf("the watcher: %q", code) @@ -87,3 +87,8 @@ func TestTheToolsAgreeWithTheManifest(t *testing.T) { checkTheToolsAgree(t, m) checkNoSecretsOrInstallationNames(t) } + +// No file resource carries its text inline (novox/hq ADR 0349 §2). +func TestEveryFileTakesItsTextFromATemplate(t *testing.T) { + readManifest(t).everyFileIsATemplate(t) +} diff --git a/modules/screen-lock/files/xinitrc-normal b/modules/screen-lock/files/xinitrc-normal new file mode 100644 index 00000000..f7f7123e --- /dev/null +++ b/modules/screen-lock/files/xinitrc-normal @@ -0,0 +1,8 @@ +# The lock screen (module screen-lock, novox/hq ADR 0208): the session locks after 30 minutes idle, +# the displays go to standby and suspend then and off after an hour, and xss-lock runs the locker on +# idle, before suspend and on logind's Lock. xss-lock needs this login session, so it starts here and +# not as a unit. It is started again if it exits, for as long as this session lasts ($$ is the +# session's own process, which becomes the window manager). +xset s 1800 1800 +xset dpms 1800 1800 3600 +(while kill -0 $$ 2>/dev/null; do xss-lock --transfer-sleep-lock -- "$HOME/.local/bin/screen-lock"; sleep 2; done) & diff --git a/modules/screen-lock/module.json b/modules/screen-lock/module.json index 28bf7e57..1791c798 100644 --- a/modules/screen-lock/module.json +++ b/modules/screen-lock/module.json @@ -1,6 +1,7 @@ { "module": "screen-lock", "version": "1", + "moved": true, "capabilities": [ "package-manager" ], @@ -21,11 +22,11 @@ "screen_lock_inhibit", "screen_lock_locked" ], - "shell": [ + "items": [ { - "for": "xinitrc", - "slot": "normal", - "code": "# The lock screen (module screen-lock, novox/hq ADR 0208): the session locks after 30 minutes idle,\n# the displays go to standby and suspend then and off after an hour, and xss-lock runs the locker on\n# idle, before suspend and on logind's Lock. xss-lock needs this login session, so it starts here and\n# not as a unit. It is started again if it exits, for as long as this session lasts ($$ is the\n# session's own process, which becomes the window manager).\nxset s 1800 1800\nxset dpms 1800 1800 3600\n(while kill -0 $$ 2>/dev/null; do xss-lock --transfer-sleep-lock -- \"$HOME/.local/bin/screen-lock\"; sleep 2; done) &\n" + "collection": "shell:xinitrc:normal", + "name": "xinitrc-normal", + "template": "files/xinitrc-normal" } ], "resources": [ @@ -51,7 +52,7 @@ "path": "${machine:account-home}/.local/bin/screen-lock", "owner": "${machine:account}", "mode": "0755", - "content": "#!/usr/bin/env bash\n# screen-lock (module screen-lock, novox/hq ADR 0208): the locker. xss-lock runs it on idle, before\n# suspend and on logind's Lock, and node-lock-screen's `lock` ends in it.\n#\n# The operator's look, adopted from the predecessor's my-i3lock: the screen as it was, blurred, with\n# an orange ring, the time and the date. That needs i3lock-color, from the distribution's user\n# repository, kept as found until the mesh carries such software (novox/hq research 027, question 1).\n# On a machine without it the distribution's i3lock shows the same blurred screen, taken here, with\n# its own plain ring; failing that, black.\n#\n# Before a suspend, xss-lock hands this script a lock on the sleep (XSS_SLEEP_LOCK_FD) and suspends\n# once it is released. The locker must not inherit it, or the machine would wait for the unlock\n# before sleeping; it is released once i3lock is up, so the machine never sleeps unlocked. This is\n# xss-lock's own documented pattern for i3lock.\nset -u\n\n# One locker: a second press of the key, or a lock while locked, changes nothing.\nif pgrep -xu \"$EUID\" i3lock >/dev/null; then\n\texit 0\nfi\n\n# The colour build numbers its versions .c. (2.13.c.5); its version line never says \"color\".\nif i3lock --version 2>&1 | grep -qE '[0-9]\\.c\\.[0-9]'; then\n\tblank='#00000000' clear='#ffffff22' accent='#ca4a00' wrong='#880000bb' verifying='#bb00bbbb'\n\toptions=(\n\t\t--insidever-color=\"$clear\" --ringver-color=\"$verifying\"\n\t\t--insidewrong-color=\"$clear\" --ringwrong-color=\"$wrong\"\n\t\t--inside-color=\"$blank\" --ring-color=\"$accent\" --line-color=\"$blank\" --separator-color=\"$accent\"\n\t\t--verif-color=\"$accent\" --wrong-color=\"$accent\" --time-color=\"$accent\" --date-color=\"$accent\"\n\t\t--layout-color=\"$accent\" --keyhl-color=\"$wrong\" --bshl-color=\"$wrong\"\n\t\t--screen 1 --blur 5 --ring-width=7.0 --clock --indicator\n\t\t--time-str=\"%H:%M:%S\" --date-str=\"%A, %Y-%m-%d\"\n\t\t--time-font=sans-serif --date-font=sans-serif --verif-font=sans-serif\n\t\t--wrong-font=sans-serif --layout-font=sans-serif --keylayout 1\n\t\t--show-failed-attempts --ignore-empty-password\n\t)\nelse\n\toptions=(--color=000000 --show-failed-attempts --ignore-empty-password)\n\tshot=\"${XDG_RUNTIME_DIR:-/tmp}/screen-lock.png\"\n\tif command -v magick >/dev/null && magick import -window root -resize 25% -blur 0x3 -resize 400% \"$shot\" 2>/dev/null; then\n\t\toptions+=(--image=\"$shot\")\n\tfi\nfi\n\nif [[ -e /dev/fd/${XSS_SLEEP_LOCK_FD:--1} ]]; then\n\tkill_i3lock() { pkill -xu \"$EUID\" \"$@\" i3lock; }\n\ttrap kill_i3lock TERM INT\n\ti3lock \"${options[@]}\" {XSS_SLEEP_LOCK_FD}<&-\n\texec {XSS_SLEEP_LOCK_FD}<&-\n\twhile kill_i3lock -0; do\n\t\tsleep 0.5\n\tdone\nelse\n\ttrap 'kill %%' TERM INT\n\ti3lock --nofork \"${options[@]}\" &\n\twait\nfi\n" + "template": "files/bin/screen-lock" } ], "build": { @@ -66,6 +67,11 @@ "loads": [ "screen-lock-tools" ] + }, + { + "name": "files", + "kind": "archive", + "from": "files" } ] } diff --git a/modules/searxng/files/config.json b/modules/searxng/files/config.json new file mode 100644 index 00000000..0967ef42 --- /dev/null +++ b/modules/searxng/files/config.json @@ -0,0 +1 @@ +{} diff --git a/modules/searxng/files/settings.yml b/modules/searxng/files/settings.yml new file mode 100644 index 00000000..6d7248ee --- /dev/null +++ b/modules/searxng/files/settings.yml @@ -0,0 +1,16 @@ +{ + "use_default_settings": true, + "server": { + "secret_key": "${secret:secret}", + "base_url": false, + "limiter": false, + "image_proxy": false, + "public_instance": false + }, + "search": { + "formats": ["html", "json"] + }, + "valkey": { + "url": "valkey://valkey:6379/0" + } +} diff --git a/modules/searxng/module.json b/modules/searxng/module.json index f6951bdf..01d40126 100644 --- a/modules/searxng/module.json +++ b/modules/searxng/module.json @@ -29,6 +29,79 @@ } ] }, + "config": { + "general": { + "type": "object", + "description": "SearXNG's general section of settings.yml, laid over the file's own and SearXNG's defaults: the instance's name, contact and debug switches.", + "trusted": true, + "default": {} + }, + "brand": { + "type": "object", + "description": "SearXNG's brand section of settings.yml, laid over the file's own and SearXNG's defaults: the links the pages show for the instance's source, issues and documentation.", + "trusted": true, + "default": {} + }, + "search": { + "type": "object", + "description": "SearXNG's search section of settings.yml, laid over the file's own and SearXNG's defaults: safe search, autocomplete, the result formats offered and how long an engine is suspended after an error.", + "trusted": true, + "default": { + "formats": [ + "html", + "json" + ] + } + }, + "server": { + "type": "object", + "description": "SearXNG's server section of settings.yml, laid over the file's own and SearXNG's defaults: the public base URL, the request method, the limiter and the image proxy; its secret_key is the vault's.", + "trusted": true, + "default": { + "secret_key": "${secret:secret}", + "base_url": false, + "limiter": false, + "image_proxy": false, + "public_instance": false + } + }, + "ui": { + "type": "object", + "description": "SearXNG's ui section of settings.yml, laid over the file's own and SearXNG's defaults: the theme, the default locale and how results are shown.", + "trusted": true, + "default": {} + }, + "outgoing": { + "type": "object", + "description": "SearXNG's outgoing section of settings.yml, laid over the file's own and SearXNG's defaults: how SearXNG reaches the engines: timeouts, connection pool and proxies.", + "trusted": true, + "default": {} + }, + "engines": { + "type": "list", + "description": "SearXNG's engines section of settings.yml, laid over the file's own and SearXNG's defaults: the engines to add or change, each by its name.", + "trusted": true, + "default": [] + }, + "plugins": { + "type": "object", + "description": "SearXNG's plugins section of settings.yml, laid over the file's own and SearXNG's defaults: the plugins to switch on or off, by their id.", + "trusted": true, + "default": {} + }, + "categories_as_tabs": { + "type": "object", + "description": "SearXNG's categories_as_tabs section of settings.yml, laid over the file's own and SearXNG's defaults: the categories shown as tabs above the results.", + "trusted": true, + "default": {} + }, + "preferences": { + "type": "object", + "description": "SearXNG's preferences section of settings.yml, laid over the file's own and SearXNG's defaults: the preferences a user cannot change (lock).", + "trusted": true, + "default": {} + } + }, "resources": [ { "id": "mesh-state", @@ -76,7 +149,7 @@ "path": "${dir:state}/settings.yml", "mode": "0600", "merge": "json", - "takes-settings": [ + "takes-config": [ "general", "brand", "search", @@ -88,7 +161,7 @@ "categories_as_tabs", "preferences" ], - "content": "{\n \"use_default_settings\": true,\n \"server\": {\n \"secret_key\": \"${secret:secret}\",\n \"base_url\": false,\n \"limiter\": false,\n \"image_proxy\": false,\n \"public_instance\": false\n },\n \"search\": {\n \"formats\": [\"html\", \"json\"]\n },\n \"valkey\": {\n \"url\": \"valkey://valkey:6379/0\"\n }\n}\n" + "template": "files/settings.yml" }, { "id": "server", @@ -111,7 +184,7 @@ "type": "file", "path": "${dir:mesh-state}/config.json", "mode": "0600", - "content": "{}\n" + "template": "files/config.json" } ], "requires": [ @@ -142,6 +215,11 @@ "MESH_SEARXNG_URL": "http://127.0.0.1:${port:8080}", "MESH_SEARXNG_CONFIG_FILE": "${dir:mesh-state}/config.json" } + }, + { + "name": "files", + "kind": "archive", + "from": "files" } ] } diff --git a/modules/showcase/module.json b/modules/showcase/module.json index ab627311..9f9e9812 100644 --- a/modules/showcase/module.json +++ b/modules/showcase/module.json @@ -1,6 +1,7 @@ { "module": "showcase", "version": "1", + "templates": "templates", "slug": "show", "capabilities": [ "container-runtime" @@ -74,6 +75,11 @@ "name": "helper", "kind": "upstream", "from": "alpine@sha256:28bd5fe8b56d1bd048e5babf5b10710ebe0bae67db86916198a6eec434943f8b" + }, + { + "name": "templates", + "kind": "archive", + "from": "templates" } ] }, @@ -108,7 +114,7 @@ "type": "file", "path": "${dir:state}/showcase.env", "mode": "0600", - "content": "SHOWCASE_GREETING=hello\nSHOWCASE_EVERY_SECONDS=30\nSHOWCASE_STATE=/var/lib/showcase\nSHOWCASE_DATABASE=${bound:postgres-database:at}\nSHOWCASE_LISTEN=${port:8080}\n" + "template": "templates/showcase.env" }, { "id": "packed", diff --git a/modules/showcase/provisioner/index.ts b/modules/showcase/provisioner/index.ts index 5e4b9bf8..aea284cd 100644 --- a/modules/showcase/provisioner/index.ts +++ b/modules/showcase/provisioner/index.ts @@ -1,20 +1,19 @@ // showcase's provisioner — how a consumer is given an instance of what this module provides. // // **A provider ships the provisioner that creates instances of the resource it offers** (ADR -// 0040). The mesh asks; this adapts that request to whatever the software actually needs, and -// hands back what the consumer is given. -import { provisioner } from "@novox/mesh-sdk/provisioner"; +// 0040). The mesh asks; this adapts that request to whatever the software actually needs. The +// reconcile loop is the sdk's (`runProvisioner`); this writes only the per-service half. +import { runProvisioner, type Provision } from "@novox/mesh-sdk/provisioner"; -await provisioner({ - provision: "greeting", - async create({ consumer }: { consumer: string }) { +runProvisioner("greeting", { + async create(p: Provision): Promise { // A real provider would create something here — a database, a vhost, an account. This one has - // nothing to create, so it returns what a consumer is told, which is the half that matters: - // the mesh seals it and delivers it, and the consumer never sees this code. - return { serves: { greeting: `hello ${consumer}` } }; + // nothing to create: what a consumer is told is the manifest's `serves` block, which the mesh + // fills for this consumer, seals and delivers (ADR 0048), so the consumer never sees this code. + console.log(`[showcase] greeting ready for ${p.consumer ?? p.as}`); }, - async remove() { + async remove(): Promise { // Removal is not optional. A provider that cannot take an instance back leaves the mesh unable - // to unassign a consumer without leaking whatever it was given. + // to unassign a consumer without leaking whatever it was given. Nothing was made, so nothing goes. }, }); diff --git a/modules/showcase/templates/showcase.env b/modules/showcase/templates/showcase.env new file mode 100644 index 00000000..8ea4a902 --- /dev/null +++ b/modules/showcase/templates/showcase.env @@ -0,0 +1,5 @@ +SHOWCASE_GREETING=hello +SHOWCASE_EVERY_SECONDS=30 +SHOWCASE_STATE=/var/lib/showcase +SHOWCASE_DATABASE=${bound:postgres-database:at} +SHOWCASE_LISTEN=${port:8080} diff --git a/modules/showcase/tools/index.ts b/modules/showcase/tools/index.ts index 45243192..65e2dad8 100644 --- a/modules/showcase/tools/index.ts +++ b/modules/showcase/tools/index.ts @@ -1,25 +1,31 @@ // showcase's tools — its operator-facing surface, served over the broker. // // Two of them, because one tool proves a tool can exist and two prove a module can have a surface. -import { tool } from "@novox/mesh-sdk/tools"; +import { registerModuleTools, type ToolDefinition } from "@novox/mesh-sdk/tools"; -tool({ - name: "showcase_greet", - description: "Greet somebody, and say which machine did it.", - input: { type: "object", properties: { who: { type: "string" } } }, - async run({ who }: { who?: string }) { - return { greeting: `hello ${who ?? "world"}`, from: process.env.MESH_NODE ?? "somewhere" }; - }, -}); +export function getShowcaseTools(env: NodeJS.ProcessEnv = process.env): ToolDefinition[] { + return [ + { + name: "showcase_greet", + description: "Greet somebody, and say which machine did it.", + input: { type: "object", properties: { who: { type: "string" } } }, + async run(args: Readonly>) { + const who = typeof args.who === "string" && args.who !== "" ? args.who : "world"; + return { greeting: `hello ${who}`, from: env.MESH_NODE ?? "somewhere" }; + }, + }, + { + name: "showcase_state", + description: "What this module was configured with, so a test can read it back.", + input: { type: "object", properties: {} }, + async run() { + return { + greeting: env.SHOWCASE_GREETING ?? "", + database: env.SHOWCASE_DATABASE ?? "", + }; + }, + }, + ]; +} -tool({ - name: "showcase_state", - description: "What this module was configured with, so a test can read it back.", - input: { type: "object", properties: {} }, - async run() { - return { - greeting: process.env.SHOWCASE_GREETING ?? "", - database: process.env.SHOWCASE_DATABASE ?? "", - }; - }, -}); +registerModuleTools("showcase", (env) => getShowcaseTools(env)); diff --git a/modules/slack/cmd/slack-tools/manifest_test.go b/modules/slack/cmd/slack-tools/manifest_test.go index b3a90e5d..01c43d74 100644 --- a/modules/slack/cmd/slack-tools/manifest_test.go +++ b/modules/slack/cmd/slack-tools/manifest_test.go @@ -24,11 +24,13 @@ type manifest struct { Claims []any `json:"claims"` Seats []any `json:"seats"` Shell []any `json:"shell"` - Contributions []struct { - Seat string `json:"seat"` - Kind string `json:"kind"` - Content string `json:"content"` - } `json:"contributions"` + Moved bool `json:"moved"` + Contributions []any `json:"contributions"` + Items []struct { + Collection string `json:"collection"` + Name string `json:"name"` + Template string `json:"template"` + } `json:"items"` Environment any `json:"environment"` Build struct { Artifacts []map[string]any `json:"artifacts"` @@ -72,15 +74,12 @@ func TestItInstallsNothingAndOwnsNoFile(t *testing.T) { func TestTheOneContributionIsTheStartAndTheWindowRules(t *testing.T) { m, _ := readManifest(t) - if len(m.Contributions) != 1 { - t.Fatalf("%+v", m.Contributions) - } - c := m.Contributions[0] - if c.Seat != "node-display-session" || c.Kind != "config" || !strings.HasSuffix(c.Content, "\n") { - t.Fatalf("%+v", c) + content := displayLines(t, m) + if !strings.HasSuffix(content, "\n") { + t.Fatalf("%q", content) } var execs, rules []string - for _, l := range strings.Split(c.Content, "\n") { + for _, l := range strings.Split(content, "\n") { l = strings.TrimSpace(l) switch { case strings.HasPrefix(l, "exec"): @@ -112,7 +111,7 @@ func TestTheOneContributionIsTheStartAndTheWindowRules(t *testing.T) { } // The placed lines start Slack once: the fake i3 configuration below holds exactly this content. f := newFake(t) - f.write(testHome+"/"+i3Main, "exec --no-startup-id dex --autostart --environment i3\n# slack\n"+c.Content) + f.write(testHome+"/"+i3Main, "exec --no-startup-id dex --autostart --environment i3\n# slack\n"+content) if s := f.starts(); len(s) != 1 || !s[0].Mine { t.Fatalf("%+v", s) } @@ -139,7 +138,7 @@ func TestTheToolsAgreeWithTheManifest(t *testing.T) { for name := range served { t.Errorf("the bundle serves %s, which module.json does not list", name) } - if len(m.Build.Artifacts) != 1 { + if len(m.Build.Artifacts) != 2 { // the tools bundle and the archive of its template t.Fatalf("%v", m.Build.Artifacts) } b := m.Build.Artifacts[0] @@ -154,3 +153,18 @@ func TestTheToolsAgreeWithTheManifest(t *testing.T) { } } } + +// displayLines is the module's lines for the window manager: the text of the template of its one item of the +// collection contribution:node-display-session:config (design 57 L3, issue 848). +func displayLines(t *testing.T, m manifest) string { + t.Helper() + if !m.Moved || len(m.Contributions) != 0 || len(m.Items) != 1 || + m.Items[0].Collection != "contribution:node-display-session:config" { + t.Fatalf("moved %v, contributions %v, items %+v", m.Moved, m.Contributions, m.Items) + } + text, err := os.ReadFile(filepath.Join("..", "..", m.Items[0].Template)) + if err != nil { + t.Fatal(err) + } + return string(text) +} diff --git a/modules/slack/cmd/slack-tools/slack_test.go b/modules/slack/cmd/slack-tools/slack_test.go index 15837ea9..04296b5f 100644 --- a/modules/slack/cmd/slack-tools/slack_test.go +++ b/modules/slack/cmd/slack-tools/slack_test.go @@ -109,7 +109,7 @@ func newSlack(t *testing.T) (*fake, map[string]string) { func contribution(t *testing.T) string { m, _ := readManifest(t) - return m.Contributions[0].Content + return displayLines(t, m) } func noSecrets(t *testing.T, v any) string { diff --git a/modules/slack/files/display-session b/modules/slack/files/display-session new file mode 100644 index 00000000..01660f84 --- /dev/null +++ b/modules/slack/files/display-session @@ -0,0 +1,11 @@ +# Slack (module slack, novox/hq ADR 0208, ADR 0212). Owned by the mesh: replaced at every push. +# Its one start: at login, to the tray (-s), with the arguments of the package's own desktop entry. +# No XDG autostart entry starts it as well; slack_check names one if it appears. +exec --no-startup-id /usr/bin/slack --gtk-version=3 -s +# The chat window. Slack owns four X windows, and the only one i3 manages has the class "slack" in +# lower case; the three of class "Slack" (the packaged entry's StartupWMClass) are its unmanaged +# helpers and its tray icon, so a rule on class="Slack" matches nothing. window_role keeps a call or +# screen-share window out of the rules. $ws3 is i3's, in scope here. +for_window [class="(?i)^slack$" window_role="browser-window"] move to workspace $ws3 +for_window [class="(?i)^slack$" window_role="browser-window"] floating disable +for_window [class="(?i)^slack$" window_role="browser-window"] border pixel 2 diff --git a/modules/slack/module.json b/modules/slack/module.json index b0b6d760..9feeedb3 100644 --- a/modules/slack/module.json +++ b/modules/slack/module.json @@ -1,6 +1,7 @@ { "module": "slack", "version": "1", + "moved": true, "requires": [ "x11-display" ], @@ -22,14 +23,19 @@ "loads": [ "slack-tools" ] + }, + { + "name": "files", + "kind": "archive", + "from": "files" } ] }, - "contributions": [ + "items": [ { - "seat": "node-display-session", - "kind": "config", - "content": "# Slack (module slack, novox/hq ADR 0208, ADR 0212). Owned by the mesh: replaced at every push.\n# Its one start: at login, to the tray (-s), with the arguments of the package's own desktop entry.\n# No XDG autostart entry starts it as well; slack_check names one if it appears.\nexec --no-startup-id /usr/bin/slack --gtk-version=3 -s\n# The chat window. Slack owns four X windows, and the only one i3 manages has the class \"slack\" in\n# lower case; the three of class \"Slack\" (the packaged entry's StartupWMClass) are its unmanaged\n# helpers and its tray icon, so a rule on class=\"Slack\" matches nothing. window_role keeps a call or\n# screen-share window out of the rules. $ws3 is i3's, in scope here.\nfor_window [class=\"(?i)^slack$\" window_role=\"browser-window\"] move to workspace $ws3\nfor_window [class=\"(?i)^slack$\" window_role=\"browser-window\"] floating disable\nfor_window [class=\"(?i)^slack$\" window_role=\"browser-window\"] border pixel 2\n" + "collection": "contribution:node-display-session:config", + "name": "display-session", + "template": "files/display-session" } ] } diff --git a/modules/ssh-client/cmd/ssh-client-tools/ssh_test.go b/modules/ssh-client/cmd/ssh-client-tools/ssh_test.go index 99f26865..7100c3a6 100644 --- a/modules/ssh-client/cmd/ssh-client-tools/ssh_test.go +++ b/modules/ssh-client/cmd/ssh-client-tools/ssh_test.go @@ -8,6 +8,7 @@ import ( "reflect" "sort" "strings" + "sync" "testing" "time" ) @@ -28,12 +29,22 @@ type call struct { type fake struct { answer func(c call) Ran + mu sync.Mutex calls []call } +// recorded returns a copy of the calls so far; scans call run from parallel goroutines. +func (f *fake) recorded() []call { + f.mu.Lock() + defer f.mu.Unlock() + return append([]call(nil), f.calls...) +} + func (f *fake) run(_ context.Context, env []string, name string, args ...string) Ran { c := call{env, name, args} + f.mu.Lock() f.calls = append(f.calls, c) + f.mu.Unlock() if f.answer == nil { return Ran{Status: 1} } @@ -152,7 +163,7 @@ func TestKeysAreDescribedByTheirPublicHalfAndAPassphraseIsAskedNotRead(t *testin if k := by["stray"]; !k.PublicMissing || k.OfferedBy != "" || k.Fingerprint != edFP { t.Errorf("stray: %+v", k) } - for _, c := range f.calls { + for _, c := range f.recorded() { if c.name == "ssh-keygen" && c.args[0] == "-y" && !reflect.DeepEqual(c.args[:3], []string{"-y", "-P", ""}) { t.Errorf("a passphrase check could prompt: %v", c.args) } @@ -263,7 +274,7 @@ func TestKnownHostComparesWhatIsKnownWithWhatIsOffered(t *testing.T) { t.Fatalf("not appended: %s", after) } sawRemove := false - for _, c := range f.calls { + for _, c := range f.recorded() { if c.name == "ssh-keygen" && reflect.DeepEqual(c.args[:2], []string{"-R", "ace.internal"}) { sawRemove = true } @@ -303,7 +314,7 @@ func TestTestSaysHowItAuthenticatedOrWhyNot(t *testing.T) { if err != nil || got["ok"] != true || got["method"] != "publickey" || got["connected_to"] != "10.0.0.2:22" { t.Fatalf("%v %v", got, err) } - args := strings.Join(f.calls[0].args, " ") + args := strings.Join(f.recorded()[0].args, " ") if !strings.Contains(args, "BatchMode=yes") || !strings.Contains(args, "StrictHostKeyChecking=yes") || !strings.HasSuffix(args, "ace true") { t.Fatalf("not a batch test: %s", args) } @@ -388,7 +399,7 @@ func TestTheManifestsRegionIsWhatSshReads(t *testing.T) { var region string for _, r := range m.Resources { if r["id"] == "config" { - region = r["content"].(string) + region = templateText(t, r) if r["into"] != "block" || r["at"] != "start" { t.Fatalf("the region is not written into the start: %v", r) } diff --git a/modules/ssh-client/cmd/ssh-client-tools/templates_test.go b/modules/ssh-client/cmd/ssh-client-tools/templates_test.go new file mode 100644 index 00000000..7d0718b5 --- /dev/null +++ b/modules/ssh-client/cmd/ssh-client-tools/templates_test.go @@ -0,0 +1,64 @@ +package main + +import ( + "encoding/json" + "os" + "path/filepath" + "testing" +) + +// The module's directory, where its manifest and its files/ are. +const moduleDir = "../.." + +// templateText is the text a file resource places on the machine: its template's, in the module's files/ +// (novox/hq ADR 0349 §2). A resource that names no template, or a template that is not there, fails the test. +func templateText(t *testing.T, r map[string]any) string { + t.Helper() + name, _ := r["template"].(string) + if name == "" { + t.Fatalf("resource %v names no template", r["id"]) + } + raw, err := os.ReadFile(filepath.Join(moduleDir, filepath.FromSlash(name))) + if err != nil { + t.Fatalf("resource %v names %s: %v", r["id"], name, err) + } + return string(raw) +} + +// No file resource carries its text inline: each names its template in files/, and the build packs files/ +// into the one archive the controller reads the templates back from (novox/hq ADR 0349 §2). +func TestEveryFileTakesItsTextFromATemplate(t *testing.T) { + raw, err := os.ReadFile(filepath.Join(moduleDir, "module.json")) + if err != nil { + t.Fatal(err) + } + var m struct { + Resources []map[string]any `json:"resources"` + Build struct { + Artifacts []map[string]any `json:"artifacts"` + } `json:"build"` + } + if err := json.Unmarshal(raw, &m); err != nil { + t.Fatal(err) + } + for _, r := range m.Resources { + if _, inline := r["content"]; inline { + t.Errorf("resource %v keeps its text inline: keep it in files/ and name it as the template", r["id"]) + } + if r["type"] == "file" { + templateText(t, r) + } + } + packs := 0 + for _, a := range m.Build.Artifacts { + if a["kind"] == "archive" && a["from"] == "files" { + packs++ + if a["name"] != "files" { + t.Errorf("the templates archive is named %v, not files", a["name"]) + } + } + } + if packs != 1 { + t.Errorf("the build packs files/ into %d archives, not one", packs) + } +} diff --git a/modules/ssh-client/files/config b/modules/ssh-client/files/config new file mode 100644 index 00000000..dac07fb6 --- /dev/null +++ b/modules/ssh-client/files/config @@ -0,0 +1,6 @@ +# The mesh's region (module ssh-client, novox/hq research 027/03). It comes first because ssh +# takes the first value it finds for each option. It brings in ~/.ssh/config.d/ in name order: +# 00-mesh, the mesh's Host per machine, then each file another module places there. Replaced at +# every push. Everything below it is yours, kept as you wrote it, and applies to every host the +# files above leave unsettled. +Include ~/.ssh/config.d/* diff --git a/modules/ssh-client/module.json b/modules/ssh-client/module.json index e6ac8f1a..5bc3c2e2 100644 --- a/modules/ssh-client/module.json +++ b/modules/ssh-client/module.json @@ -44,7 +44,7 @@ "mode": "0600", "into": "block", "at": "start", - "content": "# The mesh's region (module ssh-client, novox/hq research 027/03). It comes first because ssh\n# takes the first value it finds for each option. It brings in ~/.ssh/config.d/ in name order:\n# 00-mesh, the mesh's Host per machine, then each file another module places there. Replaced at\n# every push. Everything below it is yours, kept as you wrote it, and applies to every host the\n# files above leave unsettled.\nInclude ~/.ssh/config.d/*\n" + "template": "files/config" } ], "facts": { @@ -66,6 +66,11 @@ "loads": [ "ssh-client-tools" ] + }, + { + "name": "files", + "kind": "archive", + "from": "files" } ] } diff --git a/modules/sshd/files/10-mesh.conf b/modules/sshd/files/10-mesh.conf new file mode 100644 index 00000000..86cd883a --- /dev/null +++ b/modules/sshd/files/10-mesh.conf @@ -0,0 +1,10 @@ +# Managed by the mesh (module sshd). Replaced on every push; edit the catalogue instead. +Port 22 +PermitRootLogin no +PasswordAuthentication no +PubkeyAuthentication yes +KbdInteractiveAuthentication no +UsePAM yes +X11Forwarding no +PrintMotd no +AcceptEnv LANG LC_* diff --git a/modules/sshd/module.json b/modules/sshd/module.json index a675636b..066e058d 100644 --- a/modules/sshd/module.json +++ b/modules/sshd/module.json @@ -29,7 +29,7 @@ "type": "file", "path": "/etc/ssh/sshd_config.d/10-mesh.conf", "mode": "0644", - "content": "# Managed by the mesh (module sshd). Replaced on every push; edit the catalogue instead.\nPort 22\nPermitRootLogin no\nPasswordAuthentication no\nPubkeyAuthentication yes\nKbdInteractiveAuthentication no\nUsePAM yes\nX11Forwarding no\nPrintMotd no\nAcceptEnv LANG LC_*\n" + "template": "files/10-mesh.conf" }, { "id": "run", @@ -41,5 +41,14 @@ "config" ] } - ] + ], + "build": { + "artifacts": [ + { + "name": "files", + "kind": "archive", + "from": "files" + } + ] + } } diff --git a/modules/step-ca/files/config.json b/modules/step-ca/files/config.json new file mode 100644 index 00000000..9e26dfee --- /dev/null +++ b/modules/step-ca/files/config.json @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/modules/step-ca/files/init.env b/modules/step-ca/files/init.env new file mode 100644 index 00000000..31d0896c --- /dev/null +++ b/modules/step-ca/files/init.env @@ -0,0 +1,2 @@ +DOCKER_STEPCA_INIT_PASSWORD=${secret:password} +DOCKER_STEPCA_INIT_DNS_NAMES=${machine:at},${machine:name},localhost,127.0.0.1 diff --git a/modules/step-ca/module.json b/modules/step-ca/module.json index a391a8b4..ebb587db 100644 --- a/modules/step-ca/module.json +++ b/modules/step-ca/module.json @@ -57,7 +57,8 @@ "type": "file", "path": "${dir:mesh-state}/config.json", "mode": "0644", - "content": "{}", + "template": "files/config.json", + "takes-config": [], "merge": "json" }, { @@ -65,7 +66,7 @@ "type": "file", "path": "${dir:mesh-state}/init.env", "mode": "0600", - "content": "DOCKER_STEPCA_INIT_PASSWORD=${secret:password}\nDOCKER_STEPCA_INIT_DNS_NAMES=${machine:at},${machine:name},localhost,127.0.0.1\n" + "template": "files/init.env" }, { "id": "server", @@ -90,5 +91,14 @@ ], "secrets-in-environment": "the entrypoint honours DOCKER_STEPCA_INIT_PASSWORD_FILE; convertible, awaiting a bed that proves it" } - ] + ], + "build": { + "artifacts": [ + { + "name": "files", + "kind": "archive", + "from": "files" + } + ] + } } diff --git a/modules/sudo/cmd/sudo-tools/manifest_test.go b/modules/sudo/cmd/sudo-tools/manifest_test.go index aba08c85..3d3bb454 100644 --- a/modules/sudo/cmd/sudo-tools/manifest_test.go +++ b/modules/sudo/cmd/sudo-tools/manifest_test.go @@ -34,7 +34,7 @@ func TestItDeclaresThePackageAndTheDropInSudoReads(t *testing.T) { } func TestTheDropInGrantsExactlyTheOperatorAccountAndParses(t *testing.T) { - content := manifest(t).resource(t, "operator")["content"].(string) + content := templateText(t, manifest(t).resource(t, "operator")) var rules []string for _, l := range strings.Split(content, "\n") { if l = strings.TrimSpace(l); l != "" && !strings.HasPrefix(l, "#") { diff --git a/modules/sudo/cmd/sudo-tools/templates_test.go b/modules/sudo/cmd/sudo-tools/templates_test.go new file mode 100644 index 00000000..7d0718b5 --- /dev/null +++ b/modules/sudo/cmd/sudo-tools/templates_test.go @@ -0,0 +1,64 @@ +package main + +import ( + "encoding/json" + "os" + "path/filepath" + "testing" +) + +// The module's directory, where its manifest and its files/ are. +const moduleDir = "../.." + +// templateText is the text a file resource places on the machine: its template's, in the module's files/ +// (novox/hq ADR 0349 §2). A resource that names no template, or a template that is not there, fails the test. +func templateText(t *testing.T, r map[string]any) string { + t.Helper() + name, _ := r["template"].(string) + if name == "" { + t.Fatalf("resource %v names no template", r["id"]) + } + raw, err := os.ReadFile(filepath.Join(moduleDir, filepath.FromSlash(name))) + if err != nil { + t.Fatalf("resource %v names %s: %v", r["id"], name, err) + } + return string(raw) +} + +// No file resource carries its text inline: each names its template in files/, and the build packs files/ +// into the one archive the controller reads the templates back from (novox/hq ADR 0349 §2). +func TestEveryFileTakesItsTextFromATemplate(t *testing.T) { + raw, err := os.ReadFile(filepath.Join(moduleDir, "module.json")) + if err != nil { + t.Fatal(err) + } + var m struct { + Resources []map[string]any `json:"resources"` + Build struct { + Artifacts []map[string]any `json:"artifacts"` + } `json:"build"` + } + if err := json.Unmarshal(raw, &m); err != nil { + t.Fatal(err) + } + for _, r := range m.Resources { + if _, inline := r["content"]; inline { + t.Errorf("resource %v keeps its text inline: keep it in files/ and name it as the template", r["id"]) + } + if r["type"] == "file" { + templateText(t, r) + } + } + packs := 0 + for _, a := range m.Build.Artifacts { + if a["kind"] == "archive" && a["from"] == "files" { + packs++ + if a["name"] != "files" { + t.Errorf("the templates archive is named %v, not files", a["name"]) + } + } + } + if packs != 1 { + t.Errorf("the build packs files/ into %d archives, not one", packs) + } +} diff --git a/modules/sudo/files/10-mesh-operator b/modules/sudo/files/10-mesh-operator new file mode 100644 index 00000000..997a49a6 --- /dev/null +++ b/modules/sudo/files/10-mesh-operator @@ -0,0 +1,9 @@ +# The mesh's (module sudo, novox/hq to-be 42, research 027): the operator account escalates +# without a prompt. The mesh's tools that act as root run `sudo -n` as this account and rely on it; +# until this file, every machine said so only in a line set by hand in /etc/sudoers. +# Written whole at every push: an edit here is overwritten. A file of this directory whose name +# holds a dot or ends in ~ is not read by sudo; this name holds neither. +# Every command run through sudo gets a terminal of its own (novox/hq ADR 0266): an agent the operator +# starts as the agent account through sudo cannot push keystrokes into the operator's shell (TIOCSTI). +Defaults use_pty +${machine:account} ALL=(ALL:ALL) NOPASSWD: ALL diff --git a/modules/sudo/module.json b/modules/sudo/module.json index 386deb14..364ce5fc 100644 --- a/modules/sudo/module.json +++ b/modules/sudo/module.json @@ -21,7 +21,7 @@ "type": "file", "path": "/etc/sudoers.d/10-mesh-operator", "mode": "0440", - "content": "# The mesh's (module sudo, novox/hq to-be 42, research 027): the operator account escalates\n# without a prompt. The mesh's tools that act as root run `sudo -n` as this account and rely on it;\n# until this file, every machine said so only in a line set by hand in /etc/sudoers.\n# Written whole at every push: an edit here is overwritten. A file of this directory whose name\n# holds a dot or ends in ~ is not read by sudo; this name holds neither.\n# Every command run through sudo gets a terminal of its own (novox/hq ADR 0266): an agent the operator\n# starts as the agent account through sudo cannot push keystrokes into the operator's shell (TIOCSTI).\nDefaults use_pty\n${machine:account} ALL=(ALL:ALL) NOPASSWD: ALL\n" + "template": "files/10-mesh-operator" } ], "build": { @@ -36,6 +36,11 @@ "loads": [ "sudo-tools" ] + }, + { + "name": "files", + "kind": "archive", + "from": "files" } ] } diff --git a/modules/supabase/files/analytics-env.env b/modules/supabase/files/analytics-env.env new file mode 100644 index 00000000..b1b877f0 --- /dev/null +++ b/modules/supabase/files/analytics-env.env @@ -0,0 +1,14 @@ +LOGFLARE_NODE_HOST=127.0.0.1 +DB_USERNAME=supabase_admin +DB_DATABASE=_supabase +DB_HOSTNAME=supabase-db +DB_PORT=5432 +DB_PASSWORD=${secret:postgres} +DB_SCHEMA=_analytics +LOGFLARE_API_KEY=${secret:logflare} +LOGFLARE_SINGLE_TENANT=true +LOGFLARE_SUPABASE_MODE=true +LOGFLARE_MIN_CLUSTER_SIZE=1 +POSTGRES_BACKEND_URL=postgresql://supabase_admin:${secret:postgres}@supabase-db:5432/_supabase +POSTGRES_BACKEND_SCHEMA=_analytics +LOGFLARE_FEATURE_FLAG_OVERRIDE=multibackend=true diff --git a/modules/supabase/files/analytics-reconcile.exs b/modules/supabase/files/analytics-reconcile.exs new file mode 100644 index 00000000..179776e0 --- /dev/null +++ b/modules/supabase/files/analytics-reconcile.exs @@ -0,0 +1,86 @@ +# Generated by the mesh. Do not edit: module supabase writes this file and replaces it at every push. +# +# logflare 1.4.0 copies two of its settings into its own database once and never reads them again: +# LOGFLARE_API_KEY becomes the default user's api_key when that user is created, and +# POSTGRES_BACKEND_URL becomes each source's backend when that source is created. A changed value +# reaches neither: a rotated key is refused, and a moved database is never written to (every +# source here pointed at db:5432, a name that resolves nowhere, so nothing was stored). +# +# Run by the start script before logflare starts, through logflare's own code (its Repo, its Users +# and Backends contexts and their changesets): it makes both copies say what the environment says. +# It changes nothing when they already do, says what it changed, never prints the key or a URL's +# password, and stops the start when it cannot finish. +fail = fn why -> + IO.puts(:stderr, "analytics: #{why}; not starting") + System.halt(1) +end + +say = fn what -> IO.puts(:stderr, "analytics: #{what}") end + +key = System.get_env("LOGFLARE_API_KEY") +url = System.get_env("POSTGRES_BACKEND_URL") +schema = System.get_env("POSTGRES_BACKEND_SCHEMA") + +if key in [nil, ""], do: fail.("LOGFLARE_API_KEY is not set, so the key logflare must accept is unknown") +if url in [nil, ""], do: fail.("POSTGRES_BACKEND_URL is not set, so where logflare must store logs is unknown") + +where = fn u -> + case URI.parse(to_string(u)) do + %URI{host: host, port: port, path: path} when is_binary(host) -> "#{host}:#{port}#{path}" + _ -> "an unreadable URL" + end +end + +Application.ensure_all_started(:ssl) + +outcome = + Ecto.Migrator.with_repo(Logflare.Repo, fn _repo -> + # Backends.update_source_backend_config restarts the source's pipeline after saving, and looks + # it up in this registry to do so; logflare is not running yet, so the registry is empty and + # there is nothing to restart. + {:ok, _} = Registry.start_link(keys: :unique, name: Logflare.Backends.SourceRegistry) + + case Logflare.SingleTenant.get_default_user() do + nil -> + say.("no default user yet; logflare creates it from LOGFLARE_API_KEY and POSTGRES_BACKEND_URL as it starts") + :ok + + user -> + if user.api_key == key and is_nil(user.old_api_key) do + say.("the default user's API key is LOGFLARE_API_KEY already") + else + # old_api_key is cleared too: logflare's UI can swap it back in, and the key replaced + # here may be one that leaked. + case Logflare.Users.update_user_all_fields(user, %{api_key: key, old_api_key: nil}) do + {:ok, _} -> say.("the default user's API key was replaced with LOGFLARE_API_KEY") + {:error, changeset} -> fail.("the default user's API key could not be replaced: #{inspect(changeset.errors)}") + end + end + + results = + for source <- Logflare.Sources.list_sources_by_user(user), + backend <- Logflare.Backends.list_source_backends(source), + backend.type == :postgres do + if backend.config.url == url and backend.config.schema == schema do + :same + else + case Logflare.Backends.update_source_backend_config(backend, %{url: url, schema: schema}) do + {:ok, _} -> + say.("source #{source.name} stored logs at #{where.(backend.config.url)} schema #{inspect(backend.config.schema)}; now at #{where.(url)} schema #{inspect(schema)}") + + {:error, changeset} -> + fail.("source #{source.name}'s backend could not be pointed at #{where.(url)}: #{inspect(changeset.errors)}") + end + end + end + + say.("#{Enum.count(results, &(&1 == :same))} source backend(s) already at #{where.(url)}") + + :ok + end + end) + +case outcome do + {:ok, :ok, _} -> :ok + other -> fail.("logflare's database could not be reconciled: #{inspect(other)}") +end diff --git a/modules/supabase/files/analytics-start.sh b/modules/supabase/files/analytics-start.sh new file mode 100644 index 00000000..bf39892b --- /dev/null +++ b/modules/supabase/files/analytics-start.sh @@ -0,0 +1,41 @@ +#!/bin/sh +# Generated by the mesh. Do not edit: module supabase writes this file and replaces it at every push. +# +# Two things logflare 1.4.0 does not do for itself, done here before it starts; either failing +# stops the start, with the reason as the last line of this container's log. +# +# 1. The API key never travels in a URL. logflare prints the whole URL of every request that fails, +# query string included, in the error report Plug.Cowboy writes ("Request: POST +# /api/logs?source_name=...&api_key=..."). That is an error, so no log level hides it while errors +# are logged at all (novox/hq issue 268). logflare reads the key from the x-api-key header as well, +# and the mesh's caller, vector, sends it there; this refuses to start logflare while the vector +# config it is given still passes the key in a URL: an analytics that does not start says why +# here, and one that leaks says nothing. +# +# 2. Its database says what its environment says. logflare copies LOGFLARE_API_KEY into its default +# user and POSTGRES_BACKEND_URL into every source's backend once, when it creates them, and never +# again; reconcile.exs makes both copies match the environment through logflare's own code, after +# its migrations and before it starts. That is what lets the mesh rotate the key by starting +# logflare again, and what moves the sources' backend when the database's address or password +# changes. +set -e +conf=/run/logflare/vector.yml +if [ ! -r "$conf" ]; then + echo "analytics: $conf is not there to check, so whether the API key travels in a URL is unknown; not starting (novox/hq issue 268)" >&2 + exit 1 +fi +if grep -v '^[[:space:]]*#' "$conf" | grep -q 'api_key='; then + echo "analytics: $conf passes the API key in a URL, and logflare prints a failed request's URL; not starting (novox/hq issue 268)" >&2 + exit 1 +fi +reconcile=/run/logflare/reconcile.exs +if [ ! -r "$reconcile" ]; then + echo "analytics: $reconcile is not there, so whether logflare's stored key and backends match its environment is unknown; not starting" >&2 + exit 1 +fi +cd /opt/app/rel/logflare/bin +# The tables reconcile.exs reads exist only once the migrations have run; run.sh runs them again, +# which finds nothing left to do. +./logflare eval Logflare.Release.migrate +./logflare eval 'Code.eval_file("/run/logflare/reconcile.exs")' +exec sh run.sh diff --git a/modules/supabase/files/db-init/jwt.sql b/modules/supabase/files/db-init/jwt.sql new file mode 100644 index 00000000..90d88222 --- /dev/null +++ b/modules/supabase/files/db-init/jwt.sql @@ -0,0 +1,2 @@ +ALTER DATABASE postgres SET "app.settings.jwt_secret" TO '${secret:jwt}'; +ALTER DATABASE postgres SET "app.settings.jwt_exp" TO '3600'; diff --git a/modules/supabase/files/db-init/logs.sql b/modules/supabase/files/db-init/logs.sql new file mode 100644 index 00000000..255c0f40 --- /dev/null +++ b/modules/supabase/files/db-init/logs.sql @@ -0,0 +1,6 @@ +\set pguser `echo "$POSTGRES_USER"` + +\c _supabase +create schema if not exists _analytics; +alter schema _analytics owner to :pguser; +\c postgres diff --git a/modules/supabase/files/db-init/pooler.sql b/modules/supabase/files/db-init/pooler.sql new file mode 100644 index 00000000..162c5b96 --- /dev/null +++ b/modules/supabase/files/db-init/pooler.sql @@ -0,0 +1,6 @@ +\set pguser `echo "$POSTGRES_USER"` + +\c _supabase +create schema if not exists _supavisor; +alter schema _supavisor owner to :pguser; +\c postgres diff --git a/modules/supabase/files/db-init/realtime.sql b/modules/supabase/files/db-init/realtime.sql new file mode 100644 index 00000000..4d4b9ffb --- /dev/null +++ b/modules/supabase/files/db-init/realtime.sql @@ -0,0 +1,4 @@ +\set pguser `echo "$POSTGRES_USER"` + +create schema if not exists _realtime; +alter schema _realtime owner to :pguser; diff --git a/modules/supabase/files/db-init/roles.sql b/modules/supabase/files/db-init/roles.sql new file mode 100644 index 00000000..2a05c0b2 --- /dev/null +++ b/modules/supabase/files/db-init/roles.sql @@ -0,0 +1,7 @@ +\set pgpass `echo "$POSTGRES_PASSWORD"` + +ALTER USER authenticator WITH PASSWORD :'pgpass'; +ALTER USER pgbouncer WITH PASSWORD :'pgpass'; +ALTER USER supabase_auth_admin WITH PASSWORD :'pgpass'; +ALTER USER supabase_functions_admin WITH PASSWORD :'pgpass'; +ALTER USER supabase_storage_admin WITH PASSWORD :'pgpass'; diff --git a/modules/supabase/files/db-init/supabase.sql b/modules/supabase/files/db-init/supabase.sql new file mode 100644 index 00000000..6236ae1b --- /dev/null +++ b/modules/supabase/files/db-init/supabase.sql @@ -0,0 +1,3 @@ +\set pguser `echo "$POSTGRES_USER"` + +CREATE DATABASE _supabase WITH OWNER :pguser; diff --git a/modules/supabase/files/db-init/webhooks.sql b/modules/supabase/files/db-init/webhooks.sql new file mode 100644 index 00000000..5837b861 --- /dev/null +++ b/modules/supabase/files/db-init/webhooks.sql @@ -0,0 +1,208 @@ +BEGIN; + -- Create pg_net extension + CREATE EXTENSION IF NOT EXISTS pg_net SCHEMA extensions; + -- Create supabase_functions schema + CREATE SCHEMA supabase_functions AUTHORIZATION supabase_admin; + GRANT USAGE ON SCHEMA supabase_functions TO postgres, anon, authenticated, service_role; + ALTER DEFAULT PRIVILEGES IN SCHEMA supabase_functions GRANT ALL ON TABLES TO postgres, anon, authenticated, service_role; + ALTER DEFAULT PRIVILEGES IN SCHEMA supabase_functions GRANT ALL ON FUNCTIONS TO postgres, anon, authenticated, service_role; + ALTER DEFAULT PRIVILEGES IN SCHEMA supabase_functions GRANT ALL ON SEQUENCES TO postgres, anon, authenticated, service_role; + -- supabase_functions.migrations definition + CREATE TABLE supabase_functions.migrations ( + version text PRIMARY KEY, + inserted_at timestamptz NOT NULL DEFAULT NOW() + ); + -- Initial supabase_functions migration + INSERT INTO supabase_functions.migrations (version) VALUES ('initial'); + -- supabase_functions.hooks definition + CREATE TABLE supabase_functions.hooks ( + id bigserial PRIMARY KEY, + hook_table_id integer NOT NULL, + hook_name text NOT NULL, + created_at timestamptz NOT NULL DEFAULT NOW(), + request_id bigint + ); + CREATE INDEX supabase_functions_hooks_request_id_idx ON supabase_functions.hooks USING btree (request_id); + CREATE INDEX supabase_functions_hooks_h_table_id_h_name_idx ON supabase_functions.hooks USING btree (hook_table_id, hook_name); + COMMENT ON TABLE supabase_functions.hooks IS 'Supabase Functions Hooks: Audit trail for triggered hooks.'; + CREATE FUNCTION supabase_functions.http_request() + RETURNS trigger + LANGUAGE plpgsql + AS $function$ + DECLARE + request_id bigint; + payload jsonb; + url text := TG_ARGV[0]::text; + method text := TG_ARGV[1]::text; + headers jsonb DEFAULT '{}'::jsonb; + params jsonb DEFAULT '{}'::jsonb; + timeout_ms integer DEFAULT 1000; + BEGIN + IF url IS NULL OR url = 'null' THEN + RAISE EXCEPTION 'url argument is missing'; + END IF; + + IF method IS NULL OR method = 'null' THEN + RAISE EXCEPTION 'method argument is missing'; + END IF; + + IF TG_ARGV[2] IS NULL OR TG_ARGV[2] = 'null' THEN + headers = '{"Content-Type": "application/json"}'::jsonb; + ELSE + headers = TG_ARGV[2]::jsonb; + END IF; + + IF TG_ARGV[3] IS NULL OR TG_ARGV[3] = 'null' THEN + params = '{}'::jsonb; + ELSE + params = TG_ARGV[3]::jsonb; + END IF; + + IF TG_ARGV[4] IS NULL OR TG_ARGV[4] = 'null' THEN + timeout_ms = 1000; + ELSE + timeout_ms = TG_ARGV[4]::integer; + END IF; + + CASE + WHEN method = 'GET' THEN + SELECT http_get INTO request_id FROM net.http_get( + url, + params, + headers, + timeout_ms + ); + WHEN method = 'POST' THEN + payload = jsonb_build_object( + 'old_record', OLD, + 'record', NEW, + 'type', TG_OP, + 'table', TG_TABLE_NAME, + 'schema', TG_TABLE_SCHEMA + ); + + SELECT http_post INTO request_id FROM net.http_post( + url, + payload, + params, + headers, + timeout_ms + ); + ELSE + RAISE EXCEPTION 'method argument % is invalid', method; + END CASE; + + INSERT INTO supabase_functions.hooks + (hook_table_id, hook_name, request_id) + VALUES + (TG_RELID, TG_NAME, request_id); + + RETURN NEW; + END + $function$; + -- Supabase super admin + DO + $$ + BEGIN + IF NOT EXISTS ( + SELECT 1 + FROM pg_roles + WHERE rolname = 'supabase_functions_admin' + ) + THEN + CREATE USER supabase_functions_admin NOINHERIT CREATEROLE LOGIN NOREPLICATION; + END IF; + END + $$; + GRANT ALL PRIVILEGES ON SCHEMA supabase_functions TO supabase_functions_admin; + GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA supabase_functions TO supabase_functions_admin; + GRANT ALL PRIVILEGES ON ALL SEQUENCES IN SCHEMA supabase_functions TO supabase_functions_admin; + ALTER USER supabase_functions_admin SET search_path = "supabase_functions"; + ALTER table "supabase_functions".migrations OWNER TO supabase_functions_admin; + ALTER table "supabase_functions".hooks OWNER TO supabase_functions_admin; + ALTER function "supabase_functions".http_request() OWNER TO supabase_functions_admin; + GRANT supabase_functions_admin TO postgres; + -- Remove unused supabase_pg_net_admin role + DO + $$ + BEGIN + IF EXISTS ( + SELECT 1 + FROM pg_roles + WHERE rolname = 'supabase_pg_net_admin' + ) + THEN + REASSIGN OWNED BY supabase_pg_net_admin TO supabase_admin; + DROP OWNED BY supabase_pg_net_admin; + DROP ROLE supabase_pg_net_admin; + END IF; + END + $$; + -- pg_net grants when extension is already enabled + DO + $$ + BEGIN + IF EXISTS ( + SELECT 1 + FROM pg_extension + WHERE extname = 'pg_net' + ) + THEN + GRANT USAGE ON SCHEMA net TO supabase_functions_admin, postgres, anon, authenticated, service_role; + ALTER function net.http_get(url text, params jsonb, headers jsonb, timeout_milliseconds integer) SECURITY DEFINER; + ALTER function net.http_post(url text, body jsonb, params jsonb, headers jsonb, timeout_milliseconds integer) SECURITY DEFINER; + ALTER function net.http_get(url text, params jsonb, headers jsonb, timeout_milliseconds integer) SET search_path = net; + ALTER function net.http_post(url text, body jsonb, params jsonb, headers jsonb, timeout_milliseconds integer) SET search_path = net; + REVOKE ALL ON FUNCTION net.http_get(url text, params jsonb, headers jsonb, timeout_milliseconds integer) FROM PUBLIC; + REVOKE ALL ON FUNCTION net.http_post(url text, body jsonb, params jsonb, headers jsonb, timeout_milliseconds integer) FROM PUBLIC; + GRANT EXECUTE ON FUNCTION net.http_get(url text, params jsonb, headers jsonb, timeout_milliseconds integer) TO supabase_functions_admin, postgres, anon, authenticated, service_role; + GRANT EXECUTE ON FUNCTION net.http_post(url text, body jsonb, params jsonb, headers jsonb, timeout_milliseconds integer) TO supabase_functions_admin, postgres, anon, authenticated, service_role; + END IF; + END + $$; + -- Event trigger for pg_net + CREATE OR REPLACE FUNCTION extensions.grant_pg_net_access() + RETURNS event_trigger + LANGUAGE plpgsql + AS $$ + BEGIN + IF EXISTS ( + SELECT 1 + FROM pg_event_trigger_ddl_commands() AS ev + JOIN pg_extension AS ext + ON ev.objid = ext.oid + WHERE ext.extname = 'pg_net' + ) + THEN + GRANT USAGE ON SCHEMA net TO supabase_functions_admin, postgres, anon, authenticated, service_role; + ALTER function net.http_get(url text, params jsonb, headers jsonb, timeout_milliseconds integer) SECURITY DEFINER; + ALTER function net.http_post(url text, body jsonb, params jsonb, headers jsonb, timeout_milliseconds integer) SECURITY DEFINER; + ALTER function net.http_get(url text, params jsonb, headers jsonb, timeout_milliseconds integer) SET search_path = net; + ALTER function net.http_post(url text, body jsonb, params jsonb, headers jsonb, timeout_milliseconds integer) SET search_path = net; + REVOKE ALL ON FUNCTION net.http_get(url text, params jsonb, headers jsonb, timeout_milliseconds integer) FROM PUBLIC; + REVOKE ALL ON FUNCTION net.http_post(url text, body jsonb, params jsonb, headers jsonb, timeout_milliseconds integer) FROM PUBLIC; + GRANT EXECUTE ON FUNCTION net.http_get(url text, params jsonb, headers jsonb, timeout_milliseconds integer) TO supabase_functions_admin, postgres, anon, authenticated, service_role; + GRANT EXECUTE ON FUNCTION net.http_post(url text, body jsonb, params jsonb, headers jsonb, timeout_milliseconds integer) TO supabase_functions_admin, postgres, anon, authenticated, service_role; + END IF; + END; + $$; + COMMENT ON FUNCTION extensions.grant_pg_net_access IS 'Grants access to pg_net'; + DO + $$ + BEGIN + IF NOT EXISTS ( + SELECT 1 + FROM pg_event_trigger + WHERE evtname = 'issue_pg_net_access' + ) THEN + CREATE EVENT TRIGGER issue_pg_net_access ON ddl_command_end WHEN TAG IN ('CREATE EXTENSION') + EXECUTE PROCEDURE extensions.grant_pg_net_access(); + END IF; + END + $$; + INSERT INTO supabase_functions.migrations (version) VALUES ('20210809183423_update_grants'); + ALTER function supabase_functions.http_request() SECURITY DEFINER; + ALTER function supabase_functions.http_request() SET search_path = supabase_functions; + REVOKE ALL ON FUNCTION supabase_functions.http_request() FROM PUBLIC; + GRANT EXECUTE ON FUNCTION supabase_functions.http_request() TO postgres, anon, authenticated, service_role; +COMMIT; diff --git a/modules/supabase/files/functions-env.env b/modules/supabase/files/functions-env.env new file mode 100644 index 00000000..5eade5e6 --- /dev/null +++ b/modules/supabase/files/functions-env.env @@ -0,0 +1,6 @@ +JWT_SECRET=${secret:jwt} +SUPABASE_URL=http://supabase-kong:8000 +SUPABASE_ANON_KEY=${secret:anon-key} +SUPABASE_SERVICE_ROLE_KEY=${secret:service-role-key} +SUPABASE_DB_URL=postgresql://postgres:${secret:postgres}@supabase-db:5432/postgres +VERIFY_JWT=false diff --git a/modules/supabase/files/functions-main/index.ts b/modules/supabase/files/functions-main/index.ts new file mode 100644 index 00000000..a094010b --- /dev/null +++ b/modules/supabase/files/functions-main/index.ts @@ -0,0 +1,94 @@ +import { serve } from 'https://deno.land/std@0.131.0/http/server.ts' +import * as jose from 'https://deno.land/x/jose@v4.14.4/index.ts' + +console.log('main function started') + +const JWT_SECRET = Deno.env.get('JWT_SECRET') +const VERIFY_JWT = Deno.env.get('VERIFY_JWT') === 'true' + +function getAuthToken(req: Request) { + const authHeader = req.headers.get('authorization') + if (!authHeader) { + throw new Error('Missing authorization header') + } + const [bearer, token] = authHeader.split(' ') + if (bearer !== 'Bearer') { + throw new Error(`Auth header is not 'Bearer {token}'`) + } + return token +} + +async function verifyJWT(jwt: string): Promise { + const encoder = new TextEncoder() + const secretKey = encoder.encode(JWT_SECRET) + try { + await jose.jwtVerify(jwt, secretKey) + } catch (err) { + console.error(err) + return false + } + return true +} + +serve(async (req: Request) => { + if (req.method !== 'OPTIONS' && VERIFY_JWT) { + try { + const token = getAuthToken(req) + const isValidJWT = await verifyJWT(token) + + if (!isValidJWT) { + return new Response(JSON.stringify({ msg: 'Invalid JWT' }), { + status: 401, + headers: { 'Content-Type': 'application/json' }, + }) + } + } catch (e) { + console.error(e) + return new Response(JSON.stringify({ msg: e.toString() }), { + status: 401, + headers: { 'Content-Type': 'application/json' }, + }) + } + } + + const url = new URL(req.url) + const { pathname } = url + const path_parts = pathname.split('/') + const service_name = path_parts[1] + + if (!service_name || service_name === '') { + const error = { msg: 'missing function name in request' } + return new Response(JSON.stringify(error), { + status: 400, + headers: { 'Content-Type': 'application/json' }, + }) + } + + const servicePath = `/home/deno/functions/${service_name}` + console.error(`serving the request with ${servicePath}`) + + const memoryLimitMb = 150 + const workerTimeoutMs = 1 * 60 * 1000 + const noModuleCache = false + const importMapPath = null + const envVarsObj = Deno.env.toObject() + const envVars = Object.keys(envVarsObj).map((k) => [k, envVarsObj[k]]) + + try { + const worker = await EdgeRuntime.userWorkers.create({ + servicePath, + memoryLimitMb, + workerTimeoutMs, + noModuleCache, + importMapPath, + envVars, + }) + return await worker.fetch(req) + } catch (e) { + const error = { msg: e.toString() } + return new Response(JSON.stringify(error), { + status: 500, + headers: { 'Content-Type': 'application/json' }, + }) + } +}) diff --git a/modules/supabase/files/gotrue.env b/modules/supabase/files/gotrue.env new file mode 100644 index 00000000..f29f778a --- /dev/null +++ b/modules/supabase/files/gotrue.env @@ -0,0 +1,28 @@ +GOTRUE_API_HOST=0.0.0.0 +GOTRUE_API_PORT=9999 +API_EXTERNAL_URL=https://${bound:route:name} +GOTRUE_DB_DRIVER=postgres +GOTRUE_DB_DATABASE_URL=postgres://supabase_auth_admin:${secret:postgres}@supabase-db:5432/postgres +GOTRUE_SITE_URL=https://${bound:route:name} +GOTRUE_URI_ALLOW_LIST= +GOTRUE_DISABLE_SIGNUP=true +GOTRUE_JWT_ADMIN_ROLES=service_role +GOTRUE_JWT_AUD=authenticated +GOTRUE_JWT_DEFAULT_GROUP_NAME=authenticated +GOTRUE_JWT_EXP=3600 +GOTRUE_JWT_SECRET=${secret:jwt} +GOTRUE_EXTERNAL_EMAIL_ENABLED=false +GOTRUE_EXTERNAL_ANONYMOUS_USERS_ENABLED=false +GOTRUE_MAILER_AUTOCONFIRM=false +GOTRUE_SMTP_ADMIN_EMAIL=admin@example.com +GOTRUE_SMTP_HOST=supabase-mail +GOTRUE_SMTP_PORT=2500 +GOTRUE_SMTP_USER=fake_mail_user +GOTRUE_SMTP_PASS=fake_mail_password +GOTRUE_SMTP_SENDER_NAME=fake_sender +GOTRUE_MAILER_URLPATHS_INVITE=/auth/v1/verify +GOTRUE_MAILER_URLPATHS_CONFIRMATION=/auth/v1/verify +GOTRUE_MAILER_URLPATHS_RECOVERY=/auth/v1/verify +GOTRUE_MAILER_URLPATHS_EMAIL_CHANGE=/auth/v1/verify +GOTRUE_EXTERNAL_PHONE_ENABLED=false +GOTRUE_SMS_AUTOCONFIRM=true diff --git a/modules/supabase/files/kong.yml b/modules/supabase/files/kong.yml new file mode 100644 index 00000000..e13bdb24 --- /dev/null +++ b/modules/supabase/files/kong.yml @@ -0,0 +1,242 @@ +# Written by the mesh (modules/supabase): the gateway's declarative config, credentials rendered in. +_format_version: '2.1' +_transform: true + +### +### Consumers / Users +### +consumers: + - username: DASHBOARD + - username: anon + keyauth_credentials: + - key: "${secret:anon-key}" + - username: service_role + keyauth_credentials: + - key: "${secret:service-role-key}" + +### +### Access Control List +### +acls: + - consumer: anon + group: anon + - consumer: service_role + group: admin + +### +### Dashboard credentials +### +basicauth_credentials: + - consumer: DASHBOARD + username: "${secret:dashboard-user}" + password: "${secret:dashboard}" + +### +### API Routes +### +services: + ## Open Auth routes + - name: auth-v1-open + url: http://supabase-auth:9999/verify + routes: + - name: auth-v1-open + strip_path: true + paths: + - /auth/v1/verify + plugins: + - name: cors + - name: auth-v1-open-callback + url: http://supabase-auth:9999/callback + routes: + - name: auth-v1-open-callback + strip_path: true + paths: + - /auth/v1/callback + plugins: + - name: cors + - name: auth-v1-open-authorize + url: http://supabase-auth:9999/authorize + routes: + - name: auth-v1-open-authorize + strip_path: true + paths: + - /auth/v1/authorize + plugins: + - name: cors + + ## Secure Auth routes + - name: auth-v1 + _comment: 'GoTrue: /auth/v1/* -> http://auth:9999/*' + url: http://supabase-auth:9999/ + routes: + - name: auth-v1-all + strip_path: true + paths: + - /auth/v1/ + plugins: + - name: cors + - name: key-auth + config: + hide_credentials: false + - name: acl + config: + hide_groups_header: true + allow: + - admin + - anon + + ## Secure REST routes + - name: rest-v1 + _comment: 'PostgREST: /rest/v1/* -> http://rest:3000/*' + url: http://supabase-rest:3000/ + routes: + - name: rest-v1-all + strip_path: true + paths: + - /rest/v1/ + plugins: + - name: cors + - name: key-auth + config: + hide_credentials: true + - name: acl + config: + hide_groups_header: true + allow: + - admin + - anon + + ## Secure GraphQL routes + - name: graphql-v1 + _comment: 'PostgREST: /graphql/v1/* -> http://rest:3000/rpc/graphql' + url: http://supabase-rest:3000/rpc/graphql + routes: + - name: graphql-v1-all + strip_path: true + paths: + - /graphql/v1 + plugins: + - name: cors + - name: key-auth + config: + hide_credentials: true + - name: request-transformer + config: + add: + headers: + - Content-Profile:graphql_public + - name: acl + config: + hide_groups_header: true + allow: + - admin + - anon + + ## Secure Realtime routes + - name: realtime-v1-ws + _comment: 'Realtime: /realtime/v1/* -> ws://realtime:4000/socket/*' + url: http://realtime-dev.supabase-realtime:4000/socket + protocol: ws + routes: + - name: realtime-v1-ws + strip_path: true + paths: + - /realtime/v1/ + plugins: + - name: cors + - name: key-auth + config: + hide_credentials: false + - name: acl + config: + hide_groups_header: true + allow: + - admin + - anon + - name: realtime-v1-rest + _comment: 'Realtime: /realtime/v1/* -> ws://realtime:4000/socket/*' + url: http://realtime-dev.supabase-realtime:4000/api + protocol: http + routes: + - name: realtime-v1-rest + strip_path: true + paths: + - /realtime/v1/api + plugins: + - name: cors + - name: key-auth + config: + hide_credentials: false + - name: acl + config: + hide_groups_header: true + allow: + - admin + - anon + ## Storage routes: the storage server manages its own auth + - name: storage-v1 + _comment: 'Storage: /storage/v1/* -> http://storage:5000/*' + url: http://supabase-storage:5000/ + routes: + - name: storage-v1-all + strip_path: true + paths: + - /storage/v1/ + plugins: + - name: cors + + ## Edge Functions routes + - name: functions-v1 + _comment: 'Edge Functions: /functions/v1/* -> http://functions:9000/*' + url: http://supabase-edge-functions:9000/ + routes: + - name: functions-v1-all + strip_path: true + paths: + - /functions/v1/ + plugins: + - name: cors + + ## Analytics routes + - name: analytics-v1 + _comment: 'Analytics: /analytics/v1/* -> http://logflare:4000/*' + url: http://supabase-analytics:4000/ + routes: + - name: analytics-v1-all + strip_path: true + paths: + - /analytics/v1/ + + ## Secure Database routes + - name: meta + _comment: 'pg-meta: /pg/* -> http://pg-meta:8080/*' + url: http://supabase-meta:8080/ + routes: + - name: meta-all + strip_path: true + paths: + - /pg/ + plugins: + - name: key-auth + config: + hide_credentials: false + - name: acl + config: + hide_groups_header: true + allow: + - admin + + ## Protected Dashboard - catch all remaining routes + - name: dashboard + _comment: 'Studio: /* -> http://studio:3000/*' + url: http://supabase-studio:3000/ + routes: + - name: dashboard-all + strip_path: true + paths: + - / + plugins: + - name: cors + - name: basic-auth + config: + hide_credentials: true diff --git a/modules/supabase/files/meta-env.env b/modules/supabase/files/meta-env.env new file mode 100644 index 00000000..c9d25096 --- /dev/null +++ b/modules/supabase/files/meta-env.env @@ -0,0 +1,6 @@ +PG_META_PORT=8080 +PG_META_DB_HOST=supabase-db +PG_META_DB_PORT=5432 +PG_META_DB_NAME=postgres +PG_META_DB_USER=supabase_admin +PG_META_DB_PASSWORD=${secret:postgres} diff --git a/modules/supabase/files/pooler-env.env b/modules/supabase/files/pooler-env.env new file mode 100644 index 00000000..f02c7cd7 --- /dev/null +++ b/modules/supabase/files/pooler-env.env @@ -0,0 +1,14 @@ +PORT=4000 +POSTGRES_PORT=5432 +POSTGRES_DB=postgres +POSTGRES_PASSWORD=${secret:postgres} +DATABASE_URL=ecto://supabase_admin:${secret:postgres}@supabase-db:5432/_supabase +CLUSTER_POSTGRES=true +SECRET_KEY_BASE=${secret:key-base-a}${secret:key-base-b} +VAULT_ENC_KEY=${secret:pooler-vault} +API_JWT_SECRET=${secret:jwt} +METRICS_JWT_SECRET=${secret:jwt} +REGION=local +ERL_AFLAGS=-proto_dist inet_tcp +RELEASE_NODE=supavisor@127.0.0.1 +POOLER_POOL_MODE=transaction diff --git a/modules/supabase/files/pooler.exs b/modules/supabase/files/pooler.exs new file mode 100644 index 00000000..61b5ba8c --- /dev/null +++ b/modules/supabase/files/pooler.exs @@ -0,0 +1,43 @@ +{:ok, _} = Application.ensure_all_started(:supavisor) + +# Written by the mesh (modules/supabase). Per-machine values come from settings.json (the module's +# one merge:json file); secrets come from the environment the image reads. +settings = "/etc/pooler/settings.json" |> File.read!() |> Jason.decode!() |> Map.get("pooler", %{}) + +{:ok, version} = + case Supavisor.Repo.query!("select version()") do + %{rows: [[ver]]} -> Supavisor.Helpers.parse_pg_version(ver) + _ -> nil + end + +params = %{ + "external_id" => to_string(settings["tenant"] || "default"), + "db_host" => "supabase-db", + "db_port" => System.get_env("POSTGRES_PORT"), + "db_database" => System.get_env("POSTGRES_DB"), + "require_user" => false, + "auth_query" => "SELECT * FROM pgbouncer.get_auth($1)", + "default_max_clients" => settings["max-client-connections"] || 100, + "default_pool_size" => settings["pool-size"] || 20, + "default_parameter_status" => %{"server_version" => version}, + "users" => [%{ + "db_user" => "pgbouncer", + "db_password" => System.get_env("POSTGRES_PASSWORD"), + "mode_type" => System.get_env("POOLER_POOL_MODE"), + "pool_size" => settings["pool-size"] || 20, + "is_manager" => true + }] +} + +case Supavisor.Tenants.get_tenant_by_external_id(params["external_id"]) do + nil -> + {:ok, _} = Supavisor.Tenants.create_tenant(params) + + # A tenant made elsewhere (HAL named the database host "db") is pointed at this module's + # database container; nothing else it holds is changed. + %{db_host: host} = tenant when host != "supabase-db" -> + {:ok, _} = Supavisor.Tenants.update_tenant(tenant, %{"db_host" => "supabase-db"}) + + _ -> + :ok +end diff --git a/modules/supabase/files/postgrest.conf b/modules/supabase/files/postgrest.conf new file mode 100644 index 00000000..17557532 --- /dev/null +++ b/modules/supabase/files/postgrest.conf @@ -0,0 +1,7 @@ +db-uri = "postgres://authenticator:${secret:postgres}@supabase-db:5432/postgres" +db-schemas = "public,storage,graphql_public" +db-anon-role = "anon" +jwt-secret = "${secret:jwt}" +db-use-legacy-gucs = false +app.settings.jwt_secret = "${secret:jwt}" +app.settings.jwt_exp = "3600" diff --git a/modules/supabase/files/realtime-env.env b/modules/supabase/files/realtime-env.env new file mode 100644 index 00000000..75e21369 --- /dev/null +++ b/modules/supabase/files/realtime-env.env @@ -0,0 +1,16 @@ +PORT=4000 +DB_HOST=supabase-db +DB_PORT=5432 +DB_USER=supabase_admin +DB_PASSWORD=${secret:postgres} +DB_NAME=postgres +DB_AFTER_CONNECT_QUERY=SET search_path TO _realtime +DB_ENC_KEY=supabaserealtime +API_JWT_SECRET=${secret:jwt} +SECRET_KEY_BASE=${secret:key-base-a}${secret:key-base-b} +ERL_AFLAGS=-proto_dist inet_tcp +DNS_NODES='' +RLIMIT_NOFILE=10000 +APP_NAME=realtime +SEED_SELF_HOST=true +RUN_JANITOR=true diff --git a/modules/supabase/files/settings.json b/modules/supabase/files/settings.json new file mode 100644 index 00000000..8ab8cfa9 --- /dev/null +++ b/modules/supabase/files/settings.json @@ -0,0 +1,7 @@ +{ + "pooler": { + "tenant": "default", + "pool-size": 20, + "max-client-connections": 100 + } +} diff --git a/modules/supabase/files/storage-env.env b/modules/supabase/files/storage-env.env new file mode 100644 index 00000000..fd5da860 --- /dev/null +++ b/modules/supabase/files/storage-env.env @@ -0,0 +1,13 @@ +ANON_KEY=${secret:anon-key} +SERVICE_KEY=${secret:service-role-key} +POSTGREST_URL=http://supabase-rest:3000 +PGRST_JWT_SECRET=${secret:jwt} +DATABASE_URL=postgres://supabase_storage_admin:${secret:postgres}@supabase-db:5432/postgres +FILE_SIZE_LIMIT=52428800 +STORAGE_BACKEND=file +FILE_STORAGE_BACKEND_PATH=/var/lib/storage +TENANT_ID=stub +REGION=stub +GLOBAL_S3_BUCKET=stub +ENABLE_IMAGE_TRANSFORMATION=true +IMGPROXY_URL=http://supabase-imgproxy:5001 diff --git a/modules/supabase/files/studio-env.env b/modules/supabase/files/studio-env.env new file mode 100644 index 00000000..303dbc57 --- /dev/null +++ b/modules/supabase/files/studio-env.env @@ -0,0 +1,18 @@ +# Next.js listens on the address HOSTNAME names, and docker sets HOSTNAME to the container's id, +# so studio answered only on its network address while its image's health check asks localhost: +# unhealthy for ever while working. Every address, as the upstream compose file has it. +HOSTNAME=:: +STUDIO_PG_META_URL=http://supabase-meta:8080 +POSTGRES_PASSWORD=${secret:postgres} +DEFAULT_ORGANIZATION_NAME=Default Organization +DEFAULT_PROJECT_NAME=Default Project +OPENAI_API_KEY=${secret:openai} +SUPABASE_URL=http://supabase-kong:8000 +SUPABASE_PUBLIC_URL=https://${bound:route:name} +SUPABASE_ANON_KEY=${secret:anon-key} +SUPABASE_SERVICE_KEY=${secret:service-role-key} +AUTH_JWT_SECRET=${secret:jwt} +LOGFLARE_API_KEY=${secret:logflare} +LOGFLARE_URL=http://supabase-analytics:4000 +NEXT_PUBLIC_ENABLE_LOGS=true +NEXT_ANALYTICS_BACKEND_PROVIDER=postgres diff --git a/modules/supabase/files/vector.yml b/modules/supabase/files/vector.yml new file mode 100644 index 00000000..fb0f03d5 --- /dev/null +++ b/modules/supabase/files/vector.yml @@ -0,0 +1,258 @@ +# Written by the mesh (modules/supabase). +# +# Every sink hands logflare its API key in the x-api-key header, never as an api_key query parameter: +# logflare prints a failed request's whole URL, query string included, in its error report, so a +# key in the URL is a key in its log (novox/hq issue 268). supabase-analytics refuses to start +# while this file still puts the key in a URL. +api: + enabled: true + address: 0.0.0.0:9001 + +sources: + docker_host: + type: docker_logs + include_containers: + - supabase-kong + - supabase-auth + - supabase-rest + - realtime-dev.supabase-realtime + - supabase-storage + - supabase-edge-functions + - supabase-db + +transforms: + project_logs: + type: remap + inputs: + - docker_host + source: |- + .project = "default" + .event_message = del(.message) + .appname = del(.container_name) + del(.container_created_at) + del(.container_id) + del(.source_type) + del(.stream) + del(.label) + del(.image) + del(.host) + del(.stream) + router: + type: route + inputs: + - project_logs + route: + kong: '.appname == "supabase-kong"' + auth: '.appname == "supabase-auth"' + rest: '.appname == "supabase-rest"' + realtime: '.appname == "realtime-dev.supabase-realtime"' + storage: '.appname == "supabase-storage"' + functions: '.appname == "supabase-edge-functions"' + db: '.appname == "supabase-db"' + # Ignores non nginx errors since they are related with kong booting up + kong_logs: + type: remap + inputs: + - router.kong + source: |- + req, err = parse_nginx_log(.event_message, "combined") + if err == null { + .timestamp = req.timestamp + .metadata.request.headers.referer = req.referer + .metadata.request.headers.user_agent = req.agent + .metadata.request.headers.cf_connecting_ip = req.client + .metadata.request.method = req.method + .metadata.request.path = req.path + .metadata.request.protocol = req.protocol + .metadata.response.status_code = req.status + } + if err != null { + abort + } + # Ignores non nginx errors since they are related with kong booting up + kong_err: + type: remap + inputs: + - router.kong + source: |- + .metadata.request.method = "GET" + .metadata.response.status_code = 200 + parsed, err = parse_nginx_log(.event_message, "error") + if err == null { + .timestamp = parsed.timestamp + .severity = parsed.severity + .metadata.request.host = parsed.host + .metadata.request.headers.cf_connecting_ip = parsed.client + url, err = split(parsed.request, " ") + if err == null { + .metadata.request.method = url[0] + .metadata.request.path = url[1] + .metadata.request.protocol = url[2] + } + } + if err != null { + abort + } + # Gotrue logs are structured json strings which frontend parses directly. But we keep metadata for consistency. + auth_logs: + type: remap + inputs: + - router.auth + source: |- + parsed, err = parse_json(.event_message) + if err == null { + .metadata.timestamp = parsed.time + .metadata = merge!(.metadata, parsed) + } + # PostgREST logs are structured so we separate timestamp from message using regex + rest_logs: + type: remap + inputs: + - router.rest + source: |- + parsed, err = parse_regex(.event_message, r'^(?P