From 35ff67bda0c340d65b19fbb0ad8180c336781fb5 Mon Sep 17 00:00:00 2001 From: jochen Date: Fri, 30 Oct 2026 02:21:51 +0100 Subject: [PATCH] Design 47's last milestones: the keeper, self-check, healers, hand-acts, provisioner, vault, artifact sweep, disk readings, credential forms (issues 774, 1952, 1953, 1954, 1955) One commit with exactly the tree of the reviewed head 35b4509c0a15f7900209f8260e4801c698efe204, on main 12c2574c9d5443c58c0fa2a3e516c8c10d295993, so two compiled programs added and removed on the branch leave no trace in history (issue 1577). --- checks/credentialform/go.mod | 5 + checks/credentialform/go.sum | 2 + checks/credentialform/main.go | 70 ++ checks/credentialform/main_test.go | 49 + checks/credentialform/reads-outside | 3 + checks/handacts/fallbacks | 8 + checks/handacts/go.mod | 3 + checks/handacts/main.go | 109 ++ checks/handacts/main_test.go | 59 + checks/handacts/reads-outside | 3 + checks/health/go.mod | 2 +- checks/health/go.sum | 4 +- checks/noplacement/assignments-readers | 4 + checks/noplacement/go.mod | 4 + checks/noplacement/main.go | 75 +- checks/noplacement/main_test.go | 35 +- .../cmd/artifact-store-tools/door.go | 129 +++ .../cmd/artifact-store-tools/holder.go | 1020 +++++++++++++++++ .../holder_integration_test.go | 260 +++++ .../cmd/artifact-store-tools/holder_test.go | 351 ++++++ .../cmd/artifact-store-tools/main.go | 28 +- .../cmd/artifact-store-tools/seat.go | 122 ++ .../cmd/artifact-store-tools/store_test.go | 29 +- .../cmd/artifact-store-tools/tools.go | 11 +- .../artifact-store-tools/files/store-address | 1 + modules/artifact-store-tools/go.mod | 21 +- modules/artifact-store-tools/go.sum | 29 +- .../internal/registry/blob.go | 57 + .../internal/registry/blob_test.go | 44 + .../internal/registry/hold.go | 345 ++++++ .../internal/registry/hold_test.go | 268 +++++ .../internal/registry/platforms_test.go | 176 +++ .../internal/registry/store.go | 325 ++++++ .../internal/registry/store_test.go | 134 +++ modules/artifact-store-tools/module.json | 57 +- modules/buildah/module.json | 5 +- modules/disk-load/.gitignore | 1 + modules/disk-load/cmd/disk-load/keeper.go | 248 ++++ modules/disk-load/cmd/disk-load/main.go | 45 +- modules/disk-load/cmd/disk-load/measure.go | 215 ++++ modules/disk-load/cmd/disk-load/readings.go | 12 + .../disk-load/cmd/disk-load/readings_test.go | 144 +++ .../disk-load/cmd/disk-load/readings_tools.go | 93 ++ modules/disk-load/cmd/disk-load/tools_test.go | 9 +- modules/disk-load/go.mod | 4 +- modules/disk-load/go.sum | 16 +- modules/disk-load/module.json | 17 +- modules/distribution/module.json | 5 +- modules/dnsmasq/go.mod | 4 +- modules/dnsmasq/go.sum | 4 +- modules/dnsmasq/module.json | 5 +- .../docker/cmd/docker-tools/prune_images.go | 16 +- .../cmd/docker-tools/prune_images_test.go | 2 +- modules/docker/go.mod | 4 +- modules/docker/go.sum | 4 +- modules/docker/module.json | 2 +- .../forge-account-provider/builder_test.go | 11 +- .../cmd/forge-account-provider/bus_test.go | 9 +- .../cmd/forge-account-provider/main_test.go | 4 +- .../cmd/forge-account-provider/service.go | 4 +- .../forge-account-provider/service_test.go | 71 +- .../cmd/forge-account-provider/watch_test.go | 4 +- modules/gitea/go.mod | 2 +- modules/gitea/go.sum | 4 +- modules/gitea/module.json | 6 +- modules/go/module.json | 5 +- modules/influxdb/module.json | 3 + .../cmd/keycloak-provider/grant_moved_test.go | 29 +- .../cmd/keycloak-provider/live_retire_test.go | 8 + .../cmd/keycloak-provider/moved_test.go | 5 +- .../keycloak/cmd/keycloak-provider/oidc.go | 16 +- .../cmd/keycloak-provider/oidc_test.go | 6 +- .../cmd/keycloak-provider/provisioner.go | 5 +- modules/keycloak/go.mod | 2 +- modules/keycloak/go.sum | 4 +- modules/keycloak/module.json | 4 +- modules/mailu/module.json | 12 +- modules/mesh-build-image/.gitignore | 1 + modules/mesh-build-image/go.mod | 2 +- modules/mesh-build-image/go.sum | 4 +- modules/mesh-build-node/.gitignore | 1 + modules/mesh-build-node/go.mod | 2 +- modules/mesh-build-node/go.sum | 4 +- modules/mesh-catalog/heard.ts | 73 +- modules/mesh-catalog/index.ts | 92 +- modules/mesh-catalog/module.json | 13 +- modules/mesh-catalog/test/heard.test.ts | 27 +- .../cmd/mesh-conditions/compare.go | 221 ++++ .../cmd/mesh-conditions/hold.go | 340 ++++++ .../cmd/mesh-conditions/hold_test.go | 129 +++ .../cmd/mesh-conditions/homes.go | 160 +++ .../cmd/mesh-conditions/keeper.go | 668 +++++++++++ .../cmd/mesh-conditions/keeper_test.go | 341 ++++++ .../cmd/mesh-conditions/main.go | 164 +++ .../cmd/mesh-conditions/manifest_test.go | 69 ++ .../cmd/mesh-conditions/reads-outside | 2 + .../cmd/mesh-conditions/tools.go | 252 ++++ .../mesh-conditions/whole_integration_test.go | 335 ++++++ modules/mesh-conditions/go.mod | 22 + modules/mesh-conditions/go.sum | 27 + modules/mesh-conditions/module.json | 87 ++ modules/mesh-config/README.md | 18 + modules/mesh-config/cmd/mesh-config/config.go | 11 +- modules/mesh-config/cmd/mesh-config/main.go | 29 +- .../cmd/mesh-config/manifest_test.go | 73 ++ .../mesh-config/cmd/mesh-config/proposals.go | 1 + modules/mesh-config/cmd/mesh-config/pulls.go | 53 +- .../mesh-config/cmd/mesh-config/reads-outside | 2 + .../mesh-config/cmd/mesh-config/sources.go | 58 +- .../cmd/mesh-config/switch_tools.go | 79 ++ .../mesh-config/cmd/mesh-config/takeover.go | 771 +++++++++++++ .../cmd/mesh-config/takeover_test.go | 238 ++++ modules/mesh-config/go.mod | 11 +- modules/mesh-config/go.sum | 28 +- modules/mesh-config/module.json | 24 +- .../cmd/mesh-delivery-gate/approval.go | 9 +- .../cmd/mesh-delivery-gate/approval_test.go | 48 + .../cmd/mesh-delivery-gate/events.go | 2 +- .../cmd/mesh-delivery-gate/group_merge.go | 4 +- .../cmd/mesh-delivery-gate/holder.go | 2 +- .../cmd/mesh-delivery-gate/merges.go | 67 +- .../cmd/mesh-delivery-gate/ready.go | 43 +- .../cmd/mesh-delivery-gate/turns.go | 10 +- modules/mesh-delivery-gate/go.mod | 2 +- modules/mesh-delivery-gate/go.sum | 4 +- modules/mesh-hand-acts/.gitignore | 2 + modules/mesh-hand-acts/README.md | 26 + .../mesh-hand-acts/cmd/mesh-hand-acts/acts.go | 375 ++++++ .../cmd/mesh-hand-acts/acts_test.go | 286 +++++ .../mesh-hand-acts/cmd/mesh-hand-acts/main.go | 84 ++ .../cmd/mesh-hand-acts/manifest_test.go | 76 ++ .../cmd/mesh-hand-acts/reads-outside | 2 + .../cmd/mesh-hand-acts/tools.go | 110 ++ .../mesh-hand-acts/whole_integration_test.go | 405 +++++++ modules/mesh-hand-acts/go.mod | 22 + modules/mesh-hand-acts/go.sum | 27 + modules/mesh-hand-acts/module.json | 65 ++ modules/mesh-healers/.gitignore | 2 + modules/mesh-healers/README.md | 46 + .../mesh-healers/cmd/mesh-healers/healers.go | 663 +++++++++++ .../cmd/mesh-healers/healers_test.go | 496 ++++++++ modules/mesh-healers/cmd/mesh-healers/main.go | 118 ++ .../cmd/mesh-healers/manifest_test.go | 92 ++ .../cmd/mesh-healers/observation.go | 77 ++ .../cmd/mesh-healers/reads-outside | 2 + .../mesh-healers/cmd/mesh-healers/tools.go | 230 ++++ .../mesh-healers/cmd/mesh-healers/wanted.go | 166 +++ .../mesh-healers/whole_integration_test.go | 423 +++++++ modules/mesh-healers/go.mod | 22 + modules/mesh-healers/go.sum | 27 + modules/mesh-healers/module.json | 82 ++ modules/mesh-issues/files/settings.json | 4 + modules/mesh-issues/go.mod | 14 +- modules/mesh-issues/go.sum | 24 +- modules/mesh-issues/module.json | 4 + .../mesh-mail/cmd/mesh-mail/sdk_pin_test.go | 12 +- modules/mesh-mail/go.mod | 2 +- modules/mesh-mail/go.sum | 4 +- modules/mesh-provisioner-client/README.md | 14 +- .../asked_again_test.go | 12 +- .../cmd/mesh-provisioner-client/asks_test.go | 7 +- .../cmd/mesh-provisioner-client/client.go | 183 ++- .../mesh-provisioner-client/client_test.go | 104 +- .../mesh-provisioner-client/collect_test.go | 9 +- .../mesh-provisioner-client/consumers_test.go | 9 +- .../cmd/mesh-provisioner-client/ended_test.go | 58 - .../cmd/mesh-provisioner-client/plan.go | 17 +- .../mesh-provisioner-client/pulled_in_test.go | 4 +- .../route_names_test.go | 30 +- .../cmd/mesh-provisioner-client/secrets.go | 4 +- .../mesh-provisioner-client/secrets_test.go | 8 +- .../testdata/accounts.conf | 2 +- modules/mesh-provisioner-client/go.mod | 2 +- modules/mesh-provisioner-client/go.sum | 4 +- modules/mesh-provisioner/.gitignore | 2 + .../cmd/mesh-provisioner/holder.go | 908 +++++++++++++++ .../holder_integration_test.go | 229 ++++ .../cmd/mesh-provisioner/holder_test.go | 362 ++++++ .../cmd/mesh-provisioner/holdertools.go | 104 ++ .../cmd/mesh-provisioner/main.go | 172 ++- .../cmd/mesh-provisioner/provisioner_test.go | 44 +- .../cmd/mesh-provisioner/reads-outside | 2 + .../cmd/mesh-provisioner/registry.go | 15 +- modules/mesh-provisioner/go.mod | 19 +- modules/mesh-provisioner/go.sum | 29 +- modules/mesh-provisioner/module.json | 27 +- modules/mesh-self-check/.gitignore | 2 + modules/mesh-self-check/README.md | 33 + .../cmd/mesh-self-check/main.go | 74 ++ .../cmd/mesh-self-check/manifest_test.go | 101 ++ .../cmd/mesh-self-check/observation.go | 142 +++ .../cmd/mesh-self-check/probes.go | 453 ++++++++ .../cmd/mesh-self-check/probes_data.go | 392 +++++++ .../cmd/mesh-self-check/probes_fill.go | 112 ++ .../cmd/mesh-self-check/probes_mesh.go | 219 ++++ .../cmd/mesh-self-check/probes_root.go | 254 ++++ .../cmd/mesh-self-check/probes_test.go | 455 ++++++++ .../cmd/mesh-self-check/reads-outside | 2 + .../cmd/mesh-self-check/selfcheck.go | 417 +++++++ .../cmd/mesh-self-check/selfcheck_test.go | 295 +++++ .../cmd/mesh-self-check/tools.go | 218 ++++ .../mesh-self-check/whole_integration_test.go | 298 +++++ modules/mesh-self-check/go.mod | 22 + modules/mesh-self-check/go.sum | 27 + modules/mesh-self-check/module.json | 87 ++ modules/mesh-vault/cmd/mesh-vault/accounts.go | 12 +- modules/mesh-vault/cmd/mesh-vault/adopt.go | 29 +- modules/mesh-vault/cmd/mesh-vault/askers.go | 21 +- modules/mesh-vault/cmd/mesh-vault/intake.go | 64 +- .../mesh-vault/cmd/mesh-vault/intake_test.go | 51 +- .../cmd/mesh-vault/issued_account_test.go | 7 + modules/mesh-vault/cmd/mesh-vault/lease.go | 49 + modules/mesh-vault/cmd/mesh-vault/main.go | 140 ++- modules/mesh-vault/cmd/mesh-vault/mint.go | 376 ++++++ .../cmd/mesh-vault/mint_integration_test.go | 422 +++++++ .../mesh-vault/cmd/mesh-vault/mint_test.go | 244 ++++ modules/mesh-vault/cmd/mesh-vault/outcomes.go | 174 +++ .../cmd/mesh-vault/outcomes_test.go | 122 ++ .../cmd/mesh-vault/owncalls_test.go | 2 +- modules/mesh-vault/cmd/mesh-vault/ownnode.go | 27 +- modules/mesh-vault/cmd/mesh-vault/pair.go | 13 +- modules/mesh-vault/cmd/mesh-vault/placed.go | 134 +++ .../mesh-vault/cmd/mesh-vault/placed_test.go | 66 ++ modules/mesh-vault/cmd/mesh-vault/retire.go | 264 +++++ .../mesh-vault/cmd/mesh-vault/retire_test.go | 89 ++ modules/mesh-vault/cmd/mesh-vault/rotation.go | 94 ++ modules/mesh-vault/cmd/mesh-vault/signing.go | 463 ++++++++ .../mesh-vault/cmd/mesh-vault/signing_test.go | 196 ++++ modules/mesh-vault/cmd/mesh-vault/store.go | 2 + modules/mesh-vault/cmd/mesh-vault/tools.go | 100 +- modules/mesh-vault/cmd/mesh-vault/vault.go | 40 +- .../mesh-vault/cmd/mesh-vault/vault_test.go | 45 +- modules/mesh-vault/go.mod | 2 +- modules/mesh-vault/go.sum | 4 +- modules/mesh-vault/module.json | 32 +- modules/mesh-watcher/cmd/mesh-watcher/main.go | 81 +- .../cmd/mesh-watcher/manifest_test.go | 4 +- .../mesh-watcher/cmd/mesh-watcher/takeover.go | 59 + .../cmd/mesh-watcher/watcher_test.go | 23 +- modules/mesh-watcher/go.mod | 4 +- modules/mesh-watcher/go.sum | 16 +- modules/mesh-watcher/module.json | 14 +- modules/messenger/cmd/messenger/answers.go | 2 +- .../messenger/cmd/messenger/answers_test.go | 10 + modules/messenger/cmd/messenger/asks.go | 88 +- modules/messenger/cmd/messenger/bus_test.go | 12 +- modules/messenger/cmd/messenger/condition.go | 7 +- modules/messenger/cmd/messenger/holder.go | 3 +- modules/messenger/cmd/messenger/main.go | 18 +- .../messenger/cmd/messenger/manifest_test.go | 3 +- .../messenger/cmd/messenger/review_test.go | 26 +- modules/messenger/cmd/messenger/sync.go | 8 + modules/messenger/cmd/messenger/takeover.go | 59 + modules/messenger/go.mod | 2 +- modules/messenger/go.sum | 4 +- modules/messenger/module.json | 9 +- .../minio/cmd/object-store/manifest_test.go | 8 +- modules/minio/cmd/object-store/server.go | 82 +- modules/minio/cmd/object-store/server_test.go | 18 +- modules/minio/go.mod | 2 +- modules/minio/go.sum | 16 +- modules/minio/module.json | 7 +- modules/mongodb/module.json | 3 + modules/mosquitto/module.json | 3 + modules/mounts/cmd/mounts/bus.go | 15 +- modules/mounts/cmd/mounts/data.go | 46 +- modules/mounts/cmd/mounts/main.go | 29 +- modules/mounts/cmd/mounts/manifest_test.go | 10 +- modules/mounts/cmd/mounts/review_test.go | 29 +- modules/mounts/go.mod | 4 +- modules/mounts/go.sum | 10 +- modules/mounts/module.json | 30 +- modules/mssql/module.json | 3 + .../cmd/mesh-nats-snapshot/certificate.go | 174 +++ .../mesh-nats-snapshot/certificate_test.go | 138 +++ .../nats/cmd/mesh-nats-snapshot/connect.go | 99 +- modules/nats/cmd/mesh-nats-snapshot/main.go | 4 + modules/nats/cmd/nats-tools/bus.go | 41 + modules/nats/cmd/nats-tools/main.go | 9 +- modules/nats/cmd/nats-tools/manifest_test.go | 26 +- modules/nats/cmd/nats-tools/removal.go | 246 ++++ modules/nats/cmd/nats-tools/removal_test.go | 173 +++ modules/nats/cmd/nats-tools/tools.go | 4 + modules/nats/cmd/nats-tools/tools_test.go | 51 +- modules/nats/go.mod | 14 +- modules/nats/go.sum | 24 +- modules/nats/internal/busconn/busconn.go | 106 ++ modules/nats/module.json | 33 +- modules/nfs-server/.gitignore | 1 + modules/nfs-server/cmd/nfs-server/backup.go | 74 +- .../nfs-server/cmd/nfs-server/backup_test.go | 48 +- .../nfs-server/cmd/nfs-server/guards_test.go | 12 +- modules/nfs-server/cmd/nfs-server/main.go | 50 +- .../cmd/nfs-server/manifest_test.go | 4 +- modules/nfs-server/go.mod | 4 +- modules/nfs-server/go.sum | 16 +- modules/nfs-server/module.json | 37 +- modules/nodejs/module.json | 5 +- modules/ollama/module.json | 5 +- .../cmd/postgres-provider/harness_test.go | 4 +- .../cmd/postgres-provider/live_test.go | 8 + .../cmd/postgres-provider/provisioner.go | 15 +- .../cmd/postgres-provider/retire_pg.go | 120 +- .../cmd/postgres-provider/retire_pg_test.go | 85 +- modules/postgres/go.mod | 2 +- modules/postgres/go.sum | 4 +- modules/postgres/module.json | 4 +- modules/redis/module.json | 3 + modules/restic/cmd/restic-backups/.gitignore | 1 + modules/restic/cmd/restic-backups/backups.go | 11 + modules/restic/cmd/restic-backups/data.go | 392 +------ .../restic/cmd/restic-backups/data_test.go | 174 +-- modules/restic/cmd/restic-backups/good.go | 128 +++ .../restic/cmd/restic-backups/health_test.go | 4 +- modules/restic/cmd/restic-backups/main.go | 65 +- modules/restic/go.mod | 4 +- modules/restic/go.sum | 4 +- modules/restic/module.json | 94 +- .../cmd/route-provider/provider_test.go | 15 +- modules/route-adapter/go.mod | 2 +- modules/route-adapter/go.sum | 4 +- modules/route-adapter/module.json | 4 +- modules/showcase/module.json | 5 +- modules/step-ca/module.json | 5 +- modules/systemd-resolved/go.mod | 4 +- modules/systemd-resolved/go.sum | 4 +- modules/systemd-resolved/module.json | 5 +- modules/umami/module.json | 5 +- modules/xorg/go.mod | 4 +- modules/xorg/go.sum | 4 +- modules/xorg/module.json | 5 +- 331 files changed, 24352 insertions(+), 1792 deletions(-) create mode 100644 checks/credentialform/go.mod create mode 100644 checks/credentialform/go.sum create mode 100644 checks/credentialform/main.go create mode 100644 checks/credentialform/main_test.go create mode 100644 checks/credentialform/reads-outside create mode 100644 checks/handacts/fallbacks create mode 100644 checks/handacts/go.mod create mode 100644 checks/handacts/main.go create mode 100644 checks/handacts/main_test.go create mode 100644 checks/handacts/reads-outside create mode 100644 checks/noplacement/assignments-readers create mode 100644 modules/artifact-store-tools/cmd/artifact-store-tools/door.go create mode 100644 modules/artifact-store-tools/cmd/artifact-store-tools/holder.go create mode 100644 modules/artifact-store-tools/cmd/artifact-store-tools/holder_integration_test.go create mode 100644 modules/artifact-store-tools/cmd/artifact-store-tools/holder_test.go create mode 100644 modules/artifact-store-tools/cmd/artifact-store-tools/seat.go create mode 100644 modules/artifact-store-tools/files/store-address create mode 100644 modules/artifact-store-tools/internal/registry/blob.go create mode 100644 modules/artifact-store-tools/internal/registry/blob_test.go create mode 100644 modules/artifact-store-tools/internal/registry/hold.go create mode 100644 modules/artifact-store-tools/internal/registry/hold_test.go create mode 100644 modules/artifact-store-tools/internal/registry/platforms_test.go create mode 100644 modules/artifact-store-tools/internal/registry/store.go create mode 100644 modules/artifact-store-tools/internal/registry/store_test.go create mode 100644 modules/disk-load/.gitignore create mode 100644 modules/disk-load/cmd/disk-load/keeper.go create mode 100644 modules/disk-load/cmd/disk-load/measure.go create mode 100644 modules/disk-load/cmd/disk-load/readings.go create mode 100644 modules/disk-load/cmd/disk-load/readings_test.go create mode 100644 modules/disk-load/cmd/disk-load/readings_tools.go create mode 100644 modules/mesh-build-image/.gitignore create mode 100644 modules/mesh-build-node/.gitignore create mode 100644 modules/mesh-conditions/cmd/mesh-conditions/compare.go create mode 100644 modules/mesh-conditions/cmd/mesh-conditions/hold.go create mode 100644 modules/mesh-conditions/cmd/mesh-conditions/hold_test.go create mode 100644 modules/mesh-conditions/cmd/mesh-conditions/homes.go create mode 100644 modules/mesh-conditions/cmd/mesh-conditions/keeper.go create mode 100644 modules/mesh-conditions/cmd/mesh-conditions/keeper_test.go create mode 100644 modules/mesh-conditions/cmd/mesh-conditions/main.go create mode 100644 modules/mesh-conditions/cmd/mesh-conditions/manifest_test.go create mode 100644 modules/mesh-conditions/cmd/mesh-conditions/reads-outside create mode 100644 modules/mesh-conditions/cmd/mesh-conditions/tools.go create mode 100644 modules/mesh-conditions/cmd/mesh-conditions/whole_integration_test.go create mode 100644 modules/mesh-conditions/go.mod create mode 100644 modules/mesh-conditions/go.sum create mode 100644 modules/mesh-conditions/module.json create mode 100644 modules/mesh-config/cmd/mesh-config/manifest_test.go create mode 100644 modules/mesh-config/cmd/mesh-config/reads-outside create mode 100644 modules/mesh-config/cmd/mesh-config/switch_tools.go create mode 100644 modules/mesh-config/cmd/mesh-config/takeover.go create mode 100644 modules/mesh-config/cmd/mesh-config/takeover_test.go create mode 100644 modules/mesh-hand-acts/.gitignore create mode 100644 modules/mesh-hand-acts/README.md create mode 100644 modules/mesh-hand-acts/cmd/mesh-hand-acts/acts.go create mode 100644 modules/mesh-hand-acts/cmd/mesh-hand-acts/acts_test.go create mode 100644 modules/mesh-hand-acts/cmd/mesh-hand-acts/main.go create mode 100644 modules/mesh-hand-acts/cmd/mesh-hand-acts/manifest_test.go create mode 100644 modules/mesh-hand-acts/cmd/mesh-hand-acts/reads-outside create mode 100644 modules/mesh-hand-acts/cmd/mesh-hand-acts/tools.go create mode 100644 modules/mesh-hand-acts/cmd/mesh-hand-acts/whole_integration_test.go create mode 100644 modules/mesh-hand-acts/go.mod create mode 100644 modules/mesh-hand-acts/go.sum create mode 100644 modules/mesh-hand-acts/module.json create mode 100644 modules/mesh-healers/.gitignore create mode 100644 modules/mesh-healers/README.md create mode 100644 modules/mesh-healers/cmd/mesh-healers/healers.go create mode 100644 modules/mesh-healers/cmd/mesh-healers/healers_test.go create mode 100644 modules/mesh-healers/cmd/mesh-healers/main.go create mode 100644 modules/mesh-healers/cmd/mesh-healers/manifest_test.go create mode 100644 modules/mesh-healers/cmd/mesh-healers/observation.go create mode 100644 modules/mesh-healers/cmd/mesh-healers/reads-outside create mode 100644 modules/mesh-healers/cmd/mesh-healers/tools.go create mode 100644 modules/mesh-healers/cmd/mesh-healers/wanted.go create mode 100644 modules/mesh-healers/cmd/mesh-healers/whole_integration_test.go create mode 100644 modules/mesh-healers/go.mod create mode 100644 modules/mesh-healers/go.sum create mode 100644 modules/mesh-healers/module.json delete mode 100644 modules/mesh-provisioner-client/cmd/mesh-provisioner-client/ended_test.go create mode 100644 modules/mesh-provisioner/.gitignore create mode 100644 modules/mesh-provisioner/cmd/mesh-provisioner/holder.go create mode 100644 modules/mesh-provisioner/cmd/mesh-provisioner/holder_integration_test.go create mode 100644 modules/mesh-provisioner/cmd/mesh-provisioner/holder_test.go create mode 100644 modules/mesh-provisioner/cmd/mesh-provisioner/holdertools.go create mode 100644 modules/mesh-provisioner/cmd/mesh-provisioner/reads-outside create mode 100644 modules/mesh-self-check/.gitignore create mode 100644 modules/mesh-self-check/README.md create mode 100644 modules/mesh-self-check/cmd/mesh-self-check/main.go create mode 100644 modules/mesh-self-check/cmd/mesh-self-check/manifest_test.go create mode 100644 modules/mesh-self-check/cmd/mesh-self-check/observation.go create mode 100644 modules/mesh-self-check/cmd/mesh-self-check/probes.go create mode 100644 modules/mesh-self-check/cmd/mesh-self-check/probes_data.go create mode 100644 modules/mesh-self-check/cmd/mesh-self-check/probes_fill.go create mode 100644 modules/mesh-self-check/cmd/mesh-self-check/probes_mesh.go create mode 100644 modules/mesh-self-check/cmd/mesh-self-check/probes_root.go create mode 100644 modules/mesh-self-check/cmd/mesh-self-check/probes_test.go create mode 100644 modules/mesh-self-check/cmd/mesh-self-check/reads-outside create mode 100644 modules/mesh-self-check/cmd/mesh-self-check/selfcheck.go create mode 100644 modules/mesh-self-check/cmd/mesh-self-check/selfcheck_test.go create mode 100644 modules/mesh-self-check/cmd/mesh-self-check/tools.go create mode 100644 modules/mesh-self-check/cmd/mesh-self-check/whole_integration_test.go create mode 100644 modules/mesh-self-check/go.mod create mode 100644 modules/mesh-self-check/go.sum create mode 100644 modules/mesh-self-check/module.json create mode 100644 modules/mesh-vault/cmd/mesh-vault/lease.go create mode 100644 modules/mesh-vault/cmd/mesh-vault/mint.go create mode 100644 modules/mesh-vault/cmd/mesh-vault/mint_integration_test.go create mode 100644 modules/mesh-vault/cmd/mesh-vault/mint_test.go create mode 100644 modules/mesh-vault/cmd/mesh-vault/outcomes.go create mode 100644 modules/mesh-vault/cmd/mesh-vault/outcomes_test.go create mode 100644 modules/mesh-vault/cmd/mesh-vault/placed.go create mode 100644 modules/mesh-vault/cmd/mesh-vault/placed_test.go create mode 100644 modules/mesh-vault/cmd/mesh-vault/retire.go create mode 100644 modules/mesh-vault/cmd/mesh-vault/retire_test.go create mode 100644 modules/mesh-vault/cmd/mesh-vault/rotation.go create mode 100644 modules/mesh-vault/cmd/mesh-vault/signing.go create mode 100644 modules/mesh-vault/cmd/mesh-vault/signing_test.go create mode 100644 modules/mesh-watcher/cmd/mesh-watcher/takeover.go create mode 100644 modules/messenger/cmd/messenger/takeover.go create mode 100644 modules/nats/cmd/mesh-nats-snapshot/certificate.go create mode 100644 modules/nats/cmd/mesh-nats-snapshot/certificate_test.go create mode 100644 modules/nats/cmd/nats-tools/bus.go create mode 100644 modules/nats/cmd/nats-tools/removal.go create mode 100644 modules/nats/cmd/nats-tools/removal_test.go create mode 100644 modules/nats/internal/busconn/busconn.go create mode 100644 modules/nfs-server/.gitignore create mode 100644 modules/restic/cmd/restic-backups/.gitignore create mode 100644 modules/restic/cmd/restic-backups/good.go diff --git a/checks/credentialform/go.mod b/checks/credentialform/go.mod new file mode 100644 index 00000000..1793672d --- /dev/null +++ b/checks/credentialform/go.mod @@ -0,0 +1,5 @@ +module checks/credentialform + +go 1.26.0 + +require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 diff --git a/checks/credentialform/go.sum b/checks/credentialform/go.sum new file mode 100644 index 00000000..f8bfbd2b --- /dev/null +++ b/checks/credentialform/go.sum @@ -0,0 +1,2 @@ +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 h1:lKU3HjNXGiVbLW2hUgFjU/TMTUYMlNnnmZxG5T7biQA= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= diff --git a/checks/credentialform/main.go b/checks/credentialform/main.go new file mode 100644 index 00000000..e5a15729 --- /dev/null +++ b/checks/credentialform/main.go @@ -0,0 +1,70 @@ +// Package credentialform refuses an offer that does not say the form of its credential: an own secret, an issuer, or +// `form` pair or none (the lead, 2026-10-29). It reads every modules/*/module.json through mesh-sdk's +// CredentialFormProblems. +package credentialform + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "sort" + + "git.novox.be/novox/mesh-sdk/go/catalogue" +) + +// Problems is every offer of the manifests under root/modules that says no credential form, one line each. +func Problems(root string) ([]string, error) { + files, err := filepath.Glob(filepath.Join(root, "modules", "*", "module.json")) + if err != nil { + return nil, err + } + sort.Strings(files) + var out []string + reads := map[string]bool{} + var binders []string + for _, f := range files { + raw, err := os.ReadFile(f) + if err != nil { + return nil, err + } + var m catalogue.Manifest + if err := json.Unmarshal(raw, &m); err != nil { + return nil, fmt.Errorf("%s: %w", f, err) + } + out = append(out, catalogue.CredentialFormProblems(m)...) + var kept struct { + Reads []string `json:"reads"` + State []json.RawMessage `json:"state"` + } + _ = json.Unmarshal(raw, &kept) // reads may hold objects in another module; only the provisioner's are read + if m.Module == "mesh-provisioner" { + for _, r := range kept.Reads { + reads[r] = true + } + } + if len(m.Provides) > 0 && keepsBound(kept.State) { + binders = append(binders, m.Module) + } + } + for _, module := range binders { + if !reads[module+".bound"] { + out = append(out, fmt.Sprintf("%s provides and keeps a `bound` state, and mesh-provisioner's manifest does not read %s.bound", module, module)) + } + } + return out, nil +} + +// keepsBound says whether a manifest's `state` names `bound`, as a plain name or an object with a name. +func keepsBound(state []json.RawMessage) bool { + for _, s := range state { + var name string + var obj struct { + Name string `json:"name"` + } + if json.Unmarshal(s, &name) == nil && name == "bound" || json.Unmarshal(s, &obj) == nil && obj.Name == "bound" { + return true + } + } + return false +} diff --git a/checks/credentialform/main_test.go b/checks/credentialform/main_test.go new file mode 100644 index 00000000..33c1c4f4 --- /dev/null +++ b/checks/credentialform/main_test.go @@ -0,0 +1,49 @@ +package credentialform + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +func aCatalogue(t *testing.T, manifests map[string]string) string { + t.Helper() + root := t.TempDir() + for module, body := range manifests { + dir := filepath.Join(root, "modules", module) + if err := os.MkdirAll(dir, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "module.json"), []byte(body), 0o644); err != nil { + t.Fatal(err) + } + } + return root +} + +// An offer that says no form is refused and named; one that says pair, none or an own secret passes; and the real +// catalogue is clean. +func TestEveryOfferSaysTheFormOfItsCredential(t *testing.T) { + root := aCatalogue(t, map[string]string{ + "silent": `{"module":"silent","provides":[{"name":"thing","scope":"mesh"}]}`, + "binder": `{"module":"binder","provides":[{"name":"d","credential":{"form":"pair"}}],"state":["bound"]}`, + "mesh-provisioner": `{"module":"mesh-provisioner","reads":["other.bound"]}`, + "says": `{"module":"says","provides":[{"name":"a","credential":{"form":"pair"}},{"name":"b","credential":{"form":"none"}},{"name":"c","credential":{"own":"jwt"}}]}`, + }) + got, err := Problems(root) + if err != nil { + t.Fatal(err) + } + if len(got) != 2 || !strings.Contains(got[0], `silent provides "thing"`) || + !strings.Contains(got[1], "binder") || !strings.Contains(got[1], "mesh-provisioner") { + t.Fatalf("refused %v, wanted the silent offer and the binder the provisioner does not read", got) + } + real, err := Problems(filepath.Join("..", "..")) + if err != nil { + t.Fatal(err) + } + if len(real) != 0 { + t.Fatalf("the catalogue has offers with no credential form:\n%s", strings.Join(real, "\n")) + } +} diff --git a/checks/credentialform/reads-outside b/checks/credentialform/reads-outside new file mode 100644 index 00000000..8ff16290 --- /dev/null +++ b/checks/credentialform/reads-outside @@ -0,0 +1,3 @@ +# What these tests read outside their package, as paths from the repository's root (issue 1936). +# every manifest, for what each offer says of its credential's form. +modules/*/module.json diff --git a/checks/handacts/fallbacks b/checks/handacts/fallbacks new file mode 100644 index 00000000..c4fdf709 --- /dev/null +++ b/checks/handacts/fallbacks @@ -0,0 +1,8 @@ +# The calls of the controller's hand-act log a manifest of this catalogue may still invoke, each ` `: +# the fallback of a recorder (mesh-sdk handacts.Recorder) while the seat mesh-hand-acts has no holder that answers, until +# the controller's lease is written to a holder that is no controller. The check of this directory (handacts) refuses any +# other, and an entry whose module no longer invokes it: the list only shrinks, and it is empty before the lease write +# (novox/hq ADR 0565 §11; issue 1953, issue 774). +mesh-config hand-act fallback until the lease write: its sets, clears and held values record through mesh-hand-acts first +mesh-config warranted fallback until the lease write: its warrants record through mesh-hand-acts first +mesh-healers drill fallback until the lease write: a drill records through mesh-hand-acts first diff --git a/checks/handacts/go.mod b/checks/handacts/go.mod new file mode 100644 index 00000000..29e4c52b --- /dev/null +++ b/checks/handacts/go.mod @@ -0,0 +1,3 @@ +module git.novox.be/novox/mesh-catalog/checks/handacts + +go 1.22 diff --git a/checks/handacts/main.go b/checks/handacts/main.go new file mode 100644 index 00000000..1a222d60 --- /dev/null +++ b/checks/handacts/main.go @@ -0,0 +1,109 @@ +// Command handacts refuses a manifest that records into the controller's hand-act log (novox/hq ADR 0565 §11): every +// recorder records through the seat mesh-hand-acts from its delivery, so no manifest invokes +// `seat:mesh-controller.hand-act`, `.warranted` or `.drill`. The recorders' fallbacks to the controller's log while the +// seat has no holder are named in checks/handacts/fallbacks, each module and verb with why; that list only shrinks, and +// an entry whose module no longer invokes the verb is refused too. +// +// go run . +// +// Exits 1 on a finding. +package main + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "sort" + "strings" +) + +// Verbs are the controller's hand-act verbs, as a manifest invokes them, and Allowed the list of named fallbacks. +var Verbs = []string{"hand-act", "warranted", "drill"} + +const ( + invokePrefix = "seat:mesh-controller." + Allowed = "checks/handacts/fallbacks" +) + +// Findings is every refusal in the catalog at root, sorted. +func Findings(root string) ([]string, error) { + allowed := map[string]bool{} + raw, err := os.ReadFile(filepath.Join(root, Allowed)) + if err != nil && !os.IsNotExist(err) { + return nil, err + } + for _, line := range strings.Split(string(raw), "\n") { + if f := strings.Fields(line); len(f) >= 3 && !strings.HasPrefix(f[0], "#") { + allowed[f[0]+" "+f[1]] = true + } else if len(f) > 0 && !strings.HasPrefix(f[0], "#") { + return nil, fmt.Errorf("%s: %q is not ` `", Allowed, line) + } + } + manifests, err := filepath.Glob(filepath.Join(root, "modules", "*", "module.json")) + if err != nil { + return nil, err + } + var out []string + invoked := map[string]bool{} + for _, path := range manifests { + raw, err := os.ReadFile(path) + if err != nil { + return nil, err + } + var m struct { + Module string `json:"module"` + Invokes []string `json:"invokes"` + } + if json.Unmarshal(raw, &m) != nil || m.Module == "mesh-controller" { + continue + } + for _, inv := range m.Invokes { + verb, ok := strings.CutPrefix(strings.TrimSpace(inv), invokePrefix) + if !ok || !contains(Verbs, verb) { + continue + } + invoked[m.Module+" "+verb] = true + if !allowed[m.Module+" "+verb] { + out = append(out, fmt.Sprintf("%s invokes %s%s: record through seat:mesh-hand-acts.%s (novox/hq ADR 0565 §11), "+ + "or name the fallback in %s", m.Module, invokePrefix, verb, verb, Allowed)) + } + } + } + for entry := range allowed { + if !invoked[entry] { + out = append(out, fmt.Sprintf("%s names %q, which is no longer invoked: take it off the list, which only shrinks", + Allowed, entry)) + } + } + sort.Strings(out) + return out, nil +} + +func contains(list []string, s string) bool { + for _, x := range list { + if x == s { + return true + } + } + return false +} + +func main() { + if len(os.Args) != 2 { + fmt.Fprintln(os.Stderr, "usage: handacts ") + os.Exit(2) + } + found, err := Findings(os.Args[1]) + if err != nil { + fmt.Println(err) + os.Exit(1) + } + for _, f := range found { + fmt.Println(f) + } + if len(found) > 0 { + os.Exit(1) + } + fmt.Println("no manifest records into the controller's hand-act log but the fallbacks named") +} diff --git a/checks/handacts/main_test.go b/checks/handacts/main_test.go new file mode 100644 index 00000000..61e197e6 --- /dev/null +++ b/checks/handacts/main_test.go @@ -0,0 +1,59 @@ +package main + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +// No manifest invokes the controller's hand-act verbs but a named fallback; a named fallback no longer invoked is +// refused (novox/hq ADR 0565 §11's catalog test). +func TestFindings(t *testing.T) { + for _, row := range []struct { + name string + files map[string]string + want string // empty: clean + }{ + {"clean", map[string]string{"modules/a/module.json": `{"module":"a","invokes":["seat:mesh-hand-acts.hand-act"]}`}, ""}, + {"an unnamed call", map[string]string{"modules/a/module.json": `{"module":"a","invokes":["seat:mesh-controller.warranted"]}`}, + "a invokes seat:mesh-controller.warranted"}, + {"a named fallback, and the controller's own", map[string]string{ + "modules/a/module.json": `{"module":"a","invokes":["seat:mesh-controller.drill"]}`, + "modules/mesh-controller/module.json": `{"module":"mesh-controller","invokes":["seat:mesh-controller.hand-act"]}`, + Allowed: "# why\na drill fallback until the lease write\n"}, ""}, + {"a fallback no longer invoked", map[string]string{Allowed: "gone hand-act fallback\n"}, `names "gone hand-act"`}, + {"another verb of the controller", map[string]string{"modules/a/module.json": `{"module":"a","invokes":["seat:mesh-controller.conditions"]}`}, ""}, + } { + t.Run(row.name, func(t *testing.T) { + root := t.TempDir() + for path, text := range row.files { + p := filepath.Join(root, path) + if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(p, []byte(text), 0o644); err != nil { + t.Fatal(err) + } + } + got, err := Findings(root) + if err != nil { + t.Fatal(err) + } + if all := strings.Join(got, "\n"); (row.want == "") != (all == "") || !strings.Contains(all, row.want) { + t.Fatalf("findings = %q, want one containing %q", all, row.want) + } + }) + } +} + +// The run of this check over the real tree. +func TestTheCatalogRecordsOnlyThroughTheSeat(t *testing.T) { + got, err := Findings(filepath.Join("..", "..")) + if err != nil { + t.Fatal(err) + } + for _, f := range got { + t.Error(f) + } +} diff --git a/checks/handacts/reads-outside b/checks/handacts/reads-outside new file mode 100644 index 00000000..61f9814a --- /dev/null +++ b/checks/handacts/reads-outside @@ -0,0 +1,3 @@ +# What these tests read outside their package, as paths from the repository's root (issue 1936). +# every manifest, for what it invokes. +modules/*/module.json diff --git a/checks/health/go.mod b/checks/health/go.mod index a4d884a2..5ec7f46c 100644 --- a/checks/health/go.mod +++ b/checks/health/go.mod @@ -2,4 +2,4 @@ module checks/health go 1.26.0 -require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023150321-a2c813a6b468 +require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 diff --git a/checks/health/go.sum b/checks/health/go.sum index fc336ec2..f8bfbd2b 100644 --- a/checks/health/go.sum +++ b/checks/health/go.sum @@ -1,2 +1,2 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023150321-a2c813a6b468 h1:UcMee+IV2ii5E4dRZdQUaSaeh9IedTFYumuQbBiB0k0= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023150321-a2c813a6b468/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 h1:lKU3HjNXGiVbLW2hUgFjU/TMTUYMlNnnmZxG5T7biQA= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= diff --git a/checks/noplacement/assignments-readers b/checks/noplacement/assignments-readers new file mode 100644 index 00000000..64985d46 --- /dev/null +++ b/checks/noplacement/assignments-readers @@ -0,0 +1,4 @@ +# The modules of this catalogue that still read the controller's assignments (mesh-controller.assignments), each with +# why. The check of this directory (noplacement) refuses any other module that reads them, and an entry here whose module no longer +# does: the list only shrinks, and it is empty before the controller's lease is written to a holder that is no +# controller (novox/hq ADR 0549 §3, §6; issue 774). diff --git a/checks/noplacement/go.mod b/checks/noplacement/go.mod index a09ac5ec..782a236b 100644 --- a/checks/noplacement/go.mod +++ b/checks/noplacement/go.mod @@ -1,3 +1,7 @@ module git.novox.be/novox/mesh-catalog/checks/noplacement go 1.22 + +require git.novox.be/novox/mesh-catalog/checks/trunk v0.0.0 + +replace git.novox.be/novox/mesh-catalog/checks/trunk => ../trunk diff --git a/checks/noplacement/main.go b/checks/noplacement/main.go index 3de240c4..cd145e32 100644 --- a/checks/noplacement/main.go +++ b/checks/noplacement/main.go @@ -8,7 +8,13 @@ // 2. a `particulars` file at the root (the mesh's node names come from the checker's run, §16); // // 3. a manifest naming a state, stream, event or bucket of the indexer's placement (`mesh-delivery-indexer.placement…` -// or `mesh-delivery-indexer_placement…`) in any string value, such as `reads`. +// or `mesh-delivery-indexer_placement…`) in any string value, such as `reads`; +// +// 4. a manifest, other than mesh-controller's, that reads the controller's assignments (`mesh-controller.assignments`): +// every reader takes placement from the seat mesh-placement (novox/hq ADR 0549 §3, ADR 0567), and the lease write is +// refused while one still reads them (ADR 0549 §6; issue 774). The modules that still read them for what the +// placement does not carry are named in checks/noplacement/assignments-readers, each with why; that list only +// shrinks: an entry whose module no longer reads them is refused too. // // go run . // @@ -55,9 +61,74 @@ func Findings(root string) ([]string, error) { out = append(out, fmt.Sprintf("%s: names %q, which the indexer no longer keeps; placement is the seat mesh-placement's (novox/hq ADR 0567 §12)", module, s)) } } + readers, err := assignmentsReaders(root) + if err != nil { + return nil, err + } + out = append(out, readers...) + sort.Strings(out) return out, nil } +// AssignmentsRead is the controller's assignments as a manifest reads them, and Allowed the file, relative to the +// catalogue's root, that names the modules still reading them: ` ` per line, `#` a comment. +const ( + AssignmentsRead = "mesh-controller.assignments" + Allowed = "checks/noplacement/assignments-readers" +) + +// assignmentsReaders is every finding on the reads of the controller's assignments: a module other than the controller +// reading them that Allowed does not name, and a module Allowed names that no longer reads them (issue 774). +func assignmentsReaders(root string) ([]string, error) { + allowed := map[string]bool{} + raw, err := os.ReadFile(filepath.Join(root, Allowed)) + if err != nil && !os.IsNotExist(err) { + return nil, err + } + for _, line := range strings.Split(string(raw), "\n") { + if f := strings.Fields(line); len(f) > 0 && !strings.HasPrefix(f[0], "#") { + allowed[f[0]] = true + } + } + manifests, err := filepath.Glob(filepath.Join(root, "modules", "*", "module.json")) + if err != nil { + return nil, err + } + var found []string + reads := map[string]bool{} + for _, path := range manifests { + raw, err := os.ReadFile(path) + if err != nil { + return nil, err + } + var m struct { + Module string `json:"module"` + Reads []string `json:"reads"` + } + if json.Unmarshal(raw, &m) != nil || m.Module == "mesh-controller" { + continue // a manifest that does not read is the manifest checks' to refuse + } + for _, r := range m.Reads { + if strings.TrimSpace(r) != AssignmentsRead { + continue + } + reads[m.Module] = true + if !allowed[m.Module] { + found = append(found, fmt.Sprintf("%s reads %s: read where modules run from mesh-delivery-assigner.placement, "+ + "the seat mesh-placement's (novox/hq ADR 0549 §3, ADR 0567); the lease write is refused while a module reads "+ + "the assignments (ADR 0549 §6)", m.Module, AssignmentsRead)) + } + } + } + for module := range allowed { + if !reads[module] { + found = append(found, fmt.Sprintf("%s names %s, which no longer reads %s: take it off the list, which only shrinks", + Allowed, module, AssignmentsRead)) + } + } + return found, nil +} + // named collects the strings of a decoded manifest that start with one of the Prefixes. func named(v any, into []string) []string { switch v := v.(type) { @@ -95,5 +166,5 @@ func main() { if len(found) > 0 { os.Exit(1) } - fmt.Println("no placement directory, and no manifest reads the indexer's placement") + fmt.Println("no placement directory, no manifest reads the indexer's placement, and only the modules listed read the controller's assignments") } diff --git a/checks/noplacement/main_test.go b/checks/noplacement/main_test.go index 0ffc320f..efa884dc 100644 --- a/checks/noplacement/main_test.go +++ b/checks/noplacement/main_test.go @@ -5,6 +5,8 @@ import ( "path/filepath" "strings" "testing" + + "git.novox.be/novox/mesh-catalog/checks/trunk" ) func TestFindingsRefuseWhatThePlacementPathLeft(t *testing.T) { @@ -13,12 +15,18 @@ func TestFindingsRefuseWhatThePlacementPathLeft(t *testing.T) { files map[string]string want string // empty: clean }{ - {"clean", map[string]string{"modules/a/module.json": `{"module":"a","reads":["mesh-controller.assignments"]}`}, ""}, + {"clean", map[string]string{"modules/a/module.json": `{"module":"a","reads":["mesh-delivery-assigner.placement"]}`}, ""}, {"a placement directory", map[string]string{"placement/ace.json": `{}`}, "placement: the repository holds no placement"}, {"a particulars file", map[string]string{"particulars": "ace\n"}, "particulars: the repository lists no node names"}, {"a read of the indexer's placement", map[string]string{"modules/a/module.json": `{"module":"a","reads":["mesh-delivery-indexer.placement"]}`}, `a: names "mesh-delivery-indexer.placement"`}, {"a read of its candidates", map[string]string{"modules/b/module.json": `{"module":"b","reads":["mesh-delivery-indexer.placement-candidates"]}`}, "placement-candidates"}, {"its event", map[string]string{"modules/c/module.json": `{"module":"c","on":[{"event":"mesh-delivery-indexer.placement-changed"}]}`}, "placement-changed"}, + // Issue 774 (ADR 0549 §3, §6): a reader of the controller's assignments is refused unless the list names it, the + // controller's own manifest is not judged, and a name on the list whose module no longer reads them is refused. + {"an unlisted reader of the assignments", map[string]string{"modules/r/module.json": `{"module":"r","reads":["mesh-controller.assignments"]}`}, "r reads mesh-controller.assignments"}, + {"a listed reader and the controller", map[string]string{"modules/l/module.json": `{"module":"l","reads":["mesh-controller.assignments"]}`, + "modules/mesh-controller/module.json": `{"module":"mesh-controller","reads":["mesh-controller.assignments"]}`, Allowed: "# why\nl its plan\n"}, ""}, + {"a listed module that reads them no more", map[string]string{Allowed: "gone no longer reads\n"}, "names gone, which no longer reads"}, {"the seat's own state is allowed", map[string]string{"modules/d/module.json": `{"module":"d","reads":["mesh-placement.placement"]}`}, ""}, } { t.Run(row.name, func(t *testing.T) { @@ -52,4 +60,29 @@ func TestTheCatalogueHoldsNoPlacementOfTheRepositoryPath(t *testing.T) { for _, f := range got { t.Error(f) } + // Issue 774 (ADR 0549 §3, §6; the comparison of novox/hq ADR 0529 clause 2): the list of modules still reading the + // controller's assignments gains none the trunk's copy at the merge base does not hold. Its first copy gains nothing. + path, ok := trunk.Copy(t, Allowed) + if !ok { + return + } + names := func(path string) map[string]bool { + raw, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + out := map[string]bool{} + for _, line := range strings.Split(string(raw), "\n") { + if f := strings.Fields(line); len(f) > 0 && !strings.HasPrefix(f[0], "#") { + out[f[0]] = true + } + } + return out + } + base := names(path) + for module := range names(filepath.Join("..", "..", Allowed)) { + if !base[module] { + t.Errorf("%s gains %s, which the trunk's copy does not hold: the list only shrinks", Allowed, module) + } + } } diff --git a/modules/artifact-store-tools/cmd/artifact-store-tools/door.go b/modules/artifact-store-tools/cmd/artifact-store-tools/door.go new file mode 100644 index 00000000..1f010869 --- /dev/null +++ b/modules/artifact-store-tools/cmd/artifact-store-tools/door.go @@ -0,0 +1,129 @@ +package main + +// The artifact store's door and the bus, as the holder reaches them through this module's runtime. + +import ( + "context" + "encoding/json" + "fmt" + "os" + "strings" + "time" + + stdio "git.novox.be/novox/mesh-sdk/go" + + "artifactstoretools/internal/registry" +) + +// door is registry.Store at the store's address as this machine reaches it. +type door struct{ s registry.Store } + +func (d *door) Address() string { return d.s.Address } +func (d *door) Hold(ctx context.Context, r string) (bool, error) { return d.s.Hold(ctx, r) } +func (d *door) Holds(ctx context.Context, r string) error { return d.s.Holds(ctx, r) } +func (d *door) IsIndex(ctx context.Context, r string) (bool, error) { + return d.s.IsIndex(ctx, r) +} +func (d *door) LetGo(ctx context.Context, r string) error { return d.s.LetGo(ctx, r) } +func (d *door) SpareKept(ctx context.Context, keptImages, touching []string) error { + spare, err := d.s.SpareKeptIndexes(ctx, keptImages, touching) + if err == nil { + d.s.Spare = spare + } + return err +} + +// DoorAt reads the store's address (host:port) from the file the manifest fills from the seat mesh-artifact-store's +// reach, each time, so a store that moved is reached where it is now. +func DoorAt(file string) func() (Door, error) { + return func() (Door, error) { + if file == "" { + return nil, fmt.Errorf("this bundle was given no file naming the artifact store's address") + } + raw, err := os.ReadFile(file) + if err != nil { + return nil, fmt.Errorf("the artifact store's address: %w", err) + } + address := strings.TrimSpace(string(raw)) + if address == "" { + return nil, fmt.Errorf("the artifact store's address is empty in %s", file) + } + return &door{s: registry.Store{Address: address}}, nil + } +} + +// runtimeBus is the holder's bus through the runtime: its reads, its findings and its seat's event. +type runtimeBus struct{} + +func (runtimeBus) Keys(state string) ([]string, error) { return stdio.State(state).Keys() } +func (runtimeBus) Get(state, key string) ([]byte, bool, error) { + e, err := stdio.State(state).Get(key) + if err != nil || e == nil { + return nil, false, err + } + return e.Value, true, nil +} +func (runtimeBus) History(state, key string) ([]stdio.StateRevision, error) { + return stdio.State(state).History(key) +} +func (runtimeBus) LogSince(read string, at time.Time) ([]stdio.LogEntry, error) { + var out []stdio.LogEntry + page, err := stdio.Log(read).Since(at, "", 0) + for err == nil { + out = append(out, page.Entries...) + if page.Next == 0 { + return out, nil + } + page, err = stdio.Log(read).Read(page.Next, "", 0) + } + return nil, err +} + +func (runtimeBus) Put(state, key string, value any) error { + _, err := stdio.State(state).Put(key, value) + return err +} +func (runtimeBus) Delete(state, key string) error { return stdio.State(state).Delete(key) } +func (runtimeBus) Publish(subject string, body any) error { + return stdio.SeatPublish(subject, body, "") +} + +// follow watches the builder's done keys and the wants: a build's end sweeps (once the lease names no controller) and +// observes again; a want changed observes again. Event-driven: nothing reads them on a timer. +func follow(h *Holder) { + <-stdio.Served() + watch := func(state string, handle func(stdio.StateChange) error) { + for { + if err := stdio.State(state).Watch("", handle); err == nil { + return + } else { + h.logf("watching %s: %v; asked again in 10s", state, err) + } + time.Sleep(10 * time.Second) + } + } + observe := func() { + if err := h.Observe(context.Background()); err != nil { + h.logf("image-missing could not be observed: %v", err) + } + } + watch(BuildsRead, func(c stdio.StateChange) error { + if c.Current || c.Op != stdio.OpPut { + return nil + } + var b struct { + State string `json:"state"` + } + if json.Unmarshal(c.Value, &b) == nil && b.State == "done" { + go func() { h.AfterBuild(c.Key); observe() }() + } + return nil + }) + watch(WantsRead, func(c stdio.StateChange) error { + if !c.Current { + go observe() + } + return nil + }) + observe() +} diff --git a/modules/artifact-store-tools/cmd/artifact-store-tools/holder.go b/modules/artifact-store-tools/cmd/artifact-store-tools/holder.go new file mode 100644 index 00000000..5325b52f --- /dev/null +++ b/modules/artifact-store-tools/cmd/artifact-store-tools/holder.go @@ -0,0 +1,1020 @@ +package main + +// **The seat mesh-artifact-sweep** (novox/hq ADR 0571 clause 2, issue 1955): this module holds the artifact store's sweep, out +// of the controller. Its keep set is the store's own listing joined with every want's fingerprint (the assigner's +// `wants`) and each module's five newest builds (the builder's done keys, each build's end read from the key's history; +// ADR 0526 §1): a digest a want names is kept however old; a build beyond the five newest that no want names is +// eligible; a manifest in the store that no build names is unrecorded and never removed (ADR 0189 §10, ADR 0251 §6). +// +// At each build's end (a done key written) it sweeps as the controller's take-in did: every kept archive held first +// (ADR 0251 §8), then what is eligible let go of, oldest first, within the same bound per sweep, an index left for a +// `collect` a person confirms (ADR 0251 §7), each deletion recorded in the hand-act log (seat mesh-hand-acts) and said +// as artifacts-gone. **Nothing is let go until the controller's lease names a holder that is no controller**: until +// then the controller's take-in sweeps, and two sweeps never run at once. It also observes S25, image-missing: a want's +// image the store does not hold, a finding in its own state `findings` the condition keeper reads. + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "sort" + "strings" + "sync" + "time" + + stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/buscompose" + "git.novox.be/novox/mesh-sdk/go/catalogue" + "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/handacts" + "git.novox.be/novox/mesh-sdk/go/health" + + "artifactstoretools/internal/registry" +) + +// What the holder reads and keeps. +const ( + WantsRead = delivery.AssignerModule + ".wants" + BuildsRead = delivery.BuilderModule + ".builds" + HavesRead = "node-engine.have" + // ManifestsRead is the indexer's manifests: the images a manifest pins by itself. + ManifestsRead = delivery.IndexerModule + ".manifests" + LeaseRead = buscompose.ControllerLease + // FindingsState is the holder's own findings (health.FindingsState): image-missing. + FindingsState = health.FindingsState + // Module is this module, the observer its findings name. + Module = "artifact-store-tools" + // GoneSubject is where the seat says artifacts-gone. + GoneSubject = "mesh.seat." + catalogue.ArtifactSweepSeat + ".event." + delivery.ArtifactsGoneEvent +) + +// The sweep's bounds, the controller's (cmd/mesh-controller/collect.go mostPerSweep, sweepBudget). +const ( + KeptBuilds = 5 + MostPerSweep = 500 + MostAtAll = 5000 + SweepBudget = 45 * time.Second + ImageMissing = "image-missing" +) + +// Bus is what the holder reads and writes on the bus: its read states (wants, builds, haves, the lease), its own +// findings, and its seat's event. The runtime in main; a test's in memory. +type Bus interface { + Keys(state string) ([]string, error) + Get(state, key string) ([]byte, bool, error) + History(state, key string) ([]stdio.StateRevision, error) + // LogSince is every entry of a read log from a time on (stdio.Log(read).Since). + LogSince(read string, at time.Time) ([]stdio.LogEntry, error) + Put(state, key string, value any) error + Delete(state, key string) error + Publish(subject string, body any) error +} + +// Door is the artifact store's own door: registry.Store, behind an interface so a test needs no store. +type Door interface { + Address() string + Hold(ctx context.Context, reference string) (bool, error) + Holds(ctx context.Context, reference string) error + IsIndex(ctx context.Context, reference string) (bool, error) + LetGo(ctx context.Context, reference string) error + // SpareKept sets the platform manifests a kept index names aside before an index is let go of (ADR 0257). + SpareKept(ctx context.Context, keptImages, touching []string) error +} + +// Holder is the seat's holder. +type Holder struct { + Bus Bus + Door func() (Door, error) + // Listed is the store's own listing now (Store.View); nil reads none, and then nothing is said gone or collected. + Listed func(ctx context.Context) (*View, error) + Acts handacts.Recorder + Now func() time.Time + Logf func(string, ...any) + + sweep sync.Mutex // one sweep at a time + mu sync.Mutex + ended map[string]time.Time // a done key's end, read once from its history +} + +// Build is one done build of a module, as its key holds it, with when it ended. +type Build struct { + Module, FP, Key string + At time.Time + Artifacts []delivery.BuiltArtifact + // Manifest is the digest of the manifest it built, in the indexer's `manifests`. + Manifest string +} + +// LeaseAway says the controller's lease names a holder that is no controller: then the seat sweeps. +func (h *Holder) LeaseAway() bool { + raw, found, err := h.Bus.Get(LeaseRead, buscompose.ControllerLeaseKey) + away, _ := buscompose.LeaseHandedAway(raw, found, err) + return away +} + +// Builds is every done build the builder's keys hold, each with its end. +func (h *Holder) Builds() ([]Build, error) { + keys, err := h.Bus.Keys(BuildsRead) + if err != nil { + return nil, fmt.Errorf("the builder's builds: %w", err) + } + var out []Build + for _, k := range keys { + module, fp, err := delivery.ParseBuilderBuildKey(k) + if err != nil { + continue + } + raw, found, err := h.Bus.Get(BuildsRead, k) + if err != nil { + return nil, fmt.Errorf("the builder's build %s: %w", k, err) + } + if !found { + continue + } + b, err := delivery.DecodeBuilderBuild(raw) + if err != nil || b.State != delivery.BuildDone { + continue + } + at, err := h.endOf(k) + if err != nil { + return nil, err + } + out = append(out, Build{Module: module, FP: fp, Key: k, At: at, Artifacts: b.Artifacts, Manifest: b.Manifest}) + } + return out, nil +} + +// endOf is when a done key was last written done, read once from its history: a done key is not written again. +func (h *Holder) endOf(key string) (time.Time, error) { + h.mu.Lock() + at, known := h.ended[key] + h.mu.Unlock() + if known { + return at, nil + } + revs, err := h.Bus.History(BuildsRead, key) + if err != nil { + return time.Time{}, fmt.Errorf("the history of the builder's build %s: %w", key, err) + } + for _, r := range revs { + if b, err := delivery.DecodeBuilderBuild(r.Value); err == nil && b.State == delivery.BuildDone && r.At.After(at) { + at = r.At + } + } + h.mu.Lock() + if h.ended == nil { + h.ended = map[string]time.Time{} + } + h.ended[key] = at + h.mu.Unlock() + return at, nil +} + +// Want is one node's want for a module. +type Want struct{ Node, Module, FP string } + +// Wants is every want that names a fingerprint. +func (h *Holder) Wants() ([]Want, error) { + keys, err := h.Bus.Keys(WantsRead) + if err != nil { + return nil, fmt.Errorf("the wants: %w", err) + } + var out []Want + for _, k := range keys { + node, module, ok := wantKey(k) + if !ok { + continue + } + raw, found, err := h.Bus.Get(WantsRead, k) + if err != nil { + return nil, fmt.Errorf("the want %s: %w", k, err) + } + var w delivery.Want + if !found || json.Unmarshal(raw, &w) != nil || w.Removed || w.FP == "" { + continue + } + out = append(out, Want{Node: node, Module: module, FP: w.FP}) + } + return out, nil +} + +// wantKey reads `want..`; the node's own key and a hold are not a module's want. +func wantKey(key string) (node, module string, ok bool) { + parts := strings.Split(key, ".") + if len(parts) != 3 || parts[0] != "want" || parts[2] == "node" { + return "", "", false + } + return parts[1], parts[2], true +} + +// Ref is one artifact the mesh made, as the keep set judges it. +type Ref struct { + Reference string + Kind string // image or archive + Builds []Build + Why []string + Kept bool + // At is its newest build's end: the oldest go first. + At time.Time +} + +// KeepSet judges every artifact the done builds name in the store: kept, and why, or eligible. Packages and +// references into no store of the mesh's are no business of the store's. +func KeepSet(builds []Build, wants []Want) map[string]*Ref { + keeps := keptBuilds(builds, wants) + out := map[string]*Ref{} + for _, b := range builds { + for _, a := range b.Artifacts { + reference := catalogue.Recorded(a.Reference) + if _, ours := catalogue.InArtifactStore(reference); !ours || a.Kind == "package" { + continue + } + r := out[reference] + if r == nil { + _, _, kind := splitReference(reference) + r = &Ref{Reference: reference, Kind: kind} + out[reference] = r + } + r.Builds = append(r.Builds, b) + r.keptBy(b, keeps[b.Key]) + } + } + return out +} + +// keptBy adds a build naming r, kept for why (none: eligible). +func (r *Ref) keptBy(b Build, why []string) { + if b.At.After(r.At) { + r.At = b.At + } + if len(why) > 0 { + r.Kept = true + for _, w := range why { + if !contains(r.Why, w) { + r.Why = append(r.Why, w) + } + } + } +} + +// keptBuilds is, by build key, why each kept build is kept: one of its module's five newest, or a want names it. +func keptBuilds(builds []Build, wants []Want) map[string][]string { + wanted := map[string][]string{} + for _, w := range wants { + wanted[w.Module+" "+w.FP] = append(wanted[w.Module+" "+w.FP], w.Node) + } + byModule := map[string][]Build{} + for _, b := range builds { + byModule[b.Module] = append(byModule[b.Module], b) + } + keeps := map[string][]string{} // build key → why + for module, bs := range byModule { + sort.Slice(bs, func(i, j int) bool { + if !bs[i].At.Equal(bs[j].At) { + return bs[i].At.After(bs[j].At) + } + return bs[i].Key > bs[j].Key + }) + for i, b := range bs { + if i < KeptBuilds { + keeps[b.Key] = append(keeps[b.Key], fmt.Sprintf("one of the %d newest builds of %s", KeptBuilds, module)) + } + if nodes := wanted[module+" "+b.FP]; len(nodes) > 0 { + sort.Strings(nodes) + keeps[b.Key] = append(keeps[b.Key], "a want names it on "+strings.Join(nodes, ", ")) + } + } + } + return keeps +} + +func contains(s []string, v string) bool { + for _, x := range s { + if x == v { + return true + } + } + return false +} + +// inStore says whether the listing holds a reference: an image's manifest, an archive's blob. Unknown without a listing. +func inStore(v *View, reference string) (held, known bool) { + if v == nil || v.L == nil { + return false, false + } + repo, digest, kind := splitReference(reference) + switch kind { + case "image": + r := v.L.Repos[repo] + return r != nil && r.Revisions[digest], true + case "archive": + _, has := v.L.Blobs[digest] + return has, true + } + return false, false +} + +// judged is the keep set now, with the listing it was read against. +type judged struct { + refs map[string]*Ref + view *View + wants []Want + built []Build + // mirrors are the copies in the store's mirror repositories, by reference: recorded ones also in refs. + mirrors map[string]*Ref + // mirrorsHeld says why no copy is eligible now, when one is not: a kept build whose bases the log does not name. + mirrorsHeld string +} + +func (h *Holder) judge(ctx context.Context) (*judged, error) { + builds, err := h.Builds() + if err != nil { + return nil, err + } + wants, err := h.Wants() + if err != nil { + return nil, err + } + j := &judged{refs: KeepSet(builds, wants), wants: wants, built: builds} + if h.Listed != nil { + if j.view, err = h.Listed(ctx); err != nil { + h.logf("the store's listing could not be read, so nothing is said gone: %v", err) + j.view = nil + } + } + if j.view != nil { + against, err := h.against() + if err != nil { + h.logf("the builder's log could not be read, so every copy is kept as unrecorded: %v", err) + against = nil + } + j.mirrors, j.mirrorsHeld = Mirrors(j.view, builds, keptBuilds(builds, wants), against) + for r, m := range j.mirrors { + if len(m.Builds) > 0 { + j.refs[r] = m + } + } + } + return j, nil +} + +// against is what each build the builder's log says ended stood on, by build key: the bases' references. +func (h *Holder) against() (map[string][]string, error) { + entries, err := h.Bus.LogSince(delivery.BuildLogRead, h.now().Add(-delivery.BuilderLogDays*24*time.Hour)) + if err != nil { + return nil, err + } + out := map[string][]string{} + for _, raw := range entries { + e, err := delivery.DecodeBuildLogEntry(raw.Value) + if err != nil || e.Event != delivery.BuildEndedEvent { + continue + } + key, err := delivery.BuilderBuildKey(e.Module, e.FP) + if err != nil { + continue + } + out[key] = e.Against + } + return out, nil +} + +// Mirrors judges each copy in the store's mirror repositories (upstream//, /on-; ADR +// 0257) by what the builder's log says each build stood on (ADR 0571 clause 1): named by a kept build, kept; named +// only by builds no longer kept, eligible; named by no build the log holds, unrecorded and never removed (ADR 0189 +// §10). While a kept build's bases are not in the log (it ended before the log's 30 days), no copy is eligible: what +// it stood on cannot be told, and held says so. +func Mirrors(v *View, builds []Build, kept map[string][]string, against map[string][]string) (map[string]*Ref, string) { + out := map[string]*Ref{} + for _, name := range v.L.RepoNames() { + if !mirrorRepository.MatchString(name) { + continue + } + for d := range v.L.Repos[name].Revisions { + r := catalogue.ArtifactStoreScheme + name + "@" + d + out[r] = &Ref{Reference: r, Kind: "image"} + } + } + held := "" + for _, b := range builds { + bases, logged := against[b.Key] + if !logged { + if len(kept[b.Key]) > 0 && held == "" { + held = "the builder's log does not say what the kept build " + b.Key + " stood on, so every copy is kept" + } + continue + } + for _, base := range bases { + if m := out[catalogue.Recorded(base)]; m != nil { + m.Builds = append(m.Builds, b) + why := kept[b.Key] + if len(why) > 0 { + why = []string{"a kept build stood on it: " + b.Key} + } + m.keptBy(b, why) + } + } + } + if held != "" { + for _, m := range out { + if len(m.Builds) > 0 && !m.Kept { + m.Kept, m.Why = true, []string{held} + } + } + } + return out, held +} + +// Artifacts is the seat's `artifacts`, in the controller's answer's shape (Records): every reference the mesh made that +// lives in the store, kept with why or eligible, and with collected the ones the store no longer holds. +func (h *Holder) Artifacts(repository string) (*Records, error) { + j, err := h.judge(context.Background()) + if err != nil { + return nil, err + } + r := j.records(repository, true) + if d, err := h.Door(); err == nil { + r.Store = d.Address() + } + return r, nil +} + +func (j *judged) records(repository string, collected bool) *Records { + r := &Records{KeptBuilds: KeptBuilds, Counts: map[string]int{"kept": 0, "eligible": 0, "collected": 0}, CollectedAsked: collected} + for _, ref := range j.sorted() { + state := "eligible" + if ref.Kept { + state = "kept" + } + if held, known := inStore(j.view, ref.Reference); known && !held { + state = "collected" + if ref.Kept { + state = "kept" // a kept one gone is said missing by collect, never let go of + } + } + r.Counts[state]++ + if (repository != "" && !strings.HasPrefix(strings.TrimPrefix(ref.Reference, catalogue.ArtifactStoreScheme), repository)) || + (state == "collected" && !collected) { + continue + } + rec := Record{Reference: ref.Reference, Kind: ref.Kind, State: state, Why: ref.Why} + fill(&rec) + r.References = append(r.References, rec) + } + if j.view == nil { + r.Note = "the store's listing could not be read, so a reference it no longer holds is not said collected" + } + return r +} + +// sorted is the references oldest first, then by reference. +func (j *judged) sorted() []*Ref { + out := make([]*Ref, 0, len(j.refs)) + for _, r := range j.refs { + out = append(out, r) + } + sort.Slice(out, func(a, b int) bool { + if !out[a].At.Equal(out[b].At) { + return out[a].At.Before(out[b].At) + } + return out[a].Reference < out[b].Reference + }) + return out +} + +// Collect is the seat's `collect`, in the controller's answer's shape (CollectAnswer): a dry run unless confirm; with +// confirm and why, a sweep that also lets indexes go with their platform manifests. +func (h *Holder) Collect(why string, confirm bool, most int) (*CollectAnswer, error) { + if confirm && strings.TrimSpace(why) == "" { + return nil, errors.New("a confirmed collect needs why: nothing was let go of") + } + if most <= 0 { + most = MostPerSweep + } + if most > MostAtAll { + return nil, fmt.Errorf("collect lets go of at most %d at once, not %d", MostAtAll, most) + } + if confirm && !h.LeaseAway() { + return nil, errors.New("the controller holds its lease, and its take-in sweeps the store until it hands it away: " + + "this seat lets nothing go before then (novox/hq ADR 0571 clause 2); ask the controller's collect") + } + return h.Sweep(context.Background(), Sweeping{Confirmed: confirm, DryRun: !confirm, Most: most, Why: why, By: "collect"}) +} + +// Sweeping is one sweep's asks. +type Sweeping struct { + // Confirmed is a collect a person confirmed: an index goes with its platform manifests. + Confirmed bool + // DryRun lets nothing go and holds nothing: what would go is listed. + DryRun bool + Most int + Why string + // By names what asked: collect, or the build whose end it follows. + By string +} + +// Sweep holds every kept archive, then lets go of what is eligible, oldest first, within the bounds, recording each +// deletion and saying artifacts-gone. +func (h *Holder) Sweep(ctx context.Context, s Sweeping) (*CollectAnswer, error) { + h.sweep.Lock() + defer h.sweep.Unlock() + j, err := h.judge(ctx) + if err != nil { + return nil, err + } + a := &CollectAnswer{DryRun: s.DryRun, WouldLetGo: []string{}, LetGo: []string{}} + var keptArchives, keptImages, eligible []string + for _, r := range j.sorted() { + switch { + case r.Kept && r.Kind == "archive": + keptArchives = append(keptArchives, r.Reference) + case r.Kept: + keptImages = append(keptImages, r.Reference) + default: + if held, known := inStore(j.view, r.Reference); known && !held { + continue // collected already + } + eligible = append(eligible, r.Reference) + } + } + a.KeptArchives, a.Eligible = len(keptArchives), len(eligible) + if s.DryRun { + for i, r := range eligible { + if i >= s.Most { + a.Left = len(eligible) - i + break + } + a.WouldLetGo = append(a.WouldLetGo, r) + } + return a, nil + } + door, err := h.Door() + if err != nil { + return nil, err + } + a.Store = door.Address() + within, stop := context.WithTimeout(ctx, SweepBudget) + defer stop() + // Hold before letting go (ADR 0251 §8): a kept archive that cannot be held stops the sweep before any delete. + var missing []string + for _, r := range keptArchives { + wrote, err := door.Hold(within, r) + switch { + case err == nil: + if wrote { + a.HoldersWritten++ + } + case errors.Is(err, registry.Gone): + missing = append(missing, r) + case errors.Is(err, registry.ErrNotOurs): + default: + a.Stopped = fmt.Sprintf("not every kept archive could be held, so nothing was let go: holding %s: %v", r, err) + a.Left = len(eligible) + return a, nil + } + } + a.Held, a.Missing = len(keptArchives)-len(missing), len(missing) + h.sayGone(j, missing, "a kept archive the store does not hold") + if s.Confirmed { + if err := door.SpareKept(within, keptImages, eligible); err != nil { + a.Stopped = fmt.Sprintf("which platform manifests a kept index names could not be read, so nothing was let go: %v", err) + a.Left = len(eligible) + return a, nil + } + } + for i, r := range eligible { + if i >= s.Most || within.Err() != nil { + a.Left = len(eligible) - i + if i >= s.Most { + a.Stopped = fmt.Sprintf("asked about %d, the most one sweep asks", s.Most) + } else { + a.Stopped = fmt.Sprintf("ran out of its %s", SweepBudget) + } + break + } + if !s.Confirmed { + // An index waits for a confirmed collect (ADR 0251 §7, ADR 0257 §4). + index, err := door.IsIndex(within, r) + if err != nil && !errors.Is(err, registry.Gone) && !errors.Is(err, registry.ErrNotOurs) { + a.Stopped = fmt.Sprintf("the store could not say whether %s is an index, so nothing more was asked: %v", r, err) + a.Left = len(eligible) - i + break + } + if index { + a.Skipped++ + continue + } + } + err := door.LetGo(within, r) + switch { + case err == nil || errors.Is(err, registry.Gone): + case errors.Is(err, registry.ErrNotOurs): + a.Skipped++ + continue + default: + a.Stopped = fmt.Sprintf("the store refused to let go of %s, so nothing more was asked: %v", r, err) + a.Left = len(eligible) - i + } + if a.Stopped != "" { + break + } + // Recorded in the hand-act log, each one (ADR 0565 §10; the operator's "option A"). + why := s.Why + if why == "" { + why = "eligible: no want names it and it is not among its module's " + fmt.Sprint(KeptBuilds) + " newest builds" + } + if _, err := h.Acts.Record(handacts.Request{What: "let go of " + r + " (" + s.By + ")", Why: why, + Cause: "artifact-store-sweep", Verb: "collect"}); err != nil { + h.logf("%s was let go of and could not be recorded in the hand-act log: %v", r, err) + } + a.LetGo = append(a.LetGo, r) + } + h.sayGone(j, a.LetGo, "let go of") + return a, nil +} + +// sayGone says artifacts-gone for each build that named one of refs: the builder builds a key again whose artifact is gone. +func (h *Holder) sayGone(j *judged, refs []string, how string) { + byKey := map[string]*delivery.ArtifactsGone{} + var keys []string + for _, r := range refs { + ref := j.refs[r] + if ref == nil { + continue + } + for _, b := range ref.Builds { + g := byKey[b.Key] + if g == nil { + g = &delivery.ArtifactsGone{Module: b.Module, Key: b.Key, Fingerprint: b.FP, At: h.now().UTC()} + byKey[b.Key] = g + keys = append(keys, b.Key) + } + g.Gone = append(g.Gone, r) + } + } + sort.Strings(keys) + for _, k := range keys { + raw, err := delivery.EncodeArtifactsGone(*byKey[k]) + if err == nil { + err = h.Bus.Publish(GoneSubject, json.RawMessage(raw)) + } + if err != nil { + h.logf("%s: %s %s, and artifacts-gone could not be said: %v", k, how, strings.Join(byKey[k].Gone, ", "), err) + } + } +} + +// AfterBuild is the sweep at a build's end: only once the lease names no controller. +func (h *Holder) AfterBuild(key string) { + if !h.LeaseAway() { + return + } + a, err := h.Sweep(context.Background(), Sweeping{Most: MostPerSweep, By: "after the build " + key}) + if err != nil { + h.logf("the sweep after %s: %v", key, err) + return + } + if len(a.LetGo) > 0 || a.Stopped != "" || a.Missing > 0 { + h.logf("the sweep after %s let go of %d (%d left, %d missing): %s", key, len(a.LetGo), a.Left, a.Missing, a.Stopped) + } +} + +// Held is the seat's `held`: each reference, held, gone or unknown with why. Holds and writes nothing. +func (h *Holder) Held(refs []string) (delivery.HeldAnswer, error) { + if len(refs) == 0 { + return delivery.HeldAnswer{}, errors.New("held judges the references given, and none was") + } + if len(refs) > 64 { + return delivery.HeldAnswer{}, fmt.Errorf("held judges at most 64 references at once, and %d were given", len(refs)) + } + for _, r := range refs { + if _, ours := catalogue.InArtifactStore(r); !ours { + return delivery.HeldAnswer{}, fmt.Errorf("%s is not a reference into the mesh's artifact store (artifact-store://...): nothing was asked", r) + } + } + door, err := h.Door() + if err != nil { + return delivery.HeldAnswer{}, err + } + asking, stop := context.WithTimeout(context.Background(), 20*time.Second) + defer stop() + a := delivery.HeldAnswer{Store: door.Address(), Held: []string{}, Gone: []string{}} + for _, r := range refs { + switch err := door.Holds(asking, r); { + case err == nil: + a.Held = append(a.Held, r) + case errors.Is(err, registry.Gone): + a.Gone = append(a.Gone, r) + default: + if a.Unknown == nil { + a.Unknown = map[string]string{} + } + a.Unknown[r] = err.Error() + } + } + return a, nil +} + +// Image is one image a machine's wants and haves name, as docker reads the answer. +type Image struct { + Image string `json:"image"` + Resources []string `json:"resources"` + In []string `json:"in"` +} + +// Images is the seat's `images`, the controller's answer's shape: every image one machine's wants (in: declared) and +// haves (in: sent) name, by the builds their fingerprints name, each with the module naming it. +type Images struct { + Node string `json:"node"` + Images []Image `json:"images"` + SentKnown bool `json:"sent_known"` +} + +// ImagesOf answers `images` for node. +func (h *Holder) ImagesOf(node string, address string) (Images, error) { + out := Images{Node: node, Images: []Image{}} + wants, err := h.Wants() + if err != nil { + return out, err + } + byImage := map[string]*Image{} + add := func(module, fp, in string) error { + key, err := delivery.BuilderBuildKey(module, fp) + if err != nil { + return nil + } + raw, found, err := h.Bus.Get(BuildsRead, key) + if err != nil || !found { + return err + } + b, err := delivery.DecodeBuilderBuild(raw) + if err != nil { + return nil + } + images := []string{} + for _, a := range b.Artifacts { + if (a.Kind == "image" || a.Kind == "upstream") && a.Reference != "" { + images = append(images, catalogue.Routed(catalogue.Recorded(a.Reference), address)) + } + } + pinned, err := h.Pinned(b.Manifest) + if err != nil { + return err + } + for _, image := range append(images, pinned...) { + i := byImage[image] + if i == nil { + i = &Image{Image: image} + byImage[image] = i + } + if !contains(i.Resources, module) { + i.Resources = append(i.Resources, module) + } + if !contains(i.In, in) { + i.In = append(i.In, in) + } + } + return nil + } + for _, w := range wants { + if w.Node == node { + if err := add(w.Module, w.FP, "declared"); err != nil { + return out, err + } + } + } + keys, err := h.Bus.Keys(HavesRead) + if err != nil { + return out, fmt.Errorf("the haves: %w", err) + } + for _, k := range keys { + n, module, isNode, err := delivery.ParseHaveKey(k) + if err != nil || isNode || n != node { + continue + } + raw, found, err := h.Bus.Get(HavesRead, k) + if err != nil { + return out, err + } + var have delivery.Have + if !found || json.Unmarshal(raw, &have) != nil || have.FP == "" { + continue + } + out.SentKnown = true + if err := add(module, have.FP, "sent"); err != nil { + return out, err + } + } + for _, i := range byImage { + sort.Strings(i.Resources) + sort.Strings(i.In) + out.Images = append(out.Images, *i) + } + sort.Slice(out.Images, func(a, b int) bool { return out.Images[a].Image < out.Images[b].Image }) + return out, nil +} + +// Pinned is every container image a manifest names by itself, by digest (`/@sha256:…`), read from the +// indexer's manifests: what a definition pins outright rather than through a build's artifact. A placeholder is not one. +func (h *Holder) Pinned(manifest string) ([]string, error) { + if manifest == "" { + return nil, nil + } + key, err := delivery.ManifestsKey(manifest) + if err != nil { + return nil, nil + } + raw, found, err := h.Bus.Get(ManifestsRead, key) + if err != nil || !found { + return nil, err + } + m, err := delivery.DecodeManifest(raw) + if err != nil { + return nil, nil + } + var parsed struct { + Resources []map[string]any `json:"resources"` + } + if json.Unmarshal([]byte(m.JSON), &parsed) != nil { + return nil, nil + } + var out []string + for _, r := range parsed.Resources { + image, _ := r["image"].(string) + if fmt.Sprint(r["type"]) == "container" && strings.Contains(image, "@sha256:") && !strings.Contains(image, "${") && + !contains(out, image) { + out = append(out, image) + } + } + return out, nil +} + +// inStoreAt is the store's reference of an image its address names, "" for an image of another registry (the +// controller's imageInStore). +func inStoreAt(image, address string) string { + if address == "" || !strings.HasPrefix(image, address+"/") || !strings.Contains(image, "@sha256:") { + return "" + } + r := catalogue.Recorded(image) + if _, ours := catalogue.InArtifactStore(r); !ours { + return "" + } + return r +} + +// Observe writes S25's findings (image-missing): each want whose build names an image the store does not hold, on the +// want's machine, and deletes each finding that no longer holds. Nothing is said without a listing. +func (h *Holder) Observe(ctx context.Context) error { + j, err := h.judge(ctx) + if err != nil { + return err + } + if j.view == nil { + return nil + } + address := "" + if d, err := h.Door(); err == nil { + address = d.Address() + } + builds := map[string]Build{} + for _, b := range j.built { + builds[b.Module+" "+b.FP] = b + } + now := map[string]health.Finding{} + for _, w := range j.wants { + b, ok := builds[w.Module+" "+w.FP] + if !ok { + continue + } + var refs []string + for _, a := range b.Artifacts { + if a.Kind == "image" { + refs = append(refs, catalogue.Recorded(a.Reference)) + } + } + // And what its manifest pins by itself under the store's address, as the controller judged S25. + if pinned, err := h.Pinned(b.Manifest); err == nil && address != "" { + for _, image := range pinned { + if r := inStoreAt(image, address); r != "" { + refs = append(refs, r) + } + } + } + for _, reference := range refs { + if held, known := inStore(j.view, reference); !known || held { + continue + } + f := imageMissing(w.Node, w.Module, reference, h.now()) + now[f.Key] = f + break + } + } + keys, err := h.Bus.Keys(FindingsState) + if err != nil { + return fmt.Errorf("the findings: %w", err) + } + for _, k := range keys { + if _, still := now[k]; !still { + if err := h.Bus.Delete(FindingsState, k); err != nil { + return err + } + } + } + for k, f := range now { + raw, found, err := h.Bus.Get(FindingsState, k) + if err != nil { + return err + } + if found { + if was, err := health.DecodeFinding(raw); err == nil && was.Summary == f.Summary { + continue // written only when what it says changes + } + } + if _, err := health.EncodeFinding(f); err != nil { + return err + } + if err := h.Bus.Put(FindingsState, k, f); err != nil { + return err + } + } + return nil +} + +// imageMissing is S25's finding, in the controller's words (cmd/mesh-controller/image_missing.go watchMissingImages). +func imageMissing(node, module, image string, at time.Time) health.Finding { + id := module + "." + node + return health.Finding{V: health.FindingVersion, Key: health.FindingKey(health.ScopeModule, id, ImageMissing), Kind: ImageMissing, + Subject: health.Subject{Scope: health.ScopeModule, ID: id, Machine: node}, + Severity: health.Warning, Resolver: health.ResolverOperator, + Summary: fmt.Sprintf("%s is left out of every send to %s: its declaration names an image that the artifact store does "+ + "not hold (the evidence names it); the running container is untouched", module, node), + Headline: capital(module) + " is not updated on " + node, + Explanation: fmt.Sprintf("The new version of %s names a container image that the mesh's store does not hold, so it "+ + "was not sent to %s. What runs there is untouched, and the rest of %s is sent as usual.", module, node, node), + Needs: fmt.Sprintf("build %s again; the details name the image.", module), + Resolved: "Resolved: the store holds the image of " + module + " again", + Evidence: []health.Evidence{{At: at.UTC(), Said: "image " + image + " is not in the store"}}, + Source: "S25", Observer: Module, At: at.UTC()} +} + +func capital(s string) string { + if s == "" { + return s + } + return strings.ToUpper(s[:1]) + s[1:] +} + +func (h *Holder) now() time.Time { + if h.Now != nil { + return h.Now() + } + return time.Now() +} + +func (h *Holder) logf(format string, a ...any) { + if h.Logf != nil { + h.Logf(format, a...) + } +} + +// Mirror is one copy as the seat's `mirrors` answers it. +type Mirror struct { + Reference string `json:"reference"` + State string `json:"state"` + Why []string `json:"why,omitempty"` + StoodOn []string `json:"stood_on_by,omitempty"` +} + +// MirrorsAnswer is the seat's `mirrors`: every copy, and why none is eligible when a kept build's bases are unknown. +type MirrorsAnswer struct { + Copies []Mirror `json:"copies"` + Held string `json:"held,omitempty"` +} + +// MirrorsNow is every copy in the store's mirror repositories as the keep set judges it now. +func (h *Holder) MirrorsNow() (MirrorsAnswer, error) { + j, err := h.judge(context.Background()) + if err != nil { + return MirrorsAnswer{}, err + } + if j.view == nil { + return MirrorsAnswer{}, errors.New("the store's listing could not be read, so no copy can be named") + } + out := MirrorsAnswer{Copies: []Mirror{}, Held: j.mirrorsHeld} + for _, r := range j.mirrors { + m := Mirror{Reference: r.Reference, State: "unrecorded", Why: r.Why} + switch { + case r.Kept: + m.State = "kept" + case len(r.Builds) > 0: + m.State = "eligible" + } + for _, b := range r.Builds { + m.StoodOn = append(m.StoodOn, b.Key) + } + out.Copies = append(out.Copies, m) + } + sort.Slice(out.Copies, func(a, b int) bool { return out.Copies[a].Reference < out.Copies[b].Reference }) + return out, nil +} diff --git a/modules/artifact-store-tools/cmd/artifact-store-tools/holder_integration_test.go b/modules/artifact-store-tools/cmd/artifact-store-tools/holder_integration_test.go new file mode 100644 index 00000000..f1264823 --- /dev/null +++ b/modules/artifact-store-tools/cmd/artifact-store-tools/holder_integration_test.go @@ -0,0 +1,260 @@ +//go:build integration + +package main + +// The seat mesh-artifact-sweep's holder whole against a real bus (novox/hq ADR 0571 clause 2, issue 1955): the builder's done +// keys, the wants and the controller's lease in a real nats-server's key-value buckets, a build's end heard by a watch +// of the builds as the runtime hands it over, each deletion recorded by a request to the seat mesh-hand-acts on the bus, +// artifacts-gone said on the seat's subject, and S25's finding written to the holder's own bucket. Only the store's +// door is in memory: its HTTP is the copied registry package's, tested there. + +import ( + "context" + "encoding/json" + "strings" + "sync" + "testing" + "time" + + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" + + stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/buscompose" + "git.novox.be/novox/mesh-sdk/go/handacts" + "git.novox.be/novox/mesh-sdk/go/testbus" +) + +// realBus is the holder's Bus over a real nats-server: each read state a bucket named as the runtime names it. +type realBus struct { + ctx context.Context + nc *nats.Conn + js jetstream.JetStream + // log is the builder's log as the runtime's log verbs would read it (mesh-tools adds them; the sdk's own read of a + // log is tested in mesh-sdk's process package). + log []stdio.LogEntry +} + +func (b realBus) LogSince(_ string, at time.Time) ([]stdio.LogEntry, error) { + var out []stdio.LogEntry + for _, e := range b.log { + if !e.At.Before(at) { + out = append(out, e) + } + } + return out, nil +} + +func bucketOf(state string) string { return strings.Replace(state, ".", "_", 1) } + +func (b realBus) kv(state string) (jetstream.KeyValue, error) { + return b.js.KeyValue(b.ctx, bucketOf(state)) +} + +func (b realBus) Keys(state string) ([]string, error) { + kv, err := b.kv(state) + if err != nil { + return nil, err + } + keys, err := kv.Keys(b.ctx) + if err == jetstream.ErrNoKeysFound { + return nil, nil + } + return keys, err +} + +func (b realBus) Get(state, key string) ([]byte, bool, error) { + kv, err := b.kv(state) + if err != nil { + return nil, false, err + } + e, err := kv.Get(b.ctx, key) + if err == jetstream.ErrKeyNotFound { + return nil, false, nil + } + if err != nil { + return nil, false, err + } + return e.Value(), true, nil +} + +func (b realBus) History(state, key string) ([]stdio.StateRevision, error) { + kv, err := b.kv(state) + if err != nil { + return nil, err + } + h, err := kv.History(b.ctx, key) + if err != nil { + return nil, err + } + var out []stdio.StateRevision + for _, e := range h { + out = append(out, stdio.StateRevision{Key: key, Value: e.Value(), Revision: e.Revision(), At: e.Created()}) + } + return out, nil +} + +func (b realBus) Put(state, key string, value any) error { + kv, err := b.kv(state) + if err != nil { + return err + } + raw, _ := json.Marshal(value) + _, err = kv.Put(b.ctx, key, raw) + return err +} + +func (b realBus) Delete(state, key string) error { + kv, err := b.kv(state) + if err != nil { + return err + } + return kv.Delete(b.ctx, key) +} + +func (b realBus) Publish(subject string, body any) error { + raw, _ := json.Marshal(body) + return b.nc.Publish(subject, raw) +} + +func TestTheSeatSweepsAtABuildsEndOnARealBusOnceTheLeaseIsAway(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + nc, err := nats.Connect(testbus.URL(t)) + if err != nil { + t.Fatal(err) + } + defer nc.Close() + js, err := jetstream.New(nc) + if err != nil { + t.Fatal(err) + } + for _, s := range []string{BuildsRead, WantsRead, HavesRead, LeaseRead, ManifestsRead, "artifact-store-tools." + FindingsState} { + if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: bucketOf(s), History: 5}); err != nil { + t.Fatal(err) + } + } + bus := realBus{ctx: ctx, nc: nc, js: js} + + // The mesh as aMesh lays it, copied onto the real buckets: letta's seven builds, a want, the store's listing. + mem, memb, door, _ := aMesh(t, &buscompose.LeaseHolder{Instance: "c1", Host: "novox", Epoch: 3}) + bus.log = memb.log + for _, s := range []string{BuildsRead, WantsRead, LeaseRead, ManifestsRead} { + keys, _ := memb.Keys(s) + for _, k := range keys[:min(len(keys), 6)] { // the seventh build ends later, on the bus + raw, _, _ := memb.Get(s, k) + if err := bus.Put(s, k, json.RawMessage(raw)); err != nil { + t.Fatal(err) + } + } + } + time.Sleep(20 * time.Millisecond) // each build's end is its key's time on the bus: the seventh the newest + + // The seat mesh-hand-acts, answering every act it records; artifacts-gone heard on the seat's subject. + var mu sync.Mutex + var acts, gone []string + if _, err := nc.Subscribe("mesh.seat."+handacts.Seat+".tool.hand-act", func(m *nats.Msg) { + mu.Lock() + acts = append(acts, string(m.Data)) + mu.Unlock() + _ = m.Respond([]byte(`{"ok":true}`)) + }); err != nil { + t.Fatal(err) + } + if _, err := nc.Subscribe(GoneSubject, func(m *nats.Msg) { + mu.Lock() + gone = append(gone, string(m.Data)) + mu.Unlock() + }); err != nil { + t.Fatal(err) + } + h := &Holder{Bus: findingsAt{realBus: bus}, Door: mem.Door, Listed: mem.Listed, Now: time.Now, Logf: t.Logf} + h.Acts = handacts.Recorder{LeaseAway: h.LeaseAway, Ask: func(key string, body any) (json.RawMessage, error) { + raw, _ := json.Marshal(body) + m, err := nc.Request("mesh.seat."+strings.TrimPrefix(strings.Replace(key, ".", ".tool.", 1), "seat:"), raw, 5*time.Second) + if err != nil { + return nil, err + } + return m.Data, nil + }} + + // A build's end heard from the builds bucket, as follow hands it over. + kv, _ := bus.kv(BuildsRead) + w, err := kv.WatchAll(ctx, jetstream.UpdatesOnly()) + if err != nil { + t.Fatal(err) + } + ended := func() { + keys, _ := memb.Keys(BuildsRead) + raw, _, _ := memb.Get(BuildsRead, keys[6]) + if err := bus.Put(BuildsRead, keys[6], json.RawMessage(raw)); err != nil { + t.Fatal(err) + } + select { + case e := <-w.Updates(): + h.AfterBuild(e.Key()) + case <-ctx.Done(): + t.Fatal("the build's end was not heard") + } + } + + // While the lease names the controller, a build's end lets nothing go. + ended() + if len(door.asked) != 0 { + t.Fatalf("before the lease write the store was asked %v", door.asked) + } + // The lease handed away: the next build's end holds the kept archives and lets go of letta's 2nd, which the 7th moved + // past the five newest and no want names (the want keeps the 1st), recording each deletion. + if err := bus.Put(LeaseRead, buscompose.ControllerLeaseKey, buscompose.LeaseHolder{HandedTo: "mesh-grants", Why: "the lease write"}); err != nil { + t.Fatal(err) + } + ended() + deadline := time.Now().Add(5 * time.Second) + for { + mu.Lock() + n := len(gone) + mu.Unlock() + if n > 0 || time.Now().After(deadline) { + break + } + time.Sleep(20 * time.Millisecond) + } + mu.Lock() + defer mu.Unlock() + if len(acts) != 3 || !strings.Contains(acts[0], archiveRef("letta", 2)) || !strings.Contains(acts[1], imageRef("letta", 2)) || + !strings.Contains(acts[2], mirror(1)) { + t.Fatalf("the hand acts recorded are %v; the store was asked %v", acts, door.asked) + } + if len(gone) != 1 || !strings.Contains(gone[0], `"fingerprint":"`+fp(2)+`"`) { + t.Fatalf("artifacts-gone said %v", gone) + } + + // S25 through the real findings bucket: letta's wanted image missing from the store is a finding there. + view, _ := mem.Listed(ctx) + repo, digest, _ := splitReference(imageRef("letta", 1)) + delete(view.L.Repos[repo].Revisions, digest) + if err := h.Observe(ctx); err != nil { + t.Fatal(err) + } + if keys, _ := bus.Keys("artifact-store-tools." + FindingsState); len(keys) != 1 || keys[0] != "module.letta.ace.image-missing" { + t.Fatalf("the findings are %v", keys) + } +} + +// findingsAt reads and writes the holder's own findings in its bucket, artifact-store-tools_findings. +type findingsAt struct{ realBus } + +func (f findingsAt) own(state string) string { + if state == FindingsState { + return Module + "." + state + } + return state +} +func (f findingsAt) Keys(state string) ([]string, error) { return f.realBus.Keys(f.own(state)) } +func (f findingsAt) Get(state, key string) ([]byte, bool, error) { + return f.realBus.Get(f.own(state), key) +} +func (f findingsAt) Put(state, key string, value any) error { + return f.realBus.Put(f.own(state), key, value) +} +func (f findingsAt) Delete(state, key string) error { return f.realBus.Delete(f.own(state), key) } diff --git a/modules/artifact-store-tools/cmd/artifact-store-tools/holder_test.go b/modules/artifact-store-tools/cmd/artifact-store-tools/holder_test.go new file mode 100644 index 00000000..f1d6b7a1 --- /dev/null +++ b/modules/artifact-store-tools/cmd/artifact-store-tools/holder_test.go @@ -0,0 +1,351 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + "sort" + "strings" + "sync" + "testing" + "time" + + stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/buscompose" + "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/handacts" + "git.novox.be/novox/mesh-sdk/go/health" + + "artifactstoretools/internal/registry" +) + +// memBus is the holder's bus in memory: states by name, each key's history, and what was published. +type memBus struct { + mu sync.Mutex + states map[string]map[string][]byte + history map[string][]stdio.StateRevision + log []stdio.LogEntry + published []string +} + +func (b *memBus) LogSince(read string, at time.Time) ([]stdio.LogEntry, error) { + var out []stdio.LogEntry + for _, e := range b.log { + if read == delivery.BuildLogRead && !e.At.Before(at) { + out = append(out, e) + } + } + return out, nil +} + +func (b *memBus) Keys(state string) ([]string, error) { + b.mu.Lock() + defer b.mu.Unlock() + var out []string + for k := range b.states[state] { + out = append(out, k) + } + sort.Strings(out) + return out, nil +} + +func (b *memBus) Get(state, key string) ([]byte, bool, error) { + b.mu.Lock() + defer b.mu.Unlock() + v, ok := b.states[state][key] + return v, ok, nil +} + +func (b *memBus) History(state, key string) ([]stdio.StateRevision, error) { + return b.history[state+" "+key], nil +} + +func (b *memBus) Put(state, key string, value any) error { + raw, _ := json.Marshal(value) + b.mu.Lock() + defer b.mu.Unlock() + if b.states[state] == nil { + b.states[state] = map[string][]byte{} + } + b.states[state][key] = raw + return nil +} + +func (b *memBus) Delete(state, key string) error { + b.mu.Lock() + defer b.mu.Unlock() + delete(b.states[state], key) + return nil +} + +func (b *memBus) Publish(subject string, body any) error { + raw, _ := json.Marshal(body) + b.mu.Lock() + defer b.mu.Unlock() + b.published = append(b.published, subject+" "+string(raw)) + return nil +} + +// memDoor is the store's door in memory: what it holds, which references are indexes, and what was asked. +type memDoor struct { + holds map[string]bool + indexes map[string]bool + asked []string +} + +func (d *memDoor) Address() string { return "10.0.0.5:5000" } +func (d *memDoor) Hold(_ context.Context, r string) (bool, error) { + d.asked = append(d.asked, "hold "+r) + if !d.holds[r] { + return false, registry.Gone + } + return false, nil +} +func (d *memDoor) Holds(_ context.Context, r string) error { + if !d.holds[r] { + return registry.Gone + } + return nil +} +func (d *memDoor) IsIndex(_ context.Context, r string) (bool, error) { return d.indexes[r], nil } +func (d *memDoor) LetGo(_ context.Context, r string) error { + d.asked = append(d.asked, "let go "+r) + delete(d.holds, r) + return nil +} +func (d *memDoor) SpareKept(context.Context, []string, []string) error { + d.asked = append(d.asked, "spare") + return nil +} + +// mirror is a copy of an upstream base in the store, the n-th. +func mirror(n int) string { + return "artifact-store://upstream/docker.io/library/alpine@sha256:" + strings.Repeat(fmt.Sprintf("%x", n%16), 64) +} + +// pinnedImage is the image letta's manifest names by itself, at the store's address. +const pinnedImage = "10.0.0.5:5000/letta/pinned@sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc" + +func digestOf(ref string) string { _, d, _ := splitReference(ref); return d } + +func fp(n int) string { return "src2:" + strings.Repeat(fmt.Sprintf("%x", n%16), 64) } + +func imageRef(module string, n int) string { + return "artifact-store://" + module + "/image@sha256:" + strings.Repeat(fmt.Sprintf("%x", n%16), 64) +} + +func archiveRef(module string, n int) string { + return "artifact-store://" + module + "/files/blobs/sha256:" + strings.Repeat(fmt.Sprintf("%x", (n+8)%16), 64) +} + +// A mesh with seven done builds of letta, the n-th ending n hours after a day began, each with an image and an archive; +// a want names the first on ace; the store lists them all and one image no build names; the lease as given. +func aMesh(t *testing.T, lease *buscompose.LeaseHolder) (*Holder, *memBus, *memDoor, *[]string) { + t.Helper() + day := time.Date(2026, 10, 1, 0, 0, 0, 0, time.UTC) + b := &memBus{states: map[string]map[string][]byte{}, history: map[string][]stdio.StateRevision{}} + door := &memDoor{holds: map[string]bool{}, indexes: map[string]bool{}} + listing := &Listing{Blobs: map[string]int64{}, Repos: map[string]*Repo{}} + for n := 1; n <= 7; n++ { + key, _ := delivery.BuilderBuildKey("letta", fp(n)) + raw, err := delivery.EncodeBuilderBuild(delivery.BuilderBuild{State: delivery.BuildDone, Manifest: strings.Repeat("d", 64), Artifacts: []delivery.BuiltArtifact{ + {Name: "image", Kind: "image", Digest: digestOf(imageRef("letta", n)), Reference: imageRef("letta", n)}, + {Name: "files", Kind: "archive", Digest: digestOf(archiveRef("letta", n)), Reference: archiveRef("letta", n)}, + {Name: "pkg", Kind: "package", Digest: "sha256:" + strings.Repeat("f", 64), Reference: "letta@1.0." + fmt.Sprint(n)}}}) + if err != nil { + t.Fatal(err) + } + _ = b.Put(BuildsRead, key, json.RawMessage(raw)) + b.history[BuildsRead+" "+key] = []stdio.StateRevision{{Key: key, Value: raw, At: day.Add(time.Duration(n) * time.Hour)}} + for _, r := range []string{imageRef("letta", n), archiveRef("letta", n)} { + door.holds[r] = true + repo, digest, kind := splitReference(r) + if kind == "archive" { + listing.Blobs[digest] = 10 + } else { + listing.repo(repo).Revisions[digest] = true + } + } + } + listing.repo("letta/image").Revisions["sha256:"+strings.Repeat("e", 64)] = true // no build names it + listing.repo("letta/pinned").Revisions["sha256:"+strings.Repeat("c", 64)] = true + // Three copies of a base: letta's 2nd stood on the first, its 7th on the second, and no build on the third. + for n := 1; n <= 3; n++ { + repo, digest, _ := splitReference(mirror(n)) + listing.repo(repo).Revisions[digest] = true + door.holds[mirror(n)] = true + } + for n := 1; n <= 7; n++ { + e := delivery.BuildLogEntry{Event: delivery.BuildEndedEvent, Module: "letta", FP: fp(n), Repository: "novox/x", Commit: "c", + Node: "ace", At: day.Add(time.Duration(n) * time.Hour), Outcome: "done"} + switch n { + case 2: + e.Against = []string{mirror(1)} + case 7: + e.Against = []string{mirror(2)} + } + raw, _ := delivery.EncodeBuildLogEntry(e) + b.log = append(b.log, stdio.LogEntry{Sequence: uint64(n), Key: "letta", Value: raw, At: e.At}) + } + manifest, _ := delivery.EncodeManifest(delivery.Manifest{Module: "letta", + JSON: `{"module":"letta","resources":[{"id":"app","type":"container","image":"` + pinnedImage + `"},` + + `{"id":"built","type":"container","image":"${artifact:image}"}]}`}) + mk, _ := delivery.ManifestsKey(strings.Repeat("d", 64)) + _ = b.Put(ManifestsRead, mk, json.RawMessage(manifest)) + want, _ := json.Marshal(delivery.Want{V: delivery.Version, FP: fp(1), Self: true}) + _ = b.Put(WantsRead, "want.ace.letta", json.RawMessage(want)) + if lease != nil { + _ = b.Put(LeaseRead, buscompose.ControllerLeaseKey, lease) + } + var acts []string + h := &Holder{Bus: b, Door: func() (Door, error) { return door, nil }, + Listed: func(context.Context) (*View, error) { return &View{L: listing}, nil }, + Now: func() time.Time { return day.Add(48 * time.Hour) }, + Acts: handacts.Recorder{Ask: func(key string, body any) (json.RawMessage, error) { + raw, _ := json.Marshal(body) + acts = append(acts, key+" "+string(raw)) + return json.RawMessage(`{}`), nil + }}} + h.Acts.LeaseAway = h.LeaseAway + return h, b, door, &acts +} + +// ADR 0571 clause 2: the keep set from the store's listing, the wants and the five newest builds; nothing let go of while +// the lease names a controller; once it names another holder, a build's end lets go of the eligible within the bound, +// leaves an index for a confirmed collect, records each deletion and says artifacts-gone. +func TestTheSeatKeepsTheFiveNewestAndTheWantedAndSweepsOnlyOnceTheLeaseIsAway(t *testing.T) { + controller := &buscompose.LeaseHolder{Instance: "c1", Host: "novox", Epoch: 3} + handed := &buscompose.LeaseHolder{HandedTo: "mesh-grants", HandedBy: "person.jochen", Why: "the lease write"} + + h, _, _, _ := aMesh(t, controller) + r, err := h.Artifacts("") + if err != nil { + t.Fatal(err) + } + states := map[string]string{} + for _, x := range r.References { + states[x.Reference] = x.State + } + // letta's 3rd to 7th are the five newest; its 1st a want names on ace, however old; its 2nd is eligible; the package + // and the manifest no build names are no reference of the seat's. + for n, want := range map[int]string{1: "kept", 2: "eligible", 3: "kept", 7: "kept"} { + if states[imageRef("letta", n)] != want || states[archiveRef("letta", n)] != want { + t.Errorf("letta's build %d is %s and %s, not %s", n, states[imageRef("letta", n)], states[archiveRef("letta", n)], want) + } + } + // And the copies: the one letta's 2nd stood on is eligible with it, the 7th's kept; the third no build names. + if states[mirror(1)] != "eligible" || states[mirror(2)] != "kept" || states[mirror(3)] != "" { + t.Errorf("the copies are %s, %s and %q", states[mirror(1)], states[mirror(2)], states[mirror(3)]) + } + if m, err := h.MirrorsNow(); err != nil || len(m.Copies) != 3 || m.Copies[2].State != "unrecorded" || m.Held != "" { + t.Errorf("mirrors answered %+v: %v", m, err) + } + if len(r.References) != 16 || r.Counts["kept"] != 13 || r.Counts["eligible"] != 3 || r.Store != "10.0.0.5:5000" { + t.Errorf("the references are %d, counted %v, at %q", len(r.References), r.Counts, r.Store) + } + + for _, c := range []struct { + name string + lease *buscompose.LeaseHolder + index bool + most int + letGo []string + left int + skipped int + unlog int + }{ + {name: "the lease names a controller", lease: controller}, + {name: "no lease key", lease: nil}, + {name: "handed away", lease: handed, most: MostPerSweep, letGo: []string{archiveRef("letta", 2), imageRef("letta", 2), mirror(1)}}, + {name: "an index waits", lease: handed, index: true, most: MostPerSweep, letGo: []string{archiveRef("letta", 2), mirror(1)}, skipped: 1}, + {name: "bounded", lease: handed, most: 1, letGo: []string{archiveRef("letta", 2)}, left: 2}, + // A kept build the log no longer holds: what it stood on is unknown, so no copy goes. + {name: "a kept build's bases unknown", lease: handed, most: MostPerSweep, letGo: []string{archiveRef("letta", 2), imageRef("letta", 2)}, unlog: 3}, + } { + t.Run(c.name, func(t *testing.T) { + h, b, door, acts := aMesh(t, c.lease) + door.indexes[imageRef("letta", 2)] = c.index + if c.unlog > 0 { + b.log = append(b.log[:c.unlog-1:c.unlog-1], b.log[c.unlog:]...) + } + if c.most == 0 { + h.AfterBuild("letta.x") + if _, err := h.Collect("tidy", true, 0); err == nil { + t.Error("a confirmed collect was let through while the controller sweeps") + } + if len(door.asked) != 0 || len(*acts) != 0 || len(b.published) != 0 { + t.Fatalf("before the lease write the store was asked %v, acts %v, said %v", door.asked, *acts, b.published) + } + return + } + a, err := h.Sweep(context.Background(), Sweeping{Most: c.most, By: "after the build letta.x"}) + if err != nil { + t.Fatal(err) + } + if fmt.Sprint(a.LetGo) != fmt.Sprint(c.letGo) || a.Left != c.left || a.Skipped != c.skipped || a.Held != 6 || a.Missing != 0 { + t.Fatalf("let go of %v (left %d, skipped %d, held %d, missing %d), want %v", a.LetGo, a.Left, a.Skipped, a.Held, a.Missing, c.letGo) + } + // Every kept archive held before the first deletion. + first := -1 + for i, x := range door.asked { + if strings.HasPrefix(x, "let go ") && first < 0 { + first = i + } + } + if first != 6 { + t.Errorf("the store was asked %v: the six kept archives are held first", door.asked) + } + if len(*acts) != len(c.letGo) || !strings.HasPrefix((*acts)[0], "seat:mesh-hand-acts.hand-act ") || + !strings.Contains((*acts)[0], c.letGo[0]) { + t.Errorf("the hand acts recorded are %v", *acts) + } + if len(b.published) != 1 || !strings.HasPrefix(b.published[0], GoneSubject+" ") || + !strings.Contains(b.published[0], `"fingerprint":"`+fp(2)+`"`) { + t.Errorf("said %v", b.published) + } + }) + } +} + +// S25 (image-missing): a want whose build names an image the store does not hold is a finding in the seat's own state, +// written once and deleted once the store holds it again; images names what a machine's wants and haves name. +func TestAWantedImageTheStoreLacksIsAFindingAndImagesNamesWantsAndHaves(t *testing.T) { + h, b, _, _ := aMesh(t, nil) + view, _ := h.Listed(context.Background()) + repo, digest, _ := splitReference(imageRef("letta", 1)) + delete(view.L.Repos[repo].Revisions, digest) + h.Listed = func(context.Context) (*View, error) { return view, nil } + if err := h.Observe(context.Background()); err != nil { + t.Fatal(err) + } + key := health.FindingKey(health.ScopeModule, "letta.ace", ImageMissing) + raw, found, _ := b.Get(FindingsState, key) + f, err := health.DecodeFinding(raw) + if !found || err != nil || f.Subject.Machine != "ace" || f.Source != "S25" || !strings.Contains(f.Evidence[0].Said, imageRef("letta", 1)) { + t.Fatalf("the finding is %+v (%v, %v)", f, found, err) + } + view.L.Repos[repo].Revisions[digest] = true + if err := h.Observe(context.Background()); err != nil { + t.Fatal(err) + } + if _, found, _ := b.Get(FindingsState, key); found { + t.Fatal("the finding stayed once the store held the image again") + } + // An image the manifest pins by itself, under the store's address, is judged as the controller judged it. + delete(view.L.Repos["letta/pinned"].Revisions, "sha256:"+strings.Repeat("c", 64)) + if err := h.Observe(context.Background()); err != nil { + t.Fatal(err) + } + if raw, found, _ := b.Get(FindingsState, key); !found || !strings.Contains(string(raw), "letta/pinned@sha256:") { + t.Fatalf("the pinnedImage image missing is %s", raw) + } + + have, _ := json.Marshal(delivery.Have{V: delivery.Version, FP: fp(7), Phase: "running"}) + _ = b.Put(HavesRead, "ace.letta", json.RawMessage(have)) + images, err := h.ImagesOf("ace", "10.0.0.5:5000") + if err != nil || len(images.Images) != 3 || !images.SentKnown || images.Images[0].Image != "10.0.0.5:5000/letta/image@sha256:"+strings.Repeat("1", 64) || + fmt.Sprint(images.Images[0].In) != "[declared]" || fmt.Sprint(images.Images[1].In) != "[sent]" || + images.Images[2].Image != pinnedImage || fmt.Sprint(images.Images[2].In) != "[declared sent]" { + t.Fatalf("the images of ace are %+v: %v", images, err) + } +} diff --git a/modules/artifact-store-tools/cmd/artifact-store-tools/main.go b/modules/artifact-store-tools/cmd/artifact-store-tools/main.go index f833fecf..ea1f28e0 100644 --- a/modules/artifact-store-tools/cmd/artifact-store-tools/main.go +++ b/modules/artifact-store-tools/cmd/artifact-store-tools/main.go @@ -1,30 +1,38 @@ -// The artifact-store-tools module's Go bundle (novox/hq ADR 0251 §1–3, to-be 51): a process the node's -// runtime launches and speaks MCP over stdio to. It says what the artifact store holds — its -// repositories, its size, and what the controller's records say of each manifest — read from the -// store's own files through the store's own container, and writing nothing. A collection is asked of -// the controller, which decides and records what it lets go of (ADR 0189). +// The artifact-store-tools module's Go bundle (novox/hq ADR 0251 §1–3, ADR 0571 clause 2, to-be 51): a process the +// node's runtime launches and speaks MCP over stdio to. It says what the artifact store holds — its repositories, its +// size, and what the mesh keeps of each manifest — read from the store's own files through the store's own container, +// and it holds the seat mesh-artifact-sweep: the keep set, the sweep at each build's end once the controller's lease names +// no controller, and S25's image-missing. // -// **A module beside the store, not the store's own.** A module that provides the artifact store cannot -// also build an artifact: building publishes to the store, so the store would be needed to create -// itself, and the controller refuses that manifest. So these tools are a second module, assigned to the -// machine that holds the store. stdout is the protocol; what this bundle says, it says on stderr. +// **A module beside the store, not the store's own.** A module that provides the artifact store cannot also build an +// artifact: building publishes to the store, so the store would be needed to create itself. So these tools are a +// second module, assigned to the machine that holds the store. stdout is the protocol; what this bundle says, it says +// on stderr. package main import ( "fmt" "os" + "time" stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/handacts" ) +func logf(format string, a ...any) { fmt.Fprintf(os.Stderr, format+"\n", a...) } + func main() { s := &Store{ Container: os.Getenv("MESH_ARTIFACT_STORE_CONTAINER"), Run: ExecRunner, UID: os.Getuid(), } + h := &Holder{Bus: runtimeBus{}, Door: DoorAt(os.Getenv("MESH_ARTIFACT_STORE_ADDRESS_FILE")), Listed: s.View, + Now: time.Now, Logf: logf} + h.Acts = handacts.Recorder{Ask: stdio.Ask, LeaseAway: h.LeaseAway} + go follow(h) // An empty name serves as the module the runtime names (MESH_SERVED_MODULE): artifact-store-tools. - if err := stdio.Serve("", Tools(s, Controller{Ask: stdio.Ask}, &Catalogue{Ask: stdio.Ask})); err != nil { + if err := stdio.Serve("", append(Tools(s, h, &Catalogue{Ask: stdio.Ask}), SeatTools(h, s)...)); err != nil { fmt.Fprintln(os.Stderr, err) os.Exit(1) } diff --git a/modules/artifact-store-tools/cmd/artifact-store-tools/seat.go b/modules/artifact-store-tools/cmd/artifact-store-tools/seat.go new file mode 100644 index 00000000..37491142 --- /dev/null +++ b/modules/artifact-store-tools/cmd/artifact-store-tools/seat.go @@ -0,0 +1,122 @@ +package main + +// The seat mesh-artifact-sweep's verbs (novox/hq ADR 0571 clause 2; catalogue.ArtifactSweepSeat), each a tool named +// `artifact-store.` as the runtime serves a seat's verb, answering as the controller's verb answered: {output, +// ok, answer}. + +import ( + "fmt" + "strings" + + stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/catalogue" +) + +// verbAnswer is the controller's verbs' answer shape, which the builder and docker read (answerOf). +type verbAnswer struct { + Output string `json:"output"` + OK bool `json:"ok"` + Answer any `json:"answer"` +} + +// SeatTools are the seat's verbs over h; s reads the store's listing for mirrors. +func SeatTools(h *Holder, s *Store) []stdio.Tool { + seat := catalogue.ArtifactSweepSeat + "." + return []stdio.Tool{ + {Name: seat + "artifacts", Description: "Every artifact the mesh made that the store holds: kept and why, eligible, " + + "or with collected those it no longer holds.", + Run: func(args map[string]any) (any, error) { + r, err := h.Artifacts(text(args, "repository")) + if err != nil { + return nil, err + } + if text(args, "collected") != "true" { + refs := r.References[:0:0] + for _, x := range r.References { + if x.State != "collected" { + refs = append(refs, x) + } + } + r.References = refs + } + return verbAnswer{Output: fmt.Sprintf("%d kept, %d eligible, %d collected\n", r.Counts["kept"], + r.Counts["eligible"], r.Counts["collected"]), OK: true, Answer: r}, nil + }}, + {Name: seat + "held", Description: "Whether the store holds each reference given: held, gone or unknown with why.", + Run: func(args map[string]any) (any, error) { + a, err := h.Held(listOf(args["references"])) + if err != nil { + return nil, err + } + return verbAnswer{Output: fmt.Sprintf("the artifact store at %s holds %d of the %d reference(s) asked\n", + a.Store, len(a.Held), len(a.Held)+len(a.Gone)+len(a.Unknown)), OK: true, Answer: a}, nil + }}, + {Name: seat + "collect", Description: "The sweep now: a dry run without confirm; with confirm and why, every kept " + + "archive held, then each eligible artifact let go of, each recorded in the hand-act log.", + Run: func(args map[string]any) (any, error) { + most, err := count(args, "most", MostPerSweep, 1, MostAtAll) + if err != nil { + return nil, err + } + a, err := h.Collect(text(args, "why"), text(args, "confirm") == "true", most) + if err != nil { + return nil, err + } + return verbAnswer{Output: fmt.Sprintf("let go of %d, would let go of %d, %d left\n", len(a.LetGo), + len(a.WouldLetGo), a.Left), OK: true, Answer: a}, nil + }}, + {Name: seat + "mirrors", Description: "The bases the mesh copied into its store, each kept (a kept build stood on " + + "it), eligible (only builds no longer kept stood on it) or unrecorded (no build the builder's log holds stood " + + "on it: never removed).", + Run: func(args map[string]any) (any, error) { + if text(args, "record") != "" { + return nil, fmt.Errorf("mirrors records nothing by hand on this seat: a copy is recorded by the builds the " + + "builder's log says stood on it (novox/hq ADR 0571 clause 1)") + } + m, err := h.MirrorsNow() + if err != nil { + return nil, err + } + return verbAnswer{Output: fmt.Sprintf("%d copies\n", len(m.Copies)), OK: true, Answer: m}, nil + }}, + {Name: seat + "images", Description: "Every container image one machine's wants and haves name.", + Run: func(args map[string]any) (any, error) { + node := text(args, "node") + if node == "" { + return nil, fmt.Errorf("images names the machine") + } + address := "" + if d, err := h.Door(); err == nil { + address = d.Address() + } + a, err := h.ImagesOf(node, address) + if err != nil { + return nil, err + } + return verbAnswer{Output: fmt.Sprintf("%s: %d images\n", node, len(a.Images)), OK: true, Answer: a}, nil + }}, + } +} + +// listOf is a list argument: a list, or one text separated by commas. +func listOf(v any) []string { + var out []string + add := func(s string) { + for _, p := range strings.Split(s, ",") { + if p = strings.TrimSpace(p); p != "" && !contains(out, p) { + out = append(out, p) + } + } + } + switch x := v.(type) { + case string: + add(x) + case []any: + for _, e := range x { + if s, ok := e.(string); ok { + add(s) + } + } + } + return out +} diff --git a/modules/artifact-store-tools/cmd/artifact-store-tools/store_test.go b/modules/artifact-store-tools/cmd/artifact-store-tools/store_test.go index 72b46379..e0617a03 100644 --- a/modules/artifact-store-tools/cmd/artifact-store-tools/store_test.go +++ b/modules/artifact-store-tools/cmd/artifact-store-tools/store_test.go @@ -1,6 +1,8 @@ package main import ( + "git.novox.be/novox/mesh-sdk/go/catalogue" + "context" "encoding/json" "fmt" @@ -411,6 +413,11 @@ func TestTheToolsServedAreTheToolsTheManifestNames(t *testing.T) { var m struct { Tools []string `json:"tools"` Invokes []string `json:"invokes"` + Reads []string `json:"reads"` + Claims []struct { + Name string `json:"name"` + Serves []string `json:"serves"` + } `json:"claims"` } if err := json.Unmarshal(raw, &m); err != nil { t.Fatal(err) @@ -428,10 +435,30 @@ func TestTheToolsServedAreTheToolsTheManifestNames(t *testing.T) { if !reflect.DeepEqual(served, listed) { t.Fatalf("served %v, manifest %v", served, listed) } - want := []string{"seat:mesh-controller.artifacts", "seat:mesh-controller.collect", CatalogueModules, CatalogueModule} + // ADR 0571 clause 2: it holds the seat mesh-artifact-sweep and serves its verbs, and calls no verb of the controller's; + // the lease is the one controller state it reads (clause 10's exemption). + want := []string{"seat:mesh-hand-acts.hand-act", CatalogueModules, CatalogueModule} if !reflect.DeepEqual(m.Invokes, want) { t.Errorf("invokes %v", m.Invokes) } + var verbs []string + for _, tl := range SeatTools(&Holder{}, &Store{}) { + verbs = append(verbs, strings.TrimPrefix(tl.Name, catalogue.ArtifactSweepSeat+".")) + } + seat, _ := catalogue.SeatNamed(catalogue.ArtifactSweepSeat) + var contract []string + for _, v := range seat.Serves { + contract = append(contract, v.Name) + } + if len(m.Claims) != 1 || m.Claims[0].Name != catalogue.ArtifactSweepSeat || !reflect.DeepEqual(m.Claims[0].Serves, verbs) || + !reflect.DeepEqual(verbs, contract) { + t.Errorf("claims %+v, serves %v, the seat's verbs %v", m.Claims, verbs, contract) + } + for _, r := range m.Reads { + if strings.HasPrefix(r, "mesh-controller.") && r != LeaseRead { + t.Errorf("reads the controller's %s", r) + } + } } func TestASocketRefusalIsAskedAgainThroughSudo(t *testing.T) { diff --git a/modules/artifact-store-tools/cmd/artifact-store-tools/tools.go b/modules/artifact-store-tools/cmd/artifact-store-tools/tools.go index 65d0350e..a6656750 100644 --- a/modules/artifact-store-tools/cmd/artifact-store-tools/tools.go +++ b/modules/artifact-store-tools/cmd/artifact-store-tools/tools.go @@ -11,9 +11,16 @@ import ( stdio "git.novox.be/novox/mesh-sdk/go" ) +// Recorded is what says which references the mesh keeps and lets go of them: the seat's holder (Holder, novox/hq ADR +// 0571 clause 2), or before it the controller (Controller), each in the controller's answer's shape. +type Recorded interface { + Artifacts(repository string) (*Records, error) + Collect(why string, confirm bool, most int) (*CollectAnswer, error) +} + // Tools are the artifact store's tools (novox/hq ADR 0251 §1). The first three only read; artifact_store_collect -// only reads too, unless it is a real run, and then it asks the controller, which decides and deletes. -func Tools(s *Store, c Controller, cat *Catalogue) []stdio.Tool { +// only reads too, unless it is a real run, and then it asks the seat's sweep, which decides and deletes. +func Tools(s *Store, c Recorded, cat *Catalogue) []stdio.Tool { ctx := context.Background() repoArg := map[string]any{"type": "string", "description": "one repository, as the store names it (/)"} return []stdio.Tool{ diff --git a/modules/artifact-store-tools/files/store-address b/modules/artifact-store-tools/files/store-address new file mode 100644 index 00000000..1bcadab2 --- /dev/null +++ b/modules/artifact-store-tools/files/store-address @@ -0,0 +1 @@ +${seat:mesh-artifact-store:reach} diff --git a/modules/artifact-store-tools/go.mod b/modules/artifact-store-tools/go.mod index 74ccd152..b2a35897 100644 --- a/modules/artifact-store-tools/go.mod +++ b/modules/artifact-store-tools/go.mod @@ -1,5 +1,22 @@ module artifactstoretools -go 1.22 +go 1.26.0 -require git.novox.be/novox/mesh-sdk/go v0.1.7 +require ( + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 + github.com/nats-io/nats.go v1.54.0 +) + +require ( + github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op // indirect + github.com/google/go-tpm v0.9.8 // indirect + github.com/klauspost/compress v1.20.0 // indirect + github.com/minio/highwayhash v1.0.4 // indirect + github.com/nats-io/jwt/v2 v2.8.1 // indirect + github.com/nats-io/nats-server/v2 v2.11.17 // indirect + github.com/nats-io/nkeys v0.4.16 // indirect + github.com/nats-io/nuid v1.0.1 // indirect + golang.org/x/crypto v0.57.0 // indirect + golang.org/x/sys v0.48.0 // indirect + golang.org/x/time v0.15.0 // indirect +) diff --git a/modules/artifact-store-tools/go.sum b/modules/artifact-store-tools/go.sum index b474419e..a2b8e018 100644 --- a/modules/artifact-store-tools/go.sum +++ b/modules/artifact-store-tools/go.sum @@ -1,2 +1,27 @@ -git.novox.be/novox/mesh-sdk/go v0.1.7 h1:C0sTQmtTiyYH7bnqZb7PusXnqA37gKuT7Nqjn9gG47w= -git.novox.be/novox/mesh-sdk/go v0.1.7/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 h1:lKU3HjNXGiVbLW2hUgFjU/TMTUYMlNnnmZxG5T7biQA= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= +github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= +github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= +github.com/google/go-tpm v0.9.8/go.mod h1:h9jEsEECg7gtLis0upRBQU+GhYVH6jMjrFxI8u6bVUY= +github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA= +github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI= +github.com/minio/highwayhash v1.0.4 h1:asJizugGgchQod2ja9NJlGOWq4s7KsAWr5XUc9Clgl4= +github.com/minio/highwayhash v1.0.4/go.mod h1:GGYsuwP/fPD6Y9hMiXuapVvlIUEhFhMTh0rxU3ik1LQ= +github.com/nats-io/jwt/v2 v2.8.1 h1:V0xpGuD/N8Mi+fQNDynXohVvp7ZztevW5io8CUWlPmU= +github.com/nats-io/jwt/v2 v2.8.1/go.mod h1:nWnOEEiVMiKHQpnAy4eXlizVEtSfzacZ1Q43LIRavZg= +github.com/nats-io/nats-server/v2 v2.11.17 h1:GKEghcFK6A+aFx11Yf1LjgLC3txAwvyhnYzhBIQZA8I= +github.com/nats-io/nats-server/v2 v2.11.17/go.mod h1:B1sFVz4StNosQ903ak4N1G01Fl/9f8e06mXpFIE2K24= +github.com/nats-io/nats.go v1.54.0 h1:vsXoOxjHp/GmPUN+EcI7uOf/uB+iAP+kEsAFNQN0yzA= +github.com/nats-io/nats.go v1.54.0/go.mod h1:y+DZoD1oBOYfZTU681eTUiUjI0vbqYGixNVFHcjHJ0k= +github.com/nats-io/nkeys v0.4.16 h1:rd5oAuLOb8mnAycB0xleuEBNS1pVVnN0fv/FF34Eypg= +github.com/nats-io/nkeys v0.4.16/go.mod h1:llLgWoI0o4z/Q57q2R1kHfmocyhGV6VG/U18Glg1Afs= +github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= +github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= +golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M= +golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA= +golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo= +golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= +golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= +golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= diff --git a/modules/artifact-store-tools/internal/registry/blob.go b/modules/artifact-store-tools/internal/registry/blob.go new file mode 100644 index 00000000..bedbfa38 --- /dev/null +++ b/modules/artifact-store-tools/internal/registry/blob.go @@ -0,0 +1,57 @@ +package registry + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "fmt" + "io" + "net/http" +) + +// blobBound is the most GetBlob reads: a templates archive is a few kilobytes; anything near this is no archive a +// take-in reads text from. +const blobBound = 64 << 20 + +// GetBlob is the archive a recorded reference names (`artifact-store:///blobs/sha256:`), read whole +// and checked against the digest it names: bytes that hash to another digest are refused, never read (R4e: the +// controller reads a templates archive by the reference of a builder's done key). Gone when the store does not hold +// it; ErrNotOurs for a reference outside the store or one that is not an archive. +func (s Store) GetBlob(ctx context.Context, reference string) ([]byte, error) { + repository, digest, archive, err := s.archive(reference) + if err != nil { + return nil, err + } + if !archive { + return nil, fmt.Errorf("%w: %s is not an archive", ErrNotOurs, reference) + } + request, err := http.NewRequestWithContext(ctx, http.MethodGet, s.url(repository, "blobs", digest), nil) + if err != nil { + return nil, err + } + response, err := s.client().Do(request) + if err != nil { + return nil, fmt.Errorf("cannot reach the artifact store at %s: %w", s.Address, err) + } + defer response.Body.Close() + switch response.StatusCode { + case http.StatusOK: + case http.StatusNotFound: + return nil, Gone + default: + return nil, fmt.Errorf("the artifact store answered %s for %s/blobs/%s", response.Status, repository, digest) + } + body, err := io.ReadAll(io.LimitReader(response.Body, blobBound+1)) + if err != nil { + return nil, fmt.Errorf("reading %s/blobs/%s: %w", repository, digest, err) + } + if len(body) > blobBound { + return nil, fmt.Errorf("%s/blobs/%s is larger than %d bytes, which no archive read here is", repository, digest, blobBound) + } + sum := sha256.Sum256(body) + if got := "sha256:" + hex.EncodeToString(sum[:]); got != digest { + return nil, fmt.Errorf("the artifact store answered %s/blobs/%s with bytes of the digest %s: refused", repository, + digest, got) + } + return body, nil +} diff --git a/modules/artifact-store-tools/internal/registry/blob_test.go b/modules/artifact-store-tools/internal/registry/blob_test.go new file mode 100644 index 00000000..13487cc7 --- /dev/null +++ b/modules/artifact-store-tools/internal/registry/blob_test.go @@ -0,0 +1,44 @@ +package registry + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "net/http" + "net/http/httptest" + "strings" + "testing" +) + +// A take-in reads a templates archive by its reference, and only bytes of the digest the reference names. +func TestGetBlobReadsOnlyTheBytesOfItsDigest(t *testing.T) { + good := []byte("the archive") + sum := sha256.Sum256(good) + digest := "sha256:" + hex.EncodeToString(sum[:]) + liar := "sha256:" + strings.Repeat("0", 64) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case strings.HasSuffix(r.URL.Path, "/x/files/blobs/"+digest), strings.HasSuffix(r.URL.Path, "/x/files/blobs/"+liar): + _, _ = w.Write(good) + default: + w.WriteHeader(http.StatusNotFound) + } + })) + defer server.Close() + s := Store{Address: strings.TrimPrefix(server.URL, "http://")} + ctx := context.Background() + body, err := s.GetBlob(ctx, "artifact-store://x/files/blobs/"+digest) + if err != nil || string(body) != string(good) { + t.Fatalf("the archive: %q %v", body, err) + } + if _, err := s.GetBlob(ctx, "artifact-store://x/files/blobs/"+liar); err == nil || !strings.Contains(err.Error(), "refused") { + t.Fatalf("bytes of another digest are read: %v", err) + } + if _, err := s.GetBlob(ctx, "artifact-store://x/other/blobs/"+digest); !errors.Is(err, Gone) { + t.Fatalf("a blob the store does not hold: %v", err) + } + if _, err := s.GetBlob(ctx, "registry.example/x@"+digest); !errors.Is(err, ErrNotOurs) { + t.Fatalf("a reference outside the store: %v", err) + } +} diff --git a/modules/artifact-store-tools/internal/registry/hold.go b/modules/artifact-store-tools/internal/registry/hold.go new file mode 100644 index 00000000..391e9c89 --- /dev/null +++ b/modules/artifact-store-tools/internal/registry/hold.go @@ -0,0 +1,345 @@ +package registry + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "net/http" + "strconv" + "strings" + "time" + + "git.novox.be/novox/mesh-sdk/go/catalogue" +) + +// Every archive the store keeps is held by a manifest (novox/hq issue 253, ADR 0189). +// +// **The store's collector marks only from manifests.** The mesh's store is a stock registry, and +// its nightly `registry garbage-collect` walks every manifest in every repository, marks the blobs +// those manifests name, and deletes every blob it did not mark. An image is a manifest, so what +// the mesh keeps of an image survives. An archive was not: the builder put it in the store as a +// bare blob — upload, then `PUT ?digest=` — and nothing in the store names it. To the collector a +// bare blob is unreferenced, so the first real collection would have deleted every archive the +// mesh holds, kept or not, and every machine pinning a bundle would have found it gone. The +// collector runs `--dry-run` until this is true. +// +// **So each archive gets a holder**: the smallest OCI image manifest that names it — the empty +// config, one layer, nothing else — put in the archive's own repository, by digest, untagged. The +// collector marks it and so keeps the archive; the sweep lets go of an archive by deleting its +// holder first, which is what lets the bytes go at the next collection. +// +// **Nothing a machine reads changes.** The recorded reference stays +// `artifact-store:////blobs/sha256:…`, and machines fetch the blob exactly as +// before. The holder is the store's bookkeeping, not a second way to reach anything. +// +// **Deterministic, so it never needs recording.** The holder is composed from the archive's digest +// and size alone, in a fixed field order with no timestamps or annotations, so the sweep can +// compute which manifest holds any archive from the reference it already has plus one HEAD for the +// size. No schema change, no second record that could disagree with the store. + +const ( + // mediaManifest is the type a holder is put and asked for as. + mediaManifest = "application/vnd.oci.image.manifest.v1+json" + // mediaEmpty is the OCI empty descriptor's type: a config that says nothing, for a manifest + // whose only purpose is to name its layer. + mediaEmpty = "application/vnd.oci.empty.v1+json" + // emptyDigest is the digest of `{}`, the empty config's content, fixed by the OCI spec. + emptyDigest = "sha256:44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a" + // mediaArchive is the layer type an archive is held as. Every archive the builder publishes is + // `pack`'s gzipped tar, so this is the true type and not a placeholder — and it is a constant, + // not read from anywhere, because the holder must be recomputable from the reference alone. + mediaArchive = "application/vnd.oci.image.layer.v1.tar+gzip" +) + +// emptyConfig is the content emptyDigest names. +var emptyConfig = []byte("{}") + +// manifestAccept is what a manifest is asked for as. A registry answers a manifest HEAD only in a +// type the caller named, and answers 404 to a bare one for a manifest it holds perfectly well +// (measured 2026-09-28; internal/builder/registry.go says how that was found). +var manifestAccept = []string{ + mediaManifest, + "application/vnd.docker.distribution.manifest.v2+json", +} + +type descriptor struct { + MediaType string `json:"mediaType"` + Digest string `json:"digest"` + Size int64 `json:"size"` +} + +type holderManifest struct { + SchemaVersion int `json:"schemaVersion"` + MediaType string `json:"mediaType"` + Config descriptor `json:"config"` + Layers []descriptor `json:"layers"` +} + +// Holder is the manifest that holds an archive in the store, and its digest. +// +// A pure function of the archive's digest and size: the same two in give the same bytes out, +// always, because `encoding/json` writes a struct's fields in their declared order and there is +// nothing here that varies by when or where it was composed. +func Holder(digest string, size int64) (body []byte, holder string) { + body, err := json.Marshal(holderManifest{ + SchemaVersion: 2, + MediaType: mediaManifest, + Config: descriptor{MediaType: mediaEmpty, Digest: emptyDigest, Size: int64(len(emptyConfig))}, + Layers: []descriptor{{MediaType: mediaArchive, Digest: digest, Size: size}}, + }) + if err != nil { + // Marshalling a struct of strings and integers cannot fail. + panic(err) + } + sum := sha256.Sum256(body) + return body, "sha256:" + hex.EncodeToString(sum[:]) +} + +// Hold makes sure the store holds this archive by a manifest, and says whether it had to write one. +// +// Takes a reference as the mesh records it. An image is its own manifest and needs no holder, so +// it answers false and nothing is asked. Idempotent: a holder already there is left alone, which +// is what lets the sweep run it over every kept archive on every build and so backfill the bare +// blobs published before holders existed (novox/hq issue 253). +// +// Gone when the store does not hold the archive at all: there is nothing to hold, and that is a +// fact the caller reports rather than one this invents a remedy for. +func (s Store) Hold(ctx context.Context, reference string) (bool, error) { + repository, digest, archive, err := s.archive(reference) + if err != nil || !archive { + return false, err + } + size, err := s.blobSize(ctx, repository, digest) + if err != nil { + return false, err + } + return s.HoldBlob(ctx, repository, digest, size) +} + +// Held is whether the store holds this archive by its manifest. Asks and changes nothing — the +// question an operator needs answered with "none unheld" before the collector is let loose. +// +// An image answers true: it is its own manifest. An archive the store does not have answers Gone. +func (s Store) Held(ctx context.Context, reference string) (bool, error) { + repository, digest, archive, err := s.archive(reference) + if err != nil { + return false, err + } + if !archive { + return true, nil + } + size, err := s.blobSize(ctx, repository, digest) + if err != nil { + return false, err + } + _, holder := Holder(digest, size) + return s.has(ctx, s.url(repository, "manifests", holder), manifestAccept...) +} + +// HoldBlob puts the holder for a blob of this digest and size into its repository, unless it is +// there already. Answers whether it wrote one. +// +// The builder calls this with the size it has just uploaded; the sweep, through Hold, with the size +// the store reports. Both arrive at the same holder, which is the point of composing it. +func (s Store) HoldBlob(ctx context.Context, repository, digest string, size int64) (bool, error) { + if s.Address == "" { + return false, fmt.Errorf("this mesh has no artifact store on its network to hold %s/%s in", repository, digest) + } + body, holder := Holder(digest, size) + there, err := s.has(ctx, s.url(repository, "manifests", holder), manifestAccept...) + if err != nil { + return false, err + } + if there { + return false, nil + } + + // The config must be in the repository before a manifest naming it is accepted: a registry + // refuses a manifest whose blobs it cannot find there, which is the property that makes a + // holder mean something. + if err := s.putBlob(ctx, repository, emptyDigest, emptyConfig); err != nil { + return false, err + } + + // **By digest, never by tag.** A tag would be one more name to move and one more thing the + // collector's `--delete-untagged` would read as meaningful; the mesh names nothing by tag that + // it pins by digest, and an untagged manifest is kept by plain collection. + request, err := http.NewRequestWithContext(ctx, http.MethodPut, + s.url(repository, "manifests", holder), bytes.NewReader(body)) + if err != nil { + return false, err + } + request.Header.Set("Content-Type", mediaManifest) + response, err := s.client().Do(request) + if err != nil { + return false, err + } + defer response.Body.Close() + if response.StatusCode != http.StatusCreated { + said, _ := io.ReadAll(io.LimitReader(response.Body, 4096)) + return false, fmt.Errorf("the artifact store refused to hold %s/%s: %s %s", + repository, digest, response.Status, strings.TrimSpace(string(said))) + } + return true, nil +} + +// letGoOfHolder deletes the manifest holding an archive, before the archive's own link goes. +// +// **Holder first.** Deleting the blob link alone leaves a manifest still naming the blob, and the +// collector would keep its bytes for ever on the strength of it — the sweep would record the +// archive collected while the disk said otherwise. Deleting the holder first and failing before +// the link goes leaves an unheld archive that the next sweep still offers, which is safe. +// +// A store that no longer has the blob answers Gone: without its size the holder cannot be named, +// and without the blob there is nothing left for a holder to keep. A store that never had a holder +// for it — an archive published before holders, never backfilled — answers 404 to the delete, and +// that is the outcome wanted. +func (s Store) letGoOfHolder(ctx context.Context, repository, digest string) error { + size, err := s.blobSize(ctx, repository, digest) + if err != nil { + return err + } + _, holder := Holder(digest, size) + err = s.remove(ctx, s.url(repository, "manifests", holder), repository+"/manifests/"+holder) + if err == Gone { + return nil + } + return err +} + +// archive reads a recorded reference into its repository and digest, and whether it is an archive +// at all. Refuses as ErrNotOurs anything the mesh did not put in its own store. +func (s Store) archive(reference string) (repository, digest string, archive bool, err error) { + path, kept := catalogue.InArtifactStore(reference) + if !kept { + return "", "", false, fmt.Errorf("%w: %s", ErrNotOurs, reference) + } + if s.Address == "" { + return "", "", false, fmt.Errorf("this mesh has no artifact store on its network to ask about %s", reference) + } + repository, kind, digest, err := split(path) + if err != nil { + return "", "", false, err + } + return repository, digest, kind == "blobs", nil +} + +// blobSize is how large the store says a blob is; Gone when it does not have it. +func (s Store) blobSize(ctx context.Context, repository, digest string) (int64, error) { + request, err := http.NewRequestWithContext(ctx, http.MethodHead, s.url(repository, "blobs", digest), nil) + if err != nil { + return 0, err + } + response, err := s.client().Do(request) + if err != nil { + return 0, fmt.Errorf("cannot reach the artifact store at %s: %w", s.Address, err) + } + defer response.Body.Close() + switch response.StatusCode { + case http.StatusOK: + case http.StatusNotFound: + return 0, Gone + default: + return 0, fmt.Errorf("the artifact store answered %s for %s/blobs/%s", response.Status, repository, digest) + } + // Read from the header rather than ContentLength: a HEAD's ContentLength is what the response + // says it would have sent, which Go reports faithfully, but a proxy in between is free to drop + // it, and the header is what the registry itself wrote. + if length := response.Header.Get("Content-Length"); length != "" { + if n, err := strconv.ParseInt(length, 10, 64); err == nil && n >= 0 { + return n, nil + } + } + if response.ContentLength >= 0 { + return response.ContentLength, nil + } + return 0, fmt.Errorf("the artifact store holds %s/blobs/%s and will not say how large it is", repository, digest) +} + +// putBlob uploads a small blob unless the repository already has it: ask where, then put it there +// naming the digest — the registry's own two steps, the same the builder takes for an archive. +func (s Store) putBlob(ctx context.Context, repository, digest string, body []byte) error { + if there, err := s.has(ctx, s.url(repository, "blobs", digest)); err != nil { + return err + } else if there { + return nil + } + start, err := http.NewRequestWithContext(ctx, http.MethodPost, + "http://"+s.Address+"/v2/"+repository+"/blobs/uploads/", nil) + if err != nil { + return err + } + begun, err := s.client().Do(start) + if err != nil { + return fmt.Errorf("cannot start an upload to %s: %w", repository, err) + } + begun.Body.Close() + if begun.StatusCode != http.StatusAccepted { + return fmt.Errorf("the artifact store answered %s when asked where to put a blob in %s", begun.Status, repository) + } + where := begun.Header.Get("Location") + if where == "" { + return fmt.Errorf("the artifact store accepted an upload to %s and said nowhere to put it", repository) + } + if strings.HasPrefix(where, "/") { + where = "http://" + s.Address + where + } + separator := "?" + if strings.Contains(where, "?") { + separator = "&" + } + put, err := http.NewRequestWithContext(ctx, http.MethodPut, where+separator+"digest="+digest, bytes.NewReader(body)) + if err != nil { + return err + } + put.Header.Set("Content-Type", "application/octet-stream") + done, err := s.client().Do(put) + if err != nil { + return err + } + defer done.Body.Close() + if done.StatusCode != http.StatusCreated { + said, _ := io.ReadAll(io.LimitReader(done.Body, 4096)) + return fmt.Errorf("the artifact store refused a blob in %s: %s %s", repository, done.Status, strings.TrimSpace(string(said))) + } + return nil +} + +// has is whether the store answers 200 for a HEAD at that URL. +func (s Store) has(ctx context.Context, url string, accept ...string) (bool, error) { + request, err := http.NewRequestWithContext(ctx, http.MethodHead, url, nil) + if err != nil { + return false, err + } + for _, media := range accept { + request.Header.Add("Accept", media) + } + response, err := s.client().Do(request) + if err != nil { + return false, fmt.Errorf("cannot reach the artifact store at %s: %w", s.Address, err) + } + defer response.Body.Close() + switch response.StatusCode { + case http.StatusOK: + return true, nil + case http.StatusNotFound: + return false, nil + default: + return false, fmt.Errorf("the artifact store answered %s for %s", response.Status, url) + } +} + +func (s Store) url(repository, kind, digest string) string { + return "http://" + s.Address + "/v2/" + repository + "/" + kind + "/" + digest +} + +func (s Store) client() *http.Client { + if s.HTTP != nil { + return s.HTTP + } + return &http.Client{Timeout: 30 * time.Second} +} diff --git a/modules/artifact-store-tools/internal/registry/hold_test.go b/modules/artifact-store-tools/internal/registry/hold_test.go new file mode 100644 index 00000000..8a3a16fa --- /dev/null +++ b/modules/artifact-store-tools/internal/registry/hold_test.go @@ -0,0 +1,268 @@ +package registry + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/http/httptest" + "strconv" + "strings" + "sync" + "testing" + + "git.novox.be/novox/mesh-sdk/go/catalogue" +) + +// Every kept archive is held by a manifest (novox/hq issue 253, ADR 0189). +// +// Against an in-memory registry that keeps blobs and manifests per repository and refuses what a +// registry refuses — a blob whose digest does not match, a manifest whose digest does not match +// or whose blobs the repository does not have, a manifest asked for without an Accept naming its +// type. What is asserted is this side's decisions; the live test below asserts the registry's. + +type memRegistry struct { + mu sync.Mutex + blobs map[string][]byte // repository + "@" + digest + manifests map[string][]byte // repository + "@" + digest + writes []string // every PUT and DELETE, as "METHOD path" +} + +func digestOf(body []byte) string { + sum := sha256.Sum256(body) + return "sha256:" + hex.EncodeToString(sum[:]) +} + +func (m *memRegistry) serve(t *testing.T) Store { + t.Helper() + m.blobs = map[string][]byte{} + m.manifests = map[string][]byte{} + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + m.mu.Lock() + defer m.mu.Unlock() + path := strings.TrimPrefix(r.URL.Path, "/v2/") + if r.Method == http.MethodPut || r.Method == http.MethodDelete { + m.writes = append(m.writes, r.Method+" "+r.URL.Path) + } + switch { + case r.Method == http.MethodPost && strings.HasSuffix(path, "/blobs/uploads/"): + repository := strings.TrimSuffix(path, "/blobs/uploads/") + w.Header().Set("Location", "/upload/"+repository+"?state=x") + w.WriteHeader(http.StatusAccepted) + case r.Method == http.MethodPut && strings.HasPrefix(r.URL.Path, "/upload/"): + repository := strings.TrimPrefix(r.URL.Path, "/upload/") + body, _ := io.ReadAll(r.Body) + digest := r.URL.Query().Get("digest") + if digest != digestOf(body) { + w.WriteHeader(http.StatusBadRequest) + return + } + m.blobs[repository+"@"+digest] = body + w.WriteHeader(http.StatusCreated) + case strings.Contains(path, "/blobs/"): + repository, digest, _ := strings.Cut(path, "/blobs/") + key := repository + "@" + digest + body, ok := m.blobs[key] + if !ok { + w.WriteHeader(http.StatusNotFound) + return + } + switch r.Method { + case http.MethodHead: + w.Header().Set("Content-Length", strconv.Itoa(len(body))) + w.WriteHeader(http.StatusOK) + case http.MethodDelete: + delete(m.blobs, key) + w.WriteHeader(http.StatusAccepted) + default: + w.WriteHeader(http.StatusMethodNotAllowed) + } + case strings.Contains(path, "/manifests/"): + repository, digest, _ := strings.Cut(path, "/manifests/") + key := repository + "@" + digest + switch r.Method { + case http.MethodHead: + if _, ok := m.manifests[key]; !ok || !strings.Contains(r.Header.Get("Accept"), mediaManifest) { + w.WriteHeader(http.StatusNotFound) + return + } + w.WriteHeader(http.StatusOK) + case http.MethodPut: + body, _ := io.ReadAll(r.Body) + if digest != digestOf(body) { + w.WriteHeader(http.StatusBadRequest) + return + } + var named holderManifest + if err := json.Unmarshal(body, &named); err != nil { + w.WriteHeader(http.StatusBadRequest) + return + } + for _, d := range append([]descriptor{named.Config}, named.Layers...) { + if _, ok := m.blobs[repository+"@"+d.Digest]; !ok { + w.WriteHeader(http.StatusBadRequest) + fmt.Fprintf(w, "MANIFEST_BLOB_UNKNOWN %s", d.Digest) + return + } + } + m.manifests[key] = body + w.WriteHeader(http.StatusCreated) + case http.MethodDelete: + if _, ok := m.manifests[key]; !ok { + w.WriteHeader(http.StatusNotFound) + return + } + delete(m.manifests, key) + w.WriteHeader(http.StatusAccepted) + } + default: + w.WriteHeader(http.StatusNotFound) + } + })) + t.Cleanup(server.Close) + return Store{Address: strings.TrimPrefix(server.URL, "http://")} +} + +// bare puts an archive in the store the way the builder did before holders: a blob, nothing more. +func (m *memRegistry) bare(repository string, body []byte) string { + m.mu.Lock() + defer m.mu.Unlock() + digest := digestOf(body) + m.blobs[repository+"@"+digest] = body + return catalogue.ArtifactStoreScheme + repository + "/blobs/" + digest +} + +func TestTheHolderIsComposedFromTheDigestAndSizeAlone(t *testing.T) { + // The sweep must arrive at the very manifest the builder wrote, with nothing recorded between + // them. Same inputs, same bytes — and a different size is a different holder, so a holder can + // never be mistaken for one of a different blob. + digest := "sha256:" + strings.Repeat("a", 64) + one, first := Holder(digest, 42) + two, second := Holder(digest, 42) + if !bytes.Equal(one, two) || first != second { + t.Fatalf("the same archive composed two holders:\n%s\n%s", one, two) + } + if _, other := Holder(digest, 43); other == first { + t.Fatal("a different size composed the same holder") + } + want := `{"schemaVersion":2,"mediaType":"application/vnd.oci.image.manifest.v1+json",` + + `"config":{"mediaType":"application/vnd.oci.empty.v1+json",` + + `"digest":"sha256:44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a","size":2},` + + `"layers":[{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","digest":"` + digest + `","size":42}]}` + if string(one) != want { + t.Fatalf("the holder is\n%s\nwant\n%s", one, want) + } + if digestOf(emptyConfig) != emptyDigest { + t.Fatalf("the empty config's digest is %s, not %s", digestOf(emptyConfig), emptyDigest) + } +} + +func TestHoldBackfillsABareArchiveAndIsIdempotent(t *testing.T) { + // The archives published before this have no holder. Hold, run over every kept archive on + // every sweep, writes one the first time and nothing after. + m := &memRegistry{} + store := m.serve(t) + ctx := context.Background() + body := []byte("a theme") + reference := m.bare("shell/config", body) + + if held, err := store.Held(ctx, reference); err != nil || held { + t.Fatalf("a bare blob reads as held=%v (%v)", held, err) + } + wrote, err := store.Hold(ctx, reference) + if err != nil { + t.Fatal(err) + } + if !wrote { + t.Fatal("holding a bare archive wrote nothing") + } + _, holder := Holder(digestOf(body), int64(len(body))) + if _, ok := m.manifests["shell/config@"+holder]; !ok { + t.Fatalf("the store holds manifests %v; want %s", m.manifests, holder) + } + if held, err := store.Held(ctx, reference); err != nil || !held { + t.Fatalf("after holding, held=%v (%v)", held, err) + } + + writes := len(m.writes) + wrote, err = store.Hold(ctx, reference) + if err != nil { + t.Fatal(err) + } + if wrote || len(m.writes) != writes { + t.Fatalf("holding again wrote %v", m.writes[writes:]) + } +} + +func TestAnImageNeedsNoHolderAndAMissingArchiveIsGone(t *testing.T) { + m := &memRegistry{} + store := m.serve(t) + ctx := context.Background() + + // An image is its own manifest: nothing is asked. + wrote, err := store.Hold(ctx, catalogue.ArtifactStoreScheme+"web/app@sha256:"+strings.Repeat("b", 64)) + if err != nil || wrote || len(m.writes) != 0 { + t.Fatalf("holding an image wrote=%v err=%v writes=%v", wrote, err, m.writes) + } + // An archive the store does not have is a fact to report, not something to invent a holder for. + _, err = store.Hold(ctx, catalogue.ArtifactStoreScheme+"web/config/blobs/sha256:"+strings.Repeat("c", 64)) + if !errors.Is(err, Gone) { + t.Fatalf("holding a missing archive answered %v, want Gone", err) + } + // And a reference that is not the mesh's is refused as such. + if _, err := store.Hold(ctx, "docker.io/library/registry@sha256:abc"); !errors.Is(err, ErrNotOurs) { + t.Fatalf("holding a vendor's image answered %v, want ErrNotOurs", err) + } +} + +func TestLettingGoOfAnArchiveDeletesItsHolderFirst(t *testing.T) { + // A holder left behind would keep the bytes through every collection while the record said + // collected; the link deleted first and the holder failing after would be that exactly. + m := &memRegistry{} + store := m.serve(t) + ctx := context.Background() + body := []byte("an old theme") + reference := m.bare("shell/config", body) + if _, err := store.Hold(ctx, reference); err != nil { + t.Fatal(err) + } + m.writes = nil + + if err := store.LetGo(ctx, reference); err != nil { + t.Fatal(err) + } + _, holder := Holder(digestOf(body), int64(len(body))) + want := []string{ + "DELETE /v2/shell/config/manifests/" + holder, + "DELETE /v2/shell/config/blobs/" + digestOf(body), + } + if strings.Join(m.writes, "\n") != strings.Join(want, "\n") { + t.Fatalf("the store was asked\n%s\nwant\n%s", strings.Join(m.writes, "\n"), strings.Join(want, "\n")) + } + if len(m.manifests) != 0 { + t.Fatalf("a holder survived: %v", m.manifests) + } + // Asked again, the archive is already gone, which is the outcome wanted. + if err := store.LetGo(ctx, reference); !errors.Is(err, Gone) { + t.Fatalf("letting go twice answered %v, want Gone", err) + } +} + +func TestLettingGoOfAnUnheldArchiveStillDeletesIt(t *testing.T) { + // An archive published before holders and let go of before any sweep held it: the holder's + // delete answers 404, which is the outcome wanted, and the blob still goes. + m := &memRegistry{} + store := m.serve(t) + reference := m.bare("shell/config", []byte("never held")) + if err := store.LetGo(context.Background(), reference); err != nil { + t.Fatal(err) + } + if len(m.blobs) != 0 { + t.Fatalf("the blob survived: %v", m.blobs) + } +} diff --git a/modules/artifact-store-tools/internal/registry/platforms_test.go b/modules/artifact-store-tools/internal/registry/platforms_test.go new file mode 100644 index 00000000..a335fb2c --- /dev/null +++ b/modules/artifact-store-tools/internal/registry/platforms_test.go @@ -0,0 +1,176 @@ +package registry + +import ( + "context" + "errors" + "fmt" + "net/http" + "net/http/httptest" + "slices" + "strings" + "sync" + "testing" + + "git.novox.be/novox/mesh-sdk/go/catalogue" +) + +// An index is let go of with the platform manifests it names, except those a kept index of the same +// repository names too (novox/hq ADR 0257). Each platform manifest is a manifest of the repository in +// its own right, and the store's collector keeps every manifest a repository holds — so an index let go +// of alone frees nothing of the images it names. + +func digestN(n int) string { return fmt.Sprintf("sha256:%064x", n) } + +// indexStore holds indexes and images in one repository, answers GETs with their documents, and records +// every delete. +type indexStore struct { + mu sync.Mutex + indexes map[string][]string // digest → the platform manifests it names + images map[string]bool + deleted []string +} + +func (s *indexStore) serve(t *testing.T) Store { + t.Helper() + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + s.mu.Lock() + defer s.mu.Unlock() + digest := r.URL.Path[strings.LastIndex(r.URL.Path, "/")+1:] + switch r.Method { + case http.MethodGet: + if children, ok := s.indexes[digest]; ok { + var named []string + for _, c := range children { + named = append(named, `{"mediaType":"application/vnd.oci.image.manifest.v1+json","digest":"`+c+`","size":1}`) + } + w.Header().Set("Content-Type", "application/vnd.oci.image.index.v1+json") + _, _ = w.Write([]byte(`{"schemaVersion":2,"mediaType":"application/vnd.oci.image.index.v1+json","manifests":[` + + strings.Join(named, ",") + `]}`)) + return + } + if s.images[digest] { + w.Header().Set("Content-Type", "application/vnd.oci.image.manifest.v1+json") + _, _ = w.Write([]byte(`{"schemaVersion":2,"mediaType":"application/vnd.oci.image.manifest.v1+json","layers":[]}`)) + return + } + w.WriteHeader(http.StatusNotFound) + case http.MethodHead: + if _, ok := s.indexes[digest]; ok || s.images[digest] { + w.WriteHeader(http.StatusOK) + return + } + w.WriteHeader(http.StatusNotFound) + case http.MethodDelete: + if _, ok := s.indexes[digest]; !ok && !s.images[digest] { + w.WriteHeader(http.StatusNotFound) + return + } + s.deleted = append(s.deleted, digest) + delete(s.indexes, digest) + delete(s.images, digest) + w.WriteHeader(http.StatusAccepted) + default: + w.WriteHeader(http.StatusBadRequest) + } + })) + t.Cleanup(server.Close) + return Store{Address: strings.TrimPrefix(server.URL, "http://")} +} + +func TestAnIndexGoesWithItsPlatformsAndAKeptIndexKeepsItsOwn(t *testing.T) { + // Two indexes of one image's repository: the old one names platforms 11 and 12, the kept one 12 and 13. + s := &indexStore{ + indexes: map[string][]string{digestN(1): {digestN(11), digestN(12)}, digestN(2): {digestN(12), digestN(13)}}, + images: map[string]bool{digestN(11): true, digestN(12): true, digestN(13): true}, + } + store := s.serve(t) + repository := "upstream/docker.io/library/golang" + old := catalogue.ArtifactStoreScheme + repository + "@" + digestN(1) + kept := catalogue.ArtifactStoreScheme + repository + "@" + digestN(2) + spare, err := store.SpareKeptIndexes(context.Background(), []string{kept}, []string{old}) + if err != nil { + t.Fatal(err) + } + store.Spare = spare + + if err := store.LetGo(context.Background(), old); err != nil { + t.Fatal(err) + } + // Its own platform first, the index last; the platform the kept index names is left in place. + if !slices.Equal(s.deleted, []string{digestN(11), digestN(1)}) { + t.Fatalf("deleted %v; want the old index's own platform, then the index", s.deleted) + } + if !s.images[digestN(12)] || !s.images[digestN(13)] { + t.Fatal("a platform a kept index names was let go of") + } +} + +func TestWithoutSparingAnIndexGoesAloneAndAnImageNamesNoPlatforms(t *testing.T) { + s := &indexStore{indexes: map[string][]string{digestN(1): {digestN(11)}}, images: map[string]bool{digestN(11): true, digestN(5): true}} + store := s.serve(t) + // No Spare: as before ADR 0257, the index alone. + if err := store.LetGo(context.Background(), catalogue.ArtifactStoreScheme+"web/base@"+digestN(1)); err != nil { + t.Fatal(err) + } + if !slices.Equal(s.deleted, []string{digestN(1)}) { + t.Fatalf("without a Spare, deleted %v", s.deleted) + } + // An image is one delete, with a Spare or without. + s.deleted = nil + spare, err := store.SpareKeptIndexes(context.Background(), nil, []string{catalogue.ArtifactStoreScheme + "web/app@" + digestN(5)}) + if err != nil { + t.Fatal(err) + } + store.Spare = spare + if err := store.LetGo(context.Background(), catalogue.ArtifactStoreScheme+"web/app@"+digestN(5)); err != nil { + t.Fatal(err) + } + if !slices.Equal(s.deleted, []string{digestN(5)}) { + t.Fatalf("an image was let go of as %v", s.deleted) + } + // An index the store no longer holds is Gone, and nothing is deleted on its account. + s.deleted = nil + err = store.LetGo(context.Background(), catalogue.ArtifactStoreScheme+"web/base@"+digestN(9)) + if !errors.Is(err, Gone) || len(s.deleted) != 0 { + t.Fatalf("an index the store does not hold: %v, deleted %v", err, s.deleted) + } +} + +func TestASpareThatCannotBeReadKeepsTheIndex(t *testing.T) { + s := &indexStore{indexes: map[string][]string{digestN(1): {digestN(11)}}, images: map[string]bool{digestN(11): true}} + store := s.serve(t) + store.Spare = func(context.Context, string) (map[string]bool, error) { return nil, fmt.Errorf("the store went away") } + err := store.LetGo(context.Background(), catalogue.ArtifactStoreScheme+"web/base@"+digestN(1)) + if err == nil || !strings.Contains(err.Error(), "was kept") { + t.Fatalf("a spare that could not be read: %v", err) + } + if len(s.deleted) != 0 { + t.Fatalf("deleted %v without knowing what a kept index names", s.deleted) + } +} + +// The spare list is read whole before the sweep: a kept index that cannot be read is an error before +// anything is deleted, and a repository not read before keeps its index. +func TestASpareListIsReadWholeBeforeAnythingGoes(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusInternalServerError) + })) + t.Cleanup(server.Close) + store := Store{Address: strings.TrimPrefix(server.URL, "http://")} + kept := catalogue.ArtifactStoreScheme + "web/base@" + digestN(2) + old := catalogue.ArtifactStoreScheme + "web/base@" + digestN(1) + if _, err := store.SpareKeptIndexes(context.Background(), []string{kept}, []string{old}); err == nil { + t.Fatal("a kept index the store would not answer for was taken as naming nothing") + } + + s := &indexStore{indexes: map[string][]string{digestN(1): {digestN(11)}}, images: map[string]bool{digestN(11): true}} + good := s.serve(t) + spare, err := good.SpareKeptIndexes(context.Background(), nil, []string{catalogue.ArtifactStoreScheme + "other/repo@" + digestN(3)}) + if err != nil { + t.Fatal(err) + } + good.Spare = spare + if err := good.LetGo(context.Background(), old); err == nil || len(s.deleted) != 0 { + t.Fatalf("an index in a repository not read before the sweep was let go of: %v, deleted %v", err, s.deleted) + } +} diff --git a/modules/artifact-store-tools/internal/registry/store.go b/modules/artifact-store-tools/internal/registry/store.go new file mode 100644 index 00000000..8a1fe931 --- /dev/null +++ b/modules/artifact-store-tools/internal/registry/store.go @@ -0,0 +1,325 @@ +// Package registry speaks to the mesh's artifact store over its own door. +// +// Only what the mesh needs that nothing else does: letting go of something it put there +// (novox/hq ADR 0189, issue 108), and holding every archive it keeps by a manifest so the store's +// own collector does not take it (novox/hq issue 253). Pushing is the builder's, through the container runtime; reading +// is every machine's, through its runtime. This is the one operation that belongs to the thing +// holding the records, because it is the only one that is a decision rather than a transfer. +// +// Copied from mesh-controller internal/artifacts at e04846e9 (novox/hq ADR 0571 clause 2): the seat mesh-artifact-sweep lets +// go of what it no longer keeps exactly as the controller's sweep did; the controller's copy goes with its lease. +package registry + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "strings" + + "git.novox.be/novox/mesh-sdk/go/catalogue" +) + +// Store is the artifact store at an address, as this machine reaches it. +type Store struct { + // Address is `host:port` — the store as the caller reaches it now, composed and never + // recorded (novox/hq 04-ISSUES/102). + Address string + // HTTP is the client used; nil is a client with a modest timeout. + HTTP *http.Client + // Spare says, for one repository, which manifests a kept index there names: the platform + // manifests an index being let go of must leave in place (novox/hq ADR 0257). Nil spares every + // platform manifest — an index is then let go of alone, as it was before. + Spare func(ctx context.Context, repository string) (map[string]bool, error) +} + +// Gone is the answer when the store does not hold it: the outcome wanted, already true. +var Gone = errors.New("the store does not hold it") + +// ErrNotOurs is a reference this sweep will not address: not the mesh's own, or naming nothing +// the store holds by digest. +// +// **A fact about the record, not about the store** (novox/hq issue 226). The two deserve opposite +// responses — skip one and go on, abandon the sweep for the other — and collapsing them into "an +// error" is how a cautious loop became one that did nothing while reporting the right number. +var ErrNotOurs = errors.New("not a reference into the mesh's artifact store") + +// LetGo asks the store to drop one artifact the mesh recorded making. +// +// Takes a reference as the mesh records it — `artifact-store:///@sha256:…` for +// an image, `…/blobs/sha256:…` for an archive — because that is the identity every record uses, +// and composes the address here at the moment of use. +// +// An archive is let go of in two deletes, its holder manifest and then the blob's link (novox/hq +// issue 253); an image in one. +// +// Returns Gone when the store answers that it does not have it. That is not a failure: the sweep +// wants the artifact absent, and it is. It is distinguished from success only so a caller can say +// which of the two happened. +func (s Store) LetGo(ctx context.Context, reference string) error { + // **Strict, and deliberately** (novox/hq issue 226). Only a reference the mesh keeps in its + // own vocabulary is addressed here. `Recorded` would read `docker.io/library/registry@sha256:…` + // as the mesh's too — it cannot tell one registry host from another — so normalising belongs + // where the provenance is known, which is the sweep reading its own build records, not here + // where the only job is to refuse anything that is not plainly ours. + path, kept := catalogue.InArtifactStore(reference) + if !kept { + // Nothing the mesh put in its own store. Refused rather than attempted: composing a + // delete for a reference of unknown shape is how a sweep reaches something that is not + // the mesh's. Distinguished from a store that refuses, so a sweep skips this and goes on. + return fmt.Errorf("%w: %s", ErrNotOurs, reference) + } + if s.Address == "" { + return fmt.Errorf("this mesh has no artifact store on its network to ask about %s", reference) + } + repository, kind, digest, err := split(path) + if err != nil { + return err + } + if kind == "blobs" { + // **An archive's holder goes before the archive** (novox/hq issue 253): a manifest left + // naming the blob would keep its bytes through every collection while the record said + // collected. Gone here means the store has no such blob, so there is nothing to let go. + if err := s.letGoOfHolder(ctx, repository, digest); err != nil { + return err + } + } + if kind == "manifests" && s.Spare != nil { + // **An index's platform manifests go before the index** (novox/hq ADR 0257). Each was put + // under its own digest, so each is a manifest of the repository in its own right, and the + // store's collector keeps every manifest a repository holds: an index let go of alone frees + // no byte of the images it names. Before, not after, so a sweep cut short leaves the index + // to be asked about again, and with it the list of what is still to go. + if err := s.letGoOfPlatforms(ctx, repository, digest, reference); err != nil { + return err + } + } + return s.remove(ctx, s.url(repository, kind, digest), reference) +} + +// letGoOfPlatforms deletes the manifests an index names, except those a kept index of the same +// repository also names. A manifest that is not an index names none. +func (s Store) letGoOfPlatforms(ctx context.Context, repository, digest, reference string) error { + children, err := s.Platforms(ctx, repository, digest) + if err != nil || len(children) == 0 { + // Gone: there is no index to read, and the delete that follows says so. + if errors.Is(err, Gone) { + return nil + } + return err + } + spared, err := s.Spare(ctx, repository) + if err != nil { + return fmt.Errorf("cannot say which platform manifests of %s a kept index names, so %s was kept: %w", + repository, reference, err) + } + for _, child := range children { + if spared[child] { + continue + } + if err := s.remove(ctx, s.url(repository, "manifests", child), reference+" (its "+child+")"); err != nil && !errors.Is(err, Gone) { + return err + } + } + return nil +} + +// anyManifest is every manifest the store may hold, indexes included, so it answers with the +// document as it is. +var anyManifest = []string{ + "application/vnd.oci.image.index.v1+json", + "application/vnd.docker.distribution.manifest.list.v2+json", + "application/vnd.oci.image.manifest.v1+json", + "application/vnd.docker.distribution.manifest.v2+json", +} + +// Platforms is the manifests an index names, by digest; none for a manifest that is not an index. +// Gone when the store does not hold it. +func (s Store) Platforms(ctx context.Context, repository, digest string) ([]string, error) { + request, err := http.NewRequestWithContext(ctx, http.MethodGet, s.url(repository, "manifests", digest), nil) + if err != nil { + return nil, err + } + request.Header.Set("Accept", strings.Join(anyManifest, ", ")) + response, err := s.client().Do(request) + if err != nil { + return nil, err + } + defer response.Body.Close() + switch response.StatusCode { + case http.StatusOK: + case http.StatusNotFound: + return nil, Gone + default: + return nil, fmt.Errorf("the artifact store answered %s when asked for %s@%s", response.Status, repository, digest) + } + var document struct { + Manifests []struct { + Digest string `json:"digest"` + } `json:"manifests"` + } + if err := json.NewDecoder(io.LimitReader(response.Body, 4<<20)).Decode(&document); err != nil { + return nil, fmt.Errorf("%s@%s is not a manifest: %w", repository, digest, err) + } + var out []string + for _, m := range document.Manifests { + if strings.HasPrefix(m.Digest, "sha256:") { + out = append(out, m.Digest) + } + } + return out, nil +} + +// SpareKeptIndexes is a Spare for a sweep, read whole before it starts: for every repository an +// image reference of `touching` is in, each kept manifest there and every platform manifest a kept +// index there names. Any read that fails is an error, and the sweep lets nothing go. A repository not +// read here is answered with an error, so an index in it is kept. A kept index the store no longer +// holds names nothing. +func (s Store) SpareKeptIndexes(ctx context.Context, kept, touching []string) (func(context.Context, string) (map[string]bool, error), error) { + imageIn := func(reference string) (repository, digest string, ok bool) { + path, ours := catalogue.InArtifactStore(reference) + if !ours { + return "", "", false + } + repository, kind, digest, err := split(path) + if err != nil || kind != "manifests" { + return "", "", false + } + return repository, digest, true + } + read := map[string]map[string]bool{} + for _, reference := range touching { + if repository, _, ok := imageIn(reference); ok { + read[repository] = map[string]bool{} + } + } + for _, reference := range kept { + repository, digest, ok := imageIn(reference) + spared, touched := read[repository] + if !ok || !touched { + continue + } + spared[digest] = true + children, err := s.Platforms(ctx, repository, digest) + if errors.Is(err, Gone) { + continue + } + if err != nil { + return nil, fmt.Errorf("reading the kept %s@%s: %w", repository, digest, err) + } + for _, c := range children { + spared[c] = true + } + } + return func(_ context.Context, repository string) (map[string]bool, error) { + spared, done := read[repository] + if !done { + return nil, fmt.Errorf("%s was not read before the sweep began", repository) + } + return spared, nil + }, nil +} + +// remove asks the store to delete what is at url. Gone when it has no such thing. +func (s Store) remove(ctx context.Context, url, what string) error { + request, err := http.NewRequestWithContext(ctx, http.MethodDelete, url, nil) + if err != nil { + return err + } + response, err := s.client().Do(request) + if err != nil { + return err + } + defer response.Body.Close() + switch response.StatusCode { + case http.StatusAccepted, http.StatusOK, http.StatusNoContent: + return nil + case http.StatusNotFound: + return Gone + case http.StatusMethodNotAllowed: + // The registry was started without deletion enabled. Said plainly, because the remedy is + // a setting on the store's module and not anything about this artifact. + return fmt.Errorf( + "the artifact store refuses deletion: its server was started without it enabled "+ + "(REGISTRY_STORAGE_DELETE_ENABLED), so nothing can be collected until the store "+ + "module is applied again (novox/hq ADR 0189). Asking about %s", what) + default: + return fmt.Errorf("the artifact store answered %s for %s", response.Status, what) + } +} + +// split reads a recorded path into the repository, which endpoint names the thing, and the digest. +// +// Two shapes, which are the two the mesh records: `@sha256:` is a manifest, and +// `/blobs/sha256:` is a blob. +func split(path string) (repository, kind, digest string, err error) { + if before, after, ok := strings.Cut(path, "@sha256:"); ok { + return before, "manifests", "sha256:" + after, nil + } + if before, after, ok := strings.Cut(path, "/blobs/sha256:"); ok { + return before, "blobs", "sha256:" + after, nil + } + return "", "", "", fmt.Errorf("%w: %q names nothing the store holds by digest", ErrNotOurs, path) +} + +// IsIndex is whether a recorded reference is an index the store holds: one that names manifests. A +// blob, or a manifest that names none, is not. Gone when the store does not hold it. +func (s Store) IsIndex(ctx context.Context, reference string) (bool, error) { + path, ours := catalogue.InArtifactStore(reference) + if !ours { + return false, fmt.Errorf("%w: %s", ErrNotOurs, reference) + } + repository, kind, digest, err := split(path) + if err != nil || kind != "manifests" { + return false, err + } + children, err := s.Platforms(ctx, repository, digest) + return len(children) > 0, err +} + +// HoldsManifest is whether the store holds the manifest a recorded image reference names. +func (s Store) HoldsManifest(ctx context.Context, reference string) (bool, error) { + path, ours := catalogue.InArtifactStore(reference) + if !ours { + return false, fmt.Errorf("%w: %s", ErrNotOurs, reference) + } + repository, kind, digest, err := split(path) + if err != nil { + return false, err + } + if kind != "manifests" { + return false, fmt.Errorf("%w: %s names a blob, not a manifest", ErrNotOurs, reference) + } + return s.has(ctx, s.url(repository, kind, digest), anyManifest...) +} + +// Holds is whether the store holds what a recorded reference names: the blob of an archive, the manifest of an image. +// Gone when it does not; asks and changes nothing (ADR 0526 clause 3: a registration by hand is refused at bytes that +// are not there). +func (s Store) Holds(ctx context.Context, reference string) error { + path, ours := catalogue.InArtifactStore(reference) + if !ours { + return fmt.Errorf("%w: %s", ErrNotOurs, reference) + } + repository, kind, digest, err := split(path) + if err != nil { + return err + } + if s.Address == "" { + return fmt.Errorf("this mesh has no artifact store on its network to ask about %s", reference) + } + if kind == "blobs" { + _, err := s.blobSize(ctx, repository, digest) + return err + } + held, err := s.has(ctx, s.url(repository, kind, digest), anyManifest...) + if err != nil { + return err + } + if !held { + return Gone + } + return nil +} diff --git a/modules/artifact-store-tools/internal/registry/store_test.go b/modules/artifact-store-tools/internal/registry/store_test.go new file mode 100644 index 00000000..334b95a5 --- /dev/null +++ b/modules/artifact-store-tools/internal/registry/store_test.go @@ -0,0 +1,134 @@ +package registry + +import ( + "context" + "errors" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "git.novox.be/novox/mesh-sdk/go/catalogue" +) + +// Asking the store to let go of what the mesh no longer keeps (novox/hq ADR 0189, issue 108). +// +// A fake store records what it was asked to delete, so what is asserted is the mesh's decision +// and the shape of the request — not the registry's behaviour, which is the registry's to test. + +func fakeStore(t *testing.T, answer int) (Store, *[]string) { + t.Helper() + var asked []string + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodHead && strings.Contains(r.URL.Path, "/blobs/") { + // An archive's size, asked so its holder can be named (novox/hq issue 253). A store + // that does not have the thing does not have its blob either. + if answer == http.StatusNotFound { + w.WriteHeader(http.StatusNotFound) + return + } + w.Header().Set("Content-Length", "7") + w.WriteHeader(http.StatusOK) + return + } + if r.Method != http.MethodDelete { + t.Errorf("the store was asked %s %s; collecting is a delete", r.Method, r.URL.Path) + } + asked = append(asked, r.URL.Path) + w.WriteHeader(answer) + })) + t.Cleanup(server.Close) + return Store{Address: strings.TrimPrefix(server.URL, "http://")}, &asked +} + +func TestAnImageAndAnArchiveAreAskedForAtTheirOwnEndpoints(t *testing.T) { + // The two shapes the mesh records: a manifest by digest, and a blob by digest. They are + // different endpoints, and asking at the wrong one answers 404 — which this would then + // record as collected, leaving the bytes on disk for ever while the record says otherwise. + store, asked := fakeStore(t, http.StatusAccepted) + ctx := context.Background() + + image := catalogue.ArtifactStoreScheme + "web/app@sha256:abc123" + archive := catalogue.ArtifactStoreScheme + "web/config/blobs/sha256:def456" + if err := store.LetGo(ctx, image); err != nil { + t.Fatal(err) + } + if err := store.LetGo(ctx, archive); err != nil { + t.Fatal(err) + } + // The archive's holder goes first, then the archive (novox/hq issue 253). + _, holder := Holder("sha256:def456", 7) + want := []string{ + "/v2/web/app/manifests/sha256:abc123", + "/v2/web/config/manifests/" + holder, + "/v2/web/config/blobs/sha256:def456", + } + if strings.Join(*asked, " ") != strings.Join(want, " ") { + t.Fatalf("the store was asked %v; want %v", *asked, want) + } +} + +func TestAStoreThatDoesNotHaveItAnswersGone(t *testing.T) { + // The outcome wanted, already true. Told apart from success only so the sweep can say which + // happened; both are recorded, because retrying for ever is the thing to avoid. + store, _ := fakeStore(t, http.StatusNotFound) + err := store.LetGo(context.Background(), catalogue.ArtifactStoreScheme+"web/app@sha256:abc123") + if !errors.Is(err, Gone) { + t.Fatalf("a store that does not hold it answered %v, want Gone", err) + } +} + +func TestAStoreWithDeletionOffSaysSoAndNamesTheRemedy(t *testing.T) { + // The registry answers 405 when it was started without deletion enabled. The remedy is a + // setting on the store's module, and saying "405" would send somebody to the wrong place. + store, _ := fakeStore(t, http.StatusMethodNotAllowed) + err := store.LetGo(context.Background(), catalogue.ArtifactStoreScheme+"web/app@sha256:abc123") + if err == nil { + t.Fatal("a store that refuses deletion was read as success") + } + if !strings.Contains(err.Error(), "REGISTRY_STORAGE_DELETE_ENABLED") { + t.Fatalf("the refusal does not name the remedy: %v", err) + } +} + +func TestAReferenceThatIsNotTheMeshsOwnIsNeverAsked(t *testing.T) { + // The whole safety of the sweep is that it names only what the mesh recorded putting there. + // A reference of another shape — a vendor's image, a package version — is refused rather + // than composed into a delete somewhere that is not the mesh's store. + store, asked := fakeStore(t, http.StatusAccepted) + for _, reference := range []string{ + "docker.io/library/registry@sha256:abc123", + "registry@sha256:abc123", + "1.4.2", + } { + if err := store.LetGo(context.Background(), reference); err == nil { + t.Errorf("%s was asked about; it is not a reference into the mesh's store", reference) + } + } + if len(*asked) != 0 { + t.Fatalf("the store was asked about %v", *asked) + } +} + +// A reference this sweep will not address says so as ErrNotOurs, which is a fact about the +// record and not about the store (novox/hq issue 226). +// +// The sweep skips one and abandons itself for the other, so they cannot be the same error. The +// first live run met a reference recorded with the store's old address, read the refusal as "the +// store refuses everything", and collected none of the 1681 it had found. +func TestAReferenceThisSweepWillNotAddressIsToldApartFromAStoreRefusing(t *testing.T) { + store, asked := fakeStore(t, http.StatusAccepted) + for _, reference := range []string{ + "docker.io/library/registry@sha256:abc123", + "127.0.0.1:5100/mesh-tools/build@sha256:abc123", + "1.4.2", + } { + err := store.LetGo(context.Background(), reference) + if !errors.Is(err, ErrNotOurs) { + t.Errorf("%s answered %v; a sweep must be able to skip it and go on", reference, err) + } + } + if len(*asked) != 0 { + t.Fatalf("the store was asked about %v", *asked) + } +} diff --git a/modules/artifact-store-tools/module.json b/modules/artifact-store-tools/module.json index 208329cd..f1ef7a43 100644 --- a/modules/artifact-store-tools/module.json +++ b/modules/artifact-store-tools/module.json @@ -1,26 +1,52 @@ { "module": "artifact-store-tools", "version": "1", + "claims": [ + { + "name": "mesh-artifact-sweep", + "scope": "mesh", + "serves": [ + "artifacts", + "held", + "collect", + "mirrors", + "images" + ] + } + ], + "requires": [ + "artifact-store" + ], "invokes": [ - "seat:mesh-controller.artifacts", - "seat:mesh-controller.collect", + "seat:mesh-hand-acts.hand-act", "mesh-catalog.catalog_modules", "mesh-catalog.catalog_module" ], + "reads": [ + "mesh-delivery-assigner.wants", + "mesh-delivery-builder.builds", + "mesh-delivery-builder.build-log", + "mesh-delivery-indexer.manifests", + "node-engine.have", + "mesh-controller.lease" + ], + "state": [ + "findings" + ], "tools": [ "artifact_store_repositories", "artifact_store_usage", "artifact_store_references", "artifact_store_collect" ], - "config": { - "older-store-names": { - "type": "string", - "description": "The names the artifact store had before, as host names or host:port separated by spaces, or none when it had no other. artifact_store_references reads every definition under these as well as under the store's current name, so an image a definition pins under one is said to be named by a definition (novox/hq issue 323).", - "pattern": "none|[A-Za-z0-9.:-]+( [A-Za-z0-9.:-]+)*", - "default": "none" - } - }, + "config": { + "older-store-names": { + "type": "string", + "description": "The names the artifact store had before, as host names or host:port separated by spaces, or none when it had no other. artifact_store_references reads every definition under these as well as under the store's current name, so an image a definition pins under one is said to be named by a definition (novox/hq issue 323).", + "pattern": "none|[A-Za-z0-9.:-]+( [A-Za-z0-9.:-]+)*", + "default": "none" + } + }, "resources": [ { "id": "state", @@ -36,6 +62,14 @@ "mode": "0600", "trusted": false, "template": "files/store-names" + }, + { + "id": "store-address", + "type": "file", + "path": "${dir:state}/store-address", + "mode": "0600", + "trusted": false, + "template": "files/store-address" } ], "build": { @@ -52,7 +86,8 @@ ], "env": { "MESH_ARTIFACT_STORE_CONTAINER": "mesh-registry", - "MESH_ARTIFACT_STORE_NAMES_FILE": "${dir:state}/store-names" + "MESH_ARTIFACT_STORE_NAMES_FILE": "${dir:state}/store-names", + "MESH_ARTIFACT_STORE_ADDRESS_FILE": "${dir:state}/store-address" } }, { diff --git a/modules/buildah/module.json b/modules/buildah/module.json index 70ddd0a4..bdfbe96b 100644 --- a/modules/buildah/module.json +++ b/modules/buildah/module.json @@ -8,7 +8,10 @@ "provides": [ { "name": "rootless-image-build", - "reach": "machine" + "reach": "machine", + "credential": { + "form": "none" + } } ], "resources": [ diff --git a/modules/disk-load/.gitignore b/modules/disk-load/.gitignore new file mode 100644 index 00000000..96984eae --- /dev/null +++ b/modules/disk-load/.gitignore @@ -0,0 +1 @@ +disk-load diff --git a/modules/disk-load/cmd/disk-load/keeper.go b/modules/disk-load/cmd/disk-load/keeper.go new file mode 100644 index 00000000..388ef136 --- /dev/null +++ b/modules/disk-load/cmd/disk-load/keeper.go @@ -0,0 +1,248 @@ +package main + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "git.novox.be/novox/mesh-sdk/go/catalogue" + "git.novox.be/novox/mesh-sdk/go/health" + "os" + "os/exec" + "sort" + "strings" + "sync" + "time" + + stdio "git.novox.be/novox/mesh-sdk/go" +) + +// Store is the state `readings` as the keeper uses it. +type Store interface { + Get(key string) (json.RawMessage, error) + Put(key string, value any) error + Keys() ([]string, error) +} + +// busStore is the state through the runtime. +type busStore struct{} + +func (busStore) Get(key string) (json.RawMessage, error) { + e, err := stdio.State(StateReadings).Get(key) + if err != nil || e == nil { + return nil, err + } + return e.Value, nil +} +func (busStore) Put(key string, v any) error { + _, err := stdio.State(StateReadings).Put(key, v) + return err +} +func (busStore) Keys() ([]string, error) { return stdio.State(StateReadings).Keys() } + +// Keeper measures the items of its node and keeps the readings. +type Keeper struct { + Node string + // Items is every data item the modules placed on this node declare, composed from the bus (frombus.DataItems). + Items func(ctx context.Context) (map[string][]catalogue.PlacedDataItem, error) + Measurer Measurer + Store Store + Say func(string, ...any) + + mu sync.Mutex // one round at a time +} + +// Round measures every item that is not a cache, one at a time, and writes the readings that changed. An item +// measured by a walk is walked only when walks is true or its last walk is older than WalkEvery: the hourly round +// reads counters, and a walk runs at most once a day (ADR 0233 §22). +func (k *Keeper) Round(ctx context.Context, walks bool) (written, kept int, err error) { + k.mu.Lock() + defer k.mu.Unlock() + items, err := k.Items(ctx) + if err != nil { + return 0, 0, err + } + modules := make([]string, 0, len(items)) + for m := range items { + modules = append(modules, m) + } + sort.Strings(modules) + var failed []string + for _, module := range modules { + for _, it := range items[module] { + if it.Class == ClassCache { + continue + } + w, err := k.measureOne(ctx, it, walks) + if err != nil { + failed = append(failed, fmt.Sprintf("%s.%s: %v", module, it.ID, err)) + continue + } + if w { + written++ + } else { + kept++ + } + } + } + retired, err := k.retireGone(items) + written += retired + if err != nil { + failed = append(failed, err.Error()) + } + if len(failed) > 0 { + return written, kept, errors.New("readings not kept: " + strings.Join(failed, "; ")) + } + return written, kept, nil +} + +// retireGone marks retired every reading of this node whose item the node no longer declares — its module was unassigned +// there, or is no longer pulled in — with when and why, and keeps it: the record of what the module left behind, until a +// person deletes it (ADR 0233). It is cleared when the item is declared again (measureOne). +func (k *Keeper) retireGone(declared map[string][]catalogue.PlacedDataItem) (int, error) { + keys, err := k.Store.Keys() + if err != nil { + return 0, err + } + declaredNow := map[string]bool{} + for module, its := range declared { + for _, it := range its { + declaredNow[health.ReadingKey(k.Node, module, it.ID)] = true + } + } + now := k.Measurer.Now().UTC() + n := 0 + for _, key := range keys { + if !strings.HasPrefix(key, k.Node+".") || declaredNow[key] { + continue + } + raw, err := k.Store.Get(key) + if err != nil || len(raw) == 0 { + continue + } + r, err := health.DecodeReading(raw) + if err != nil || r.Node != k.Node || !r.RetiredAt.IsZero() { + continue + } + r.RetiredAt = now + r.RetiredWhy = fmt.Sprintf("no longer declared on %s: its module was unassigned there, or is no longer pulled in", k.Node) + if err := k.Store.Put(key, r); err != nil { + return n, err + } + n++ + } + return n, nil +} + +func (k *Keeper) measureOne(ctx context.Context, it catalogue.PlacedDataItem, walks bool) (bool, error) { + key := health.ReadingKey(k.Node, it.Module, it.ID) + var prev *health.Reading + raw, err := k.Store.Get(key) + if err != nil { + return false, err + } + if len(raw) > 0 { + if p, err := health.DecodeReading(raw); err == nil { + prev = &p + } + } + now := k.Measurer.Now() + if prev != nil && prev.RetiredAt.IsZero() && it.Measure == catalogue.MeasureWalk && !walks && prev.MeasuredAt != nil && now.Sub(*prev.MeasuredAt) < WalkEvery { + return false, nil + } + m := k.Measurer.Measure(ctx, it) + next, changed := health.NextReading(prev, k.Node, it, m, now) + if prev != nil && !prev.RetiredAt.IsZero() { + // Declared again: it is no longer retired. + next.RetiredAt, next.RetiredWhy, changed = time.Time{}, "", true + } + if !changed { + return false, nil + } + return true, k.Store.Put(key, next) +} + +// MeasureNow measures one item now, walk or not, and writes its reading when it changed. +func (k *Keeper) MeasureNow(ctx context.Context, module, item string) (health.Reading, bool, error) { + k.mu.Lock() + defer k.mu.Unlock() + items, err := k.Items(ctx) + if err != nil { + return health.Reading{}, false, err + } + for _, it := range items[module] { + if it.ID != item { + continue + } + if it.Class == ClassCache { + return health.Reading{}, false, fmt.Errorf("%s.%s is a cache, which is listed and never measured", module, item) + } + w, err := k.measureOne(ctx, it, true) + if err != nil { + return health.Reading{}, false, err + } + raw, err := k.Store.Get(health.ReadingKey(k.Node, module, item)) + if err != nil { + return health.Reading{}, w, err + } + r, err := health.DecodeReading(raw) + return r, w, err + } + return health.Reading{}, false, fmt.Errorf("%s declares no data item %q on this machine", module, item) +} + +// Loop runs a round after a few minutes and then every `every`, with the walks of the night's round when it is the +// hour after the night; ctx ends it. +func (k *Keeper) Loop(ctx context.Context, every time.Duration, first time.Duration) { + select { + case <-ctx.Done(): + return + case <-time.After(first): + } + for { + w, kept, err := k.Round(ctx, false) + if err != nil { + k.Say("measuring the data declared here: %v", err) + } else { + k.Say("measured the data declared here: %d readings changed, %d unchanged", w, kept) + } + select { + case <-ctx.Done(): + return + case <-time.After(every): + } + } +} + +// RootRunner runs a command as root, through sudo without a prompt when this process is not root. A walk is bounded +// by WalkFor itself; the runner's own bound is a little longer. +func RootRunner(ctx context.Context, name string, args ...string) (string, error) { + ctx, cancel := context.WithTimeout(ctx, WalkFor+time.Minute) + defer cancel() + program, argv := name, args + if os.Getuid() != 0 { + program, argv = "sudo", append([]string{"-n", name}, args...) + } + var stdout, stderr bytes.Buffer + cmd := exec.CommandContext(ctx, program, argv...) + cmd.Stdout, cmd.Stderr = &stdout, &stderr + if err := cmd.Run(); err != nil { + said := strings.TrimSpace(stderr.String()) + if said == "" { + said = strings.TrimSpace(stdout.String()) + } + if program == "sudo" && strings.HasPrefix(said, "sudo:") { + return "", fmt.Errorf("%s needs root and the runtime's account may not run it without a prompt: %s", name, said) + } + lines := strings.Split(said, "\n") + if len(lines) > 3 { + lines = lines[len(lines)-3:] + } + if said == "" { + return "", fmt.Errorf("%s: %w", name, err) + } + return "", errors.New(strings.Join(lines, " / ")) + } + return stdout.String(), nil +} diff --git a/modules/disk-load/cmd/disk-load/main.go b/modules/disk-load/cmd/disk-load/main.go index 92fd2ef4..2eeff0b8 100644 --- a/modules/disk-load/cmd/disk-load/main.go +++ b/modules/disk-load/cmd/disk-load/main.go @@ -1,20 +1,61 @@ // The disk-load module's Go bundle (novox/hq ADR 0188, ADR 0193, issue 312): a process the node's // runtime launches and speaks MCP over stdio to. It reads the machine's storage load from /proc and // /sys — iowait, pressure stall, each disk's busy share, throughput, IOPS, wait and queue depth, the -// filesystems' space and the processes doing the I/O — and changes nothing. +// filesystems' space and the processes doing the I/O — and changes nothing of the machine. +// +// It also measures the data items the modules of its node declare (ADR 0233) and keeps each reading and the item's +// seven-day peak as its own state `readings` (ADR 0571 clause 3), composed from the bus (no file). package main import ( + "context" "fmt" "os" + "strings" + "time" stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/announce" + "git.novox.be/novox/mesh-sdk/go/catalogue" + "git.novox.be/novox/mesh-sdk/go/compose/frombus" + "git.novox.be/novox/mesh-sdk/go/compose/fromruntime" ) +func logf(format string, args ...any) { fmt.Fprintf(os.Stderr, "[disk-load] "+format+"\n", args...) } + func main() { + if err := stdio.Keeps(announce.State{Name: StateReadings}); err != nil { + logf("%v", err) + os.Exit(1) + } + node := os.Getenv("MESH_NODE") + k := &Keeper{Node: node, Store: busStore{}, Say: logf, Measurer: Measurer{Run: RootRunner, Now: time.Now}, + Items: func(ctx context.Context) (map[string][]catalogue.PlacedDataItem, error) { + return placedItems(ctx, node) + }} + if node != "" { + every := time.Hour + if d, err := time.ParseDuration(os.Getenv("MESH_DISK_MEASURE_EVERY")); err == nil && d >= time.Minute { + every = d + } + go k.Loop(context.Background(), every, 3*time.Minute) + } + tools := append(Tools(Here()), ReadingTools(k)...) // An empty name serves as the module the runtime names (MESH_SERVED_MODULE): disk-load. - if err := stdio.Serve("", Tools(Here())); err != nil { + if err := stdio.Serve("", tools); err != nil { fmt.Fprintln(os.Stderr, err) os.Exit(1) } } + +// placedItems is every data item the modules placed on node declare, composed from the bus as the node composes itself. +func placedItems(ctx context.Context, node string) (map[string][]catalogue.PlacedDataItem, error) { + items, unplaced, err := frombus.DataItems(ctx, fromruntime.Reader(), frombus.Mesh{}, frombus.Scope{Node: node}) + if err != nil { + return nil, err + } + if len(unplaced) > 0 { + logf("data items left out as not placeable: %s", strings.Join(unplaced, "; ")) + } + return items, nil +} diff --git a/modules/disk-load/cmd/disk-load/measure.go b/modules/disk-load/cmd/disk-load/measure.go new file mode 100644 index 00000000..2973080b --- /dev/null +++ b/modules/disk-load/cmd/disk-load/measure.go @@ -0,0 +1,215 @@ +package main + +// How an item is measured (novox/hq ADR 0233 §22 to §25), moved here from the backup holder, which only reads the +// readings now (ADR 0571 clause 3). Never an unbounded walk of something large. + +import ( + "context" + "fmt" + "git.novox.be/novox/mesh-sdk/go/catalogue" + "git.novox.be/novox/mesh-sdk/go/health" + "regexp" + "strconv" + "strings" + "time" +) + +// The bounds on a walk: a walk is for small items, and one that meets a large one stops and says so. +var ( + WalkFor = 10 * time.Minute + WalkFiles = 2_000_000 + WalkEvery = 23 * time.Hour +) + +// walkCommand sums the files under a path and finds its newest change, in one walk, bounded by time and by files. +// One line out: " ", then the status of find. +func walkCommand(path string) string { + return fmt.Sprintf("timeout %d find '%s' -xdev -printf '%%y %%s %%T@\\n' 2>/dev/null | head -n %d | "+ + "awk 'BEGIN{s=0;m=0;n=0} {n++; if ($1==\"f\") s+=$2; if ($3>m) m=$3} END {printf \"%%d %%.0f %%d\\n\", s, m, n}'; "+ + "echo \"${PIPESTATUS[0]:-0}\"", int(WalkFor.Seconds()), path, WalkFiles) +} + +// topCommand is the newest change among a path and its top-level entries, and how many there are. +func topCommand(path string) string { + return "timeout 60 find '" + path + "' -maxdepth 1 -printf '%T@\\n' 2>/dev/null | " + + "awk 'BEGIN{m=0;n=0} {n++; if ($1>m) m=$1} END {printf \"%.0f %d\\n\", m, n-1}'" +} + +func epochTime(s string) *time.Time { + epoch, err := strconv.ParseInt(s, 10, 64) + if err != nil || epoch <= 0 { + return nil + } + t := time.Unix(epoch, 0).UTC() + return &t +} + +// Measurer measures items through a runner that has the root the walks need. +type Measurer struct { + Run Runner + Now func() time.Time +} + +func (b Measurer) exists(ctx context.Context, path string) bool { + _, err := b.Run(ctx, "test", "-e", path) + return err == nil +} + +// Measure is one item, measured now, by its own method. +func (b Measurer) Measure(ctx context.Context, it catalogue.PlacedDataItem) health.Measured { + at := b.Now().UTC() + m := health.Measured{MeasuredAt: &at} + if !b.exists(ctx, it.Path) { + m.Error = it.Path + " does not exist" + zero := int64(0) + m.SizeBytes, m.Precision = &zero, "exact" + return m + } + m.Redundancy = b.redundancyOf(ctx, it.Path) + switch it.Measure { + case catalogue.MeasureDataset, catalogue.MeasureShallow: + out, err := b.Run(ctx, "bash", "-c", topCommand(it.Path)) + if f := strings.Fields(out); err == nil && len(f) == 2 { + m.LastWrite = epochTime(f[0]) + } else if err != nil { + m.Error = err.Error() + } + m.Precision = "no size: the newest change among its top-level entries only" + if it.Measure == catalogue.MeasureShallow { + return m + } + out, err = b.Run(ctx, "zfs", "list", "-Hp", "-o", "name,used", it.Path) + f := strings.Fields(out) + if err != nil || len(f) != 2 { + m.Precision = "no size: it is on no ZFS dataset to read one from; the newest change among its top-level entries only" + return m + } + used, err := strconv.ParseInt(f[1], 10, 64) + if err != nil { + return m + } + m.SizeBytes = &used + m.Precision = "dataset " + f[0] + ": its whole size, from the filesystem's counters; the newest change among the top-level entries" + return m + } + out, err := b.Run(ctx, "bash", "-c", walkCommand(it.Path)) + if err != nil { + m.Error = err.Error() + return m + } + lines := strings.Split(strings.TrimSpace(out), "\n") + f := strings.Fields(lines[0]) + if len(f) != 3 || len(lines) < 2 { + m.Error = "the measurement said " + strings.TrimSpace(out) + return m + } + size, err1 := strconv.ParseInt(f[0], 10, 64) + files, err2 := strconv.Atoi(f[2]) + if err1 != nil || err2 != nil { + m.Error = "the measurement said " + strings.TrimSpace(out) + return m + } + m.SizeBytes, m.LastWrite = &size, epochTime(f[1]) + m.Precision = "exact" + if status := strings.TrimSpace(lines[len(lines)-1]); status == "124" || files >= WalkFiles { + m.Precision = fmt.Sprintf("partial: measured partially — stopped after %s or %d files; the size is a lower bound, "+ + "and this item wants measure: dataset or shallow", WalkFor, WalkFiles) + } + return m +} + +// redundancyOf is the redundant storage a path is on: a ZFS pool's health, an md array's members, a btrfs +// filesystem's error counters. Nil for storage with no redundancy it knows how to read. +func (b Measurer) redundancyOf(ctx context.Context, path string) *health.Redundancy { + out, err := b.Run(ctx, "findmnt", "-no", "SOURCE,FSTYPE", "--target", path) + if err != nil { + return nil + } + f := strings.Fields(out) + if len(f) < 2 { + return nil + } + source, fstype := f[0], f[1] + yes, no := true, false + switch { + case fstype == "zfs": + pool, _, _ := strings.Cut(source, "/") + r := &health.Redundancy{Kind: "zfs", Where: pool} + health, err := b.Run(ctx, "zpool", "list", "-H", "-o", "health", pool) + if err != nil { + r.Said = "zpool list: " + err.Error() + return r + } + status, err := b.Run(ctx, "zpool", "status", "-x", pool) + if err != nil { + r.Said = "zpool status: " + err.Error() + return r + } + health, status = strings.TrimSpace(health), strings.TrimSpace(status) + r.Said = "health " + health + "; " + firstLineOf(status) + if health == "ONLINE" && strings.Contains(status, "is healthy") { + r.Healthy = &yes + } else { + r.Healthy = &no + r.Said = "health " + health + "; " + oneLineOf(status) + } + return r + case strings.HasPrefix(source, "/dev/md"): + device := strings.TrimPrefix(source, "/dev/") + r := &health.Redundancy{Kind: "md", Where: device} + mdstat, err := b.Run(ctx, "cat", "/proc/mdstat") + if err != nil { + r.Said = err.Error() + return r + } + healthy, said, found := mdHealth(mdstat, device) + if !found { + r.Said = device + " is not in /proc/mdstat" + return r + } + r.Said = said + if healthy { + r.Healthy = &yes + } else { + r.Healthy = &no + } + return r + case fstype == "btrfs": + r := &health.Redundancy{Kind: "btrfs", Where: source} + stats, err := b.Run(ctx, "btrfs", "device", "stats", "--check", path) + if err != nil { + r.Healthy, r.Said = &no, "device errors: "+oneLineOf(err.Error()) + return r + } + r.Healthy, r.Said = &yes, firstLineOf(stats) + return r + } + return nil +} + +var mdMembers = regexp.MustCompile(`\[([U_]+)\]`) + +// mdHealth reads one array's block of /proc/mdstat: healthy when every member is up and it is not recovering. +func mdHealth(mdstat, device string) (bool, string, bool) { + lines := strings.Split(mdstat, "\n") + for i, l := range lines { + if !strings.HasPrefix(l, device+" ") { + continue + } + block := l + for j := i + 1; j < len(lines) && strings.HasPrefix(lines[j], " "); j++ { + block += " " + strings.TrimSpace(lines[j]) + } + members := mdMembers.FindStringSubmatch(block) + healthy := members != nil && !strings.Contains(members[1], "_") && !strings.Contains(block, "recovery") + return healthy, oneLineOf(block), true + } + return false, "", false +} + +func firstLineOf(s string) string { + line, _, _ := strings.Cut(strings.TrimSpace(s), "\n") + return line +} + +func oneLineOf(s string) string { return strings.Join(strings.Fields(s), " ") } diff --git a/modules/disk-load/cmd/disk-load/readings.go b/modules/disk-load/cmd/disk-load/readings.go new file mode 100644 index 00000000..3cad1734 --- /dev/null +++ b/modules/disk-load/cmd/disk-load/readings.go @@ -0,0 +1,12 @@ +package main + +// The readings layout, their key and the peak are mesh-sdk's (health.Reading, health.NextReading; novox/hq ADR 0571 +// clause 3): disk-load measures each placed data item of its node and keeps the readings as its own state. + +import "git.novox.be/novox/mesh-sdk/go/health" + +// StateReadings is the state disk-load keeps. +const StateReadings = health.ReadingsState + +// ClassCache is the class of an item that is listed and never measured. +const ClassCache = "cache" diff --git a/modules/disk-load/cmd/disk-load/readings_test.go b/modules/disk-load/cmd/disk-load/readings_test.go new file mode 100644 index 00000000..d934ad27 --- /dev/null +++ b/modules/disk-load/cmd/disk-load/readings_test.go @@ -0,0 +1,144 @@ +package main + +import ( + "context" + "encoding/json" + "os" + "strings" + "testing" + "time" + + "git.novox.be/novox/mesh-sdk/go/catalogue" + "git.novox.be/novox/mesh-sdk/go/health" +) + +type memStore struct{ m map[string]json.RawMessage } + +func (s *memStore) Get(k string) (json.RawMessage, error) { return s.m[k], nil } +func (s *memStore) Put(k string, v any) error { + raw, _ := json.Marshal(v) + s.m[k] = raw + return nil +} +func (s *memStore) Keys() ([]string, error) { + var out []string + for k := range s.m { + out = append(out, k) + } + return out, nil +} + +// A round measures what the data file declares but caches, keeps the readings, writes nothing the second time, and a +// walk is not repeated within a day unless asked. +func TestARoundMeasuresTheDeclaredItemsAndWritesWhatChanged(t *testing.T) { + text := "# pg\nitem data valuable /srv/pg /srv/pg backup walk\nitem tmp cache /srv/tmp - none\n# bad-line-module\nitem logs rebuildable /srv/logs - none shallow\n" + placed, err := catalogue.ParseDataItems(text) + if err != nil { + t.Fatal(err) + } + walks := 0 + run := func(_ context.Context, name string, args ...string) (string, error) { + switch { + case name == "test": + return "", nil + case name == "findmnt": + return "", os.ErrNotExist + case name == "bash" && strings.Contains(args[1], "-maxdepth 1"): + return "1760000000 2\n", nil + case name == "bash": + walks++ + return "4096 1760000000 3\n0\n", nil + } + return "", os.ErrNotExist + } + now := time.Date(2026, 10, 29, 12, 0, 0, 0, time.UTC) + store := &memStore{m: map[string]json.RawMessage{}} + k := &Keeper{Node: "ace", Items: func(context.Context) (map[string][]catalogue.PlacedDataItem, error) { return placed, nil }, Store: store, Say: func(string, ...any) {}, + Measurer: Measurer{Run: run, Now: func() time.Time { return now }}} + w, kept, err := k.Round(context.Background(), false) + if err != nil || w != 2 || kept != 0 || len(store.m) != 2 { + t.Fatalf("first round: %d written %d kept %v, keys %d", w, kept, err, len(store.m)) + } + now = now.Add(time.Hour) + if w, kept, _ := k.Round(context.Background(), false); w != 0 || kept != 2 || walks != 1 { + t.Fatalf("second round: %d written, %d kept, %d walks; wanted none written and the walk not repeated", w, kept, walks) + } + if _, _, err := k.MeasureNow(context.Background(), "pg", "data"); err != nil || walks != 2 { + t.Fatalf("measure now: %v, %d walks", err, walks) + } + // An item no longer declared is kept, retired with when and why; declared again, it is not retired any more. + delete(placed, "pg") + if _, _, err := k.Round(context.Background(), false); err != nil { + t.Fatal(err) + } + gone, _ := health.DecodeReading(store.m["ace.pg.data"]) + if gone.RetiredAt.IsZero() || !strings.Contains(gone.RetiredWhy, "no longer declared on ace") || gone.FirstSeen.IsZero() { + t.Fatalf("a reading of an item no longer declared: %+v", gone) + } + placed["pg"] = []catalogue.PlacedDataItem{{Module: "pg", ID: "data", Class: "valuable", Path: "/srv/pg", Measure: catalogue.MeasureWalk}} + if _, _, err := k.Round(context.Background(), false); err != nil { + t.Fatal(err) + } + if back, _ := health.DecodeReading(store.m["ace.pg.data"]); !back.RetiredAt.IsZero() { + t.Fatalf("a reading declared again is still retired: %+v", back) + } + if _, _, err := k.MeasureNow(context.Background(), "pg", "tmp"); err == nil { + t.Fatal("a cache was measured") + } +} + +// The redundant storage under an item is read: a ZFS pool healthy, then degraded; an md array with a member missing; and +// plain storage, which is no redundancy at all. A dataset is measured from the filesystem's counters and a shallow item +// from its top-level entries only; a walk that meets more than its bound stops and says so. +func TestTheStorageUnderAnItemAndTheLargeMethodsAreRead(t *testing.T) { + poolHealth, statusX, fs := "ONLINE\n", "pool 'storage' is healthy\n", "storage/media zfs\n" + run := func(_ context.Context, name string, args ...string) (string, error) { + line := name + " " + strings.Join(args, " ") + switch { + case name == "test": + return "", nil + case name == "findmnt": + return fs, nil + case name == "zpool" && args[0] == "list": + return poolHealth, nil + case name == "zpool": + return statusX, nil + case name == "cat": + return "Personalities : [raid1]\nmd127 : active raid1 sdb1[1] sda1[0]\n 976630464 blocks super 1.2 [2/1] [U_]\n\nunused devices: \n", nil + case name == "zfs" && args[len(args)-1] == "/storage/media/movies": + return "storage/media\t97067690722560\n", nil + case name == "zfs": + return "", os.ErrNotExist + case name == "bash" && strings.Contains(line, "-maxdepth 1"): + return "1759744800 12673\n", nil + case name == "bash": + return "123 1759700000 2000000\n141\n", nil + } + return "", nil + } + m := Measurer{Run: run, Now: time.Now} + ctx := context.Background() + got := m.Measure(ctx, catalogue.PlacedDataItem{Module: "media", ID: "movies", Path: "/storage/media/movies", Measure: catalogue.MeasureDataset}) + if got.SizeBytes == nil || *got.SizeBytes != 97067690722560 || !strings.HasPrefix(got.Precision, "dataset storage/media") || + got.LastWrite == nil || got.Redundancy == nil || got.Redundancy.Kind != "zfs" || got.Redundancy.Healthy == nil || !*got.Redundancy.Healthy { + t.Fatalf("the library from its dataset: %+v", got) + } + poolHealth, statusX = "DEGRADED\n", " pool: storage\n state: DEGRADED\nstatus: One or more devices has been removed\n" + if r := m.redundancyOf(ctx, "/storage/media/movies"); r.Healthy == nil || *r.Healthy || !strings.Contains(r.Said, "DEGRADED") { + t.Fatalf("a degraded pool: %+v", r) + } + fs = "/dev/md127 ext4\n" + if r := m.redundancyOf(ctx, "/srv/x"); r == nil || r.Kind != "md" || r.Healthy == nil || *r.Healthy { + t.Fatalf("an array missing a member: %+v", r) + } + fs = "/dev/nvme0n1p3 ext4\n" + if r := m.redundancyOf(ctx, "/var/lib/x"); r != nil { + t.Fatalf("plain storage read as redundant: %+v", r) + } + if s := m.Measure(ctx, catalogue.PlacedDataItem{Path: "/mnt/plex/config", Measure: catalogue.MeasureShallow}); s.SizeBytes != nil || !strings.HasPrefix(s.Precision, "no size") { + t.Fatalf("a shallow item: %+v", s) + } + if w := m.Measure(ctx, catalogue.PlacedDataItem{Path: "/srv/big", Measure: catalogue.MeasureWalk}); !strings.HasPrefix(w.Precision, "partial") { + t.Fatalf("a walk stopped at its bound: %+v", w) + } +} diff --git a/modules/disk-load/cmd/disk-load/readings_tools.go b/modules/disk-load/cmd/disk-load/readings_tools.go new file mode 100644 index 00000000..322fe275 --- /dev/null +++ b/modules/disk-load/cmd/disk-load/readings_tools.go @@ -0,0 +1,93 @@ +package main + +import ( + "context" + "fmt" + "sort" + "strings" + + stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/health" +) + +// ReadingTools are the tools over the state `readings`: every key, one key, how full the state is, and one item +// measured now (a module's tools are its command line). +func ReadingTools(k *Keeper) []stdio.Tool { + return []stdio.Tool{ + { + Name: "disk_readings", + Description: "The data readings disk-load keeps for this machine's data items (ADR 0233, ADR 0571 clause 3): every key " + + "`..` with its size, newest write, measured-at, precision and seven-day peak, or one key with " + + "all it holds when `key` is given; and how full the state is: keys kept, their bytes, and the cap the bus puts on " + + "a state (none declared). Filter by `module`. (r)", + Input: map[string]any{ + "key": map[string]any{"type": "string", "description": "one key, .., with everything it holds"}, + "module": map[string]any{"type": "string", "description": "only this module's items"}, + }, + Run: func(args map[string]any) (any, error) { + if key := text(args, "key"); key != "" { + raw, err := k.Store.Get(key) + if err != nil { + return nil, err + } + if len(raw) == 0 { + return nil, fmt.Errorf("no reading is kept at %q", key) + } + r, err := health.DecodeReading(raw) + return r, err + } + keys, err := k.Store.Keys() + if err != nil { + return nil, err + } + sort.Strings(keys) + module := text(args, "module") + type line struct { + Key string `json:"key"` + SizeBytes *int64 `json:"size_bytes,omitempty"` + PeakBytes *int64 `json:"peak_bytes,omitempty"` + Precision string `json:"precision,omitempty"` + Error string `json:"error,omitempty"` + Reading health.Reading `json:"-"` + } + var lines []line + total := 0 + for _, key := range keys { + raw, err := k.Store.Get(key) + if err != nil { + return nil, err + } + total += len(raw) + r, err := health.DecodeReading(raw) + if err != nil || (module != "" && r.Module != module) { + continue + } + lines = append(lines, line{Key: key, SizeBytes: r.SizeBytes, PeakBytes: r.Peak, Precision: r.Precision, Error: r.Error}) + } + return map[string]any{"readings": lines, "fill": map[string]any{"keys": len(keys), "bytes": total, + "cap": "none declared: the bus's own limit on a state applies"}}, nil + }, + }, + { + Name: "disk_measure", + Description: "Measure one data item of this machine now, whatever its method (a walk too), and write its reading when it " + + "changed. Reads the disks as root: a walk is bounded by time and files and says when it was partial.", + Input: map[string]any{ + "module": map[string]any{"type": "string", "description": "the module that declares the item"}, + "item": map[string]any{"type": "string", "description": "the item's id"}, + }, + Run: func(args map[string]any) (any, error) { + module, item := text(args, "module"), text(args, "item") + if module == "" || item == "" { + return nil, fmt.Errorf("module and item are required") + } + r, written, err := k.MeasureNow(context.Background(), module, item) + if err != nil { + return nil, err + } + return map[string]any{"written": written, "reading": r, "key": health.ReadingKey(k.Node, module, item), + "note": strings.TrimSpace(map[bool]string{true: "the reading changed and was written", false: "the reading had not changed"}[written])}, nil + }, + }, + } +} diff --git a/modules/disk-load/cmd/disk-load/tools_test.go b/modules/disk-load/cmd/disk-load/tools_test.go index 88147715..672e6e94 100644 --- a/modules/disk-load/cmd/disk-load/tools_test.go +++ b/modules/disk-load/cmd/disk-load/tools_test.go @@ -3,6 +3,7 @@ package main import ( "encoding/json" "os" + "regexp" "sort" "strings" "testing" @@ -35,9 +36,11 @@ func TestTheManifestNamesExactlyTheToolsTheBundleServesAndNoMachine(t *testing.T t.Fatal(err) } var served []string - for _, tl := range Tools(newFake(t).machine()) { + k := &Keeper{Store: &memStore{m: map[string]json.RawMessage{}}} + for _, tl := range append(Tools(newFake(t).machine()), ReadingTools(k)...) { served = append(served, tl.Name) - if !strings.HasSuffix(tl.Description, "(r)") { + // disk_measure writes the module's own state of readings and nothing else of the machine. + if !strings.HasSuffix(tl.Description, "(r)") && tl.Name != "disk_measure" { t.Errorf("%s only reads and does not say so", tl.Name) } } @@ -50,7 +53,7 @@ func TestTheManifestNamesExactlyTheToolsTheBundleServesAndNoMachine(t *testing.T t.Errorf("the module only reads, and declares %d resources", len(m.Resources)) } for _, banned := range []string{"g14", "novox", "ace", "shanks", "jochen", "/home/"} { - if strings.Contains(string(raw), banned) { + if regexp.MustCompile(`(^|[^a-z0-9-])` + regexp.QuoteMeta(banned) + `($|[^a-z0-9-])`).Match(raw) { t.Errorf("the manifest says %q", banned) } } diff --git a/modules/disk-load/go.mod b/modules/disk-load/go.mod index edf8078e..c3b85927 100644 --- a/modules/disk-load/go.mod +++ b/modules/disk-load/go.mod @@ -1,5 +1,5 @@ module diskload -go 1.22 +go 1.26.0 -require git.novox.be/novox/mesh-sdk/go v0.1.6 +require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 diff --git a/modules/disk-load/go.sum b/modules/disk-load/go.sum index 0dd60617..09ad064e 100644 --- a/modules/disk-load/go.sum +++ b/modules/disk-load/go.sum @@ -1,2 +1,14 @@ -git.novox.be/novox/mesh-sdk/go v0.1.6 h1:9qzdYONYbJdWcu6sxQcq9v1LI0JxcfkiKYkMUzJSkVQ= -git.novox.be/novox/mesh-sdk/go v0.1.6/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 h1:lKU3HjNXGiVbLW2hUgFjU/TMTUYMlNnnmZxG5T7biQA= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA= +github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI= +github.com/nats-io/nats.go v1.54.0 h1:vsXoOxjHp/GmPUN+EcI7uOf/uB+iAP+kEsAFNQN0yzA= +github.com/nats-io/nats.go v1.54.0/go.mod h1:y+DZoD1oBOYfZTU681eTUiUjI0vbqYGixNVFHcjHJ0k= +github.com/nats-io/nkeys v0.4.16 h1:rd5oAuLOb8mnAycB0xleuEBNS1pVVnN0fv/FF34Eypg= +github.com/nats-io/nkeys v0.4.16/go.mod h1:llLgWoI0o4z/Q57q2R1kHfmocyhGV6VG/U18Glg1Afs= +github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= +github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= +golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M= +golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA= +golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo= +golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= diff --git a/modules/disk-load/module.json b/modules/disk-load/module.json index 9b48041d..4b4313ca 100644 --- a/modules/disk-load/module.json +++ b/modules/disk-load/module.json @@ -7,7 +7,9 @@ "disk_filesystems", "disk_folders", "disk_devices", - "cpu_top" + "cpu_top", + "disk_readings", + "disk_measure" ], "build": { "artifacts": [ @@ -23,5 +25,16 @@ ] } ] - } + }, + "state": [ + "readings" + ], + "reads": [ + "mesh-config.resolved", + "mesh-delivery-assigner.placement", + "mesh-delivery-builder.builds", + "mesh-delivery-indexer.manifests", + "mesh-delivery-indexer.modules", + "node-engine.have" + ] } diff --git a/modules/distribution/module.json b/modules/distribution/module.json index eb9f7c24..5037f429 100644 --- a/modules/distribution/module.json +++ b/modules/distribution/module.json @@ -4,7 +4,10 @@ "provides": [ { "name": "artifact-store", - "scope": "mesh" + "scope": "mesh", + "credential": { + "form": "none" + } } ], "claims": [ diff --git a/modules/dnsmasq/go.mod b/modules/dnsmasq/go.mod index d439da63..e0d8d944 100644 --- a/modules/dnsmasq/go.mod +++ b/modules/dnsmasq/go.mod @@ -1,5 +1,5 @@ module dnsmasq -go 1.25.0 +go 1.26.0 -require git.novox.be/novox/mesh-sdk/go v0.1.7 +require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 diff --git a/modules/dnsmasq/go.sum b/modules/dnsmasq/go.sum index b474419e..f8bfbd2b 100644 --- a/modules/dnsmasq/go.sum +++ b/modules/dnsmasq/go.sum @@ -1,2 +1,2 @@ -git.novox.be/novox/mesh-sdk/go v0.1.7 h1:C0sTQmtTiyYH7bnqZb7PusXnqA37gKuT7Nqjn9gG47w= -git.novox.be/novox/mesh-sdk/go v0.1.7/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 h1:lKU3HjNXGiVbLW2hUgFjU/TMTUYMlNnnmZxG5T7biQA= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= diff --git a/modules/dnsmasq/module.json b/modules/dnsmasq/module.json index 58cc0da5..7e832067 100644 --- a/modules/dnsmasq/module.json +++ b/modules/dnsmasq/module.json @@ -9,7 +9,10 @@ { "name": "wildcard-resolution", "scope": "mesh", - "identity": false + "identity": false, + "credential": { + "form": "none" + } } ], "requires": [ diff --git a/modules/docker/cmd/docker-tools/prune_images.go b/modules/docker/cmd/docker-tools/prune_images.go index 49f75436..e15f7fdf 100644 --- a/modules/docker/cmd/docker-tools/prune_images.go +++ b/modules/docker/cmd/docker-tools/prune_images.go @@ -9,7 +9,7 @@ package main // // - every image a container on this machine uses, in any state; // - every image this machine's declaration names — what the mesh would send it now and what it was -// last sent — as the controller's `images` verb answers. **No answer, no removal**: what the +// last sent — as the mesh-artifact-sweep seat's `images` verb answers. **No answer, no removal**: what the // declaration names is the one thing that cannot be guessed; // - every image younger than the floor (seven days, the weekly prune's week); // - in each line of images holding one of those, the newest image besides them: the previous @@ -43,7 +43,7 @@ type PruneImagesAsk struct { Limit int } -// declaredImages is the controller's `images` answer (novox/hq ADR 0251 §5). +// declaredImages is the mesh-artifact-sweep seat's `images` answer (novox/hq ADR 0251 §5). type declaredImages struct { Node string `json:"node"` Images []struct { @@ -129,7 +129,7 @@ func digestOf(ref string) string { return "" } -// askDeclared asks the controller what this machine's declarations name. An error means unknown. +// askDeclared asks the mesh-artifact-sweep seat what this machine's declarations name. An error means unknown. func (c *Client) askDeclared() (*declaredImages, error) { if c.Node == "" { return nil, errors.New("the runtime did not say which machine this is (MESH_NODE is empty)") @@ -137,20 +137,20 @@ func (c *Client) askDeclared() (*declaredImages, error) { if c.Ask == nil { return nil, errors.New("this bundle cannot ask the mesh") } - raw, err := c.Ask("seat:mesh-controller.images", map[string]any{"node": c.Node}) + raw, err := c.Ask("seat:mesh-artifact-sweep.images", map[string]any{"node": c.Node}) if err != nil { - return nil, fmt.Errorf("the controller's images verb did not answer: %w", err) + return nil, fmt.Errorf("the mesh-artifact-sweep seat's images verb did not answer: %w", err) } answer, output, ok := answerOf(raw) if !ok { - return nil, fmt.Errorf("the controller refused images: %s", firstLine(output)) + return nil, fmt.Errorf("the mesh-artifact-sweep seat refused images: %s", firstLine(output)) } var d declaredImages if err := json.Unmarshal(answer, &d); err != nil { - return nil, fmt.Errorf("the controller's images answer is not readable: %v", err) + return nil, fmt.Errorf("the mesh-artifact-sweep seat's images answer is not readable: %v", err) } if d.Node != "" && d.Node != c.Node { - return nil, fmt.Errorf("the controller answered for %q, not for this machine %q", d.Node, c.Node) + return nil, fmt.Errorf("the mesh-artifact-sweep seat answered for %q, not for this machine %q", d.Node, c.Node) } return &d, nil } diff --git a/modules/docker/cmd/docker-tools/prune_images_test.go b/modules/docker/cmd/docker-tools/prune_images_test.go index 62a5d6c7..df019fed 100644 --- a/modules/docker/cmd/docker-tools/prune_images_test.go +++ b/modules/docker/cmd/docker-tools/prune_images_test.go @@ -43,7 +43,7 @@ func imagesMachine() *fake { func declaring(t *testing.T, answer string) Asker { return func(key string, body any) (json.RawMessage, error) { - if key != "seat:mesh-controller.images" { + if key != "seat:mesh-artifact-sweep.images" { t.Errorf("asked %s", key) } if b, _ := body.(map[string]any); b["node"] != "laptop" { diff --git a/modules/docker/go.mod b/modules/docker/go.mod index 5faa536e..fe26ae06 100644 --- a/modules/docker/go.mod +++ b/modules/docker/go.mod @@ -1,5 +1,5 @@ module docker -go 1.22 +go 1.26.0 -require git.novox.be/novox/mesh-sdk/go v0.1.6 +require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 diff --git a/modules/docker/go.sum b/modules/docker/go.sum index 0dd60617..f8bfbd2b 100644 --- a/modules/docker/go.sum +++ b/modules/docker/go.sum @@ -1,2 +1,2 @@ -git.novox.be/novox/mesh-sdk/go v0.1.6 h1:9qzdYONYbJdWcu6sxQcq9v1LI0JxcfkiKYkMUzJSkVQ= -git.novox.be/novox/mesh-sdk/go v0.1.6/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 h1:lKU3HjNXGiVbLW2hUgFjU/TMTUYMlNnnmZxG5T7biQA= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= diff --git a/modules/docker/module.json b/modules/docker/module.json index fae954c3..500faea6 100644 --- a/modules/docker/module.json +++ b/modules/docker/module.json @@ -7,7 +7,7 @@ "privileged" ], "invokes": [ - "seat:mesh-controller.images" + "seat:mesh-artifact-sweep.images" ], "claims": [ { diff --git a/modules/gitea/cmd/forge-account-provider/builder_test.go b/modules/gitea/cmd/forge-account-provider/builder_test.go index f325acd7..8897013f 100644 --- a/modules/gitea/cmd/forge-account-provider/builder_test.go +++ b/modules/gitea/cmd/forge-account-provider/builder_test.go @@ -192,17 +192,14 @@ func (r *teamRig) pass() []provider.Outcome { return out } -// bind makes module on node bind gitea's forge-account in the assignments mirror. +// bind makes module on node bind gitea's forge-account in the seat mesh-provisioner's plans. func (r *teamRig) bind(node, module string) { - r.bus.set(provider.AssignmentsBucket, node+"."+module, []byte(fmt.Sprintf( - `{"node":%q,"module":%q,"provisioning":{"provisions":[{"provision":"forge-account","provider":"novox","provider-module":"gitea","as":"mesh_%s"}]}}`, - node, module, strings.ReplaceAll(node+"_"+module, "-", "_")))) + r.bus.bindPlan(node, module, "mesh_"+strings.ReplaceAll(node+"_"+module, "-", "_")) } +// unbind is the operator's yes to retiring module's forge-account on node (ADR 0570 clause 5). func (r *teamRig) unbind(node, module string) { - r.bus.mu.Lock() - delete(r.bus.buckets[provider.AssignmentsBucket], node+"."+module) - r.bus.mu.Unlock() + r.bus.retire(node, module) } // key reports a key in module's state keys, of the kind and for the account given. diff --git a/modules/gitea/cmd/forge-account-provider/bus_test.go b/modules/gitea/cmd/forge-account-provider/bus_test.go index 996a1816..7cacfe5b 100644 --- a/modules/gitea/cmd/forge-account-provider/bus_test.go +++ b/modules/gitea/cmd/forge-account-provider/bus_test.go @@ -30,6 +30,7 @@ import ( "github.com/nats-io/nats.go/jetstream" "git.novox.be/novox/mesh-sdk/go/provider" + "git.novox.be/novox/mesh-sdk/go/provisions" ) const bundleVar = "MESH_FORGE_ACCOUNT_TEST_BUNDLE" @@ -80,7 +81,7 @@ func startBus(t *testing.T) (*runtime, context.CancelFunc) { rt := &runtime{t: t, js: js, kvs: map[string]jetstream.KeyValue{}, ctx: ctx, wait: map[string]chan json.RawMessage{}} // The controller's job: a bucket per state the module declares (bound) or reads (the others). for state, bucket := range map[string]string{"bound": "gitea_bound", "ssh-client.keys": "ssh-client_keys", - "mesh-controller.assignments": "mesh-controller_assignments"} { + "mesh-provisioner.plans": "mesh-provisioner_plans", "mesh-provisioner.provisions": "mesh-provisioner_provisions"} { kv, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: bucket, History: 1}) if err != nil { t.Fatal(err) @@ -296,9 +297,11 @@ func TestTheProviderWholeAnswersAMachineThatReportedItsKey(t *testing.T) { } rt, cancel := startBus(t) t.Cleanup(cancel) - rt.put("mesh-controller.assignments", "g14.ssh-client", map[string]any{"node": "g14", "module": "ssh-client", - "provisioning": map[string]any{"provisions": []map[string]any{{"provision": "forge-account", "provider": "novox", + rt.put("mesh-provisioner.plans", "g14.ssh-client", map[string]any{"v": 1, "node": "g14", "module": "ssh-client", + "plan": map[string]any{"provisions": []map[string]any{{"provision": "forge-account", "provider": "novox", "provider-module": "gitea", "as": "mesh_g14_ssh_client"}}}}) + rt.put("mesh-provisioner.provisions", "g14.ssh-client.forge-account", + json.RawMessage(provisionKey("g14", "ssh-client", provisions.Asked, nil))) rt.put("ssh-client.keys", "g14.nox", map[string]string{"node": "g14", "account": "nox", "kind": "machine", "public": pubA + " nox@g14 (mesh)", "fingerprint": "SHA256:x", "at": "2026-10-16T10:00:00Z"}) rt.start([]string{"MESH_NODE=novox", "MESH_MODULE=gitea", "MESH_SERVED_MODULE=gitea", diff --git a/modules/gitea/cmd/forge-account-provider/main_test.go b/modules/gitea/cmd/forge-account-provider/main_test.go index 024b5f6d..12583c16 100644 --- a/modules/gitea/cmd/forge-account-provider/main_test.go +++ b/modules/gitea/cmd/forge-account-provider/main_test.go @@ -25,7 +25,7 @@ func TestConfigFromEnvNamesWhatIsMissing(t *testing.T) { } // Issue 955: the runtime grants a module only the states its manifest `reads`, and the provider watches the machines' -// keys and, through the SDK's provider, the controller's assignments (where it learns its consumers). Run as the +// keys and, through the SDK's provider, the seat mesh-provisioner's plans and provisions (where it learns its consumers, ADR 0570). Run as the // bus test does, with every bucket made, a missing grant passes unseen; the manifest is read here instead. func TestTheManifestReadsEveryStateTheProviderWatches(t *testing.T) { raw, err := os.ReadFile("../../module.json") @@ -38,7 +38,7 @@ func TestTheManifestReadsEveryStateTheProviderWatches(t *testing.T) { if err := json.Unmarshal(raw, &m); err != nil { t.Fatal(err) } - for _, state := range []string{"mesh-controller.assignments", "ssh-client.keys"} { + for _, state := range []string{"mesh-provisioner.plans", "mesh-provisioner.provisions", "ssh-client.keys"} { if !slices.Contains(m.Reads, state) { t.Errorf("gitea's reads %v lack %q, so the runtime refuses its watch", m.Reads, state) } diff --git a/modules/gitea/cmd/forge-account-provider/service.go b/modules/gitea/cmd/forge-account-provider/service.go index 2e9716fb..0b2ad113 100644 --- a/modules/gitea/cmd/forge-account-provider/service.go +++ b/modules/gitea/cmd/forge-account-provider/service.go @@ -184,7 +184,9 @@ func (s *Service) Make(ctx context.Context, c provider.Consumer) error { s.mu.Unlock() if !loaded { // After a restart the first pass can come before the keys arrive: nothing is removed or deleted on that. - return fmt.Errorf("the keys of %s are not read yet, so nothing is made, removed or deleted", source) + // Not ready, not failed: the entry waits and keeps serving what it served (provider.ErrNotReady). + return fmt.Errorf("the keys of %s are not read yet, so nothing is made, removed or deleted: %w", source, + provider.ErrNotReady) } have := s.keysFrom(source, node, kind) if len(have) == 0 { diff --git a/modules/gitea/cmd/forge-account-provider/service_test.go b/modules/gitea/cmd/forge-account-provider/service_test.go index 7f47709a..60242c91 100644 --- a/modules/gitea/cmd/forge-account-provider/service_test.go +++ b/modules/gitea/cmd/forge-account-provider/service_test.go @@ -17,6 +17,7 @@ import ( "time" "git.novox.be/novox/mesh-sdk/go/provider" + "git.novox.be/novox/mesh-sdk/go/provisions" "gitea/internal/forgehttp" ) @@ -78,6 +79,22 @@ func (b *fakeBus) entry(t *testing.T, key string) provider.Entry { } return e } + +// serves is whether gitea's entry under key serves its consumer: there, in state (provided when none is given) and with +// values (ADR 0570 clause 3: an entry the provider could not make is kept, failed, serving nothing; one it is not +// ready to make waits and keeps serving). +func (b *fakeBus) serves(key string, state ...string) bool { + want := provisions.Provided + if len(state) > 0 { + want = state[0] + } + b.mu.Lock() + raw, ok := b.buckets[provider.BucketOf("gitea")][key] + b.mu.Unlock() + var e provider.Entry + return ok && json.Unmarshal(raw, &e) == nil && e.State == want && len(e.Values) > 0 +} + func (b *fakeBus) has(key string) bool { b.mu.Lock() defer b.mu.Unlock() @@ -85,11 +102,41 @@ func (b *fakeBus) has(key string) bool { return ok } -// bindClient makes ssh-client on node bind gitea's forge-account in the assignments mirror. +// bindClient makes ssh-client on node bind gitea's forge-account in the seat mesh-provisioner's plans. func (b *fakeBus) bindClient(node string) { - b.set(provider.AssignmentsBucket, node+".ssh-client", []byte(fmt.Sprintf( - `{"node":%q,"module":"ssh-client","provisioning":{"provisions":[{"provision":"forge-account","provider":"novox","provider-module":"gitea","as":"mesh_%s_ssh_client"}]}}`, - node, node))) + b.bindPlan(node, "ssh-client", "mesh_"+node+"_ssh_client") +} + +// bindPlan makes module on node bind gitea's forge-account as the seat mesh-provisioner keeps it: its plan, and the +// provision key asked of gitea on novox (novox/hq ADR 0570 clauses 1 to 3). +func (b *fakeBus) bindPlan(node, module, as string) { + b.set(provisions.PlansBucket, node+"."+module, []byte(fmt.Sprintf( + `{"v":1,"node":%q,"module":%q,"plan":{"provisions":[{"provision":"forge-account","provider":"novox","provider-module":"gitea","as":%q}]}}`, + node, module, as))) + b.set(provisions.ProvisionsBucket, node+"."+module+".forge-account", provisionKey(node, module, provisions.Asked, nil)) +} + +// retire is the operator's yes to the retiring of module's forge-account on node: its plan gone, its key retiring +// with the warrant (ADR 0570 clause 5), the only way the provider lets an entry go. +func (b *fakeBus) retire(node, module string) { + b.mu.Lock() + delete(b.buckets[provisions.PlansBucket], node+"."+module) + b.mu.Unlock() + b.set(provisions.ProvisionsBucket, node+"."+module+".forge-account", provisionKey(node, module, provisions.Retiring, + &provisions.Warrant{Ask: "ask-1", Option: "yes", By: "operator", At: time.Date(2026, 10, 29, 12, 0, 0, 0, time.UTC)})) +} + +func provisionKey(node, module, state string, w *provisions.Warrant) []byte { + pr := provisions.Provision{Provision: "forge-account", State: state, Warrant: w, + Consumer: provisions.End{Node: node, Module: module}, Provider: provisions.End{Node: "novox", Module: "gitea"}} + if state == provisions.Retiring { + pr.Why = "no want needs it" + } + raw, err := provisions.Encode(pr) + if err != nil { + panic(err) + } + return raw } // fakeForge is gitea's API as far as the provider uses it. @@ -343,9 +390,7 @@ func TestAConsumerThatNoLongerBindsLosesItsMeshKeysAndKeepsItsAccount(t *testing r.forge.mu.Lock() r.forge.held["nox-agent"] = append(r.forge.held["nox-agent"], map[string]any{"id": int64(99), "title": "my own", "key": pubB}) r.forge.mu.Unlock() - r.bus.mu.Lock() - delete(r.bus.buckets[provider.AssignmentsBucket], "g14.ssh-client") - r.bus.mu.Unlock() + r.bus.retire("g14", "ssh-client") r.pass() if r.bus.has("g14.ssh-client.forge-account") { t.Fatal("the entry stayed") @@ -460,7 +505,7 @@ func TestADeletedKeyEntryRemovesTheMeshKeyAndTheBoundEntryButNotTheOperatorsKey( if got := r.forge.titles("nox-agent"); len(got) != 1 || got[0] != "my own" { t.Fatalf("keys = %v", got) } - if r.bus.has("g14.ssh-client.forge-account") { + if r.bus.serves("g14.ssh-client.forge-account") { t.Fatal("the entry still serves a machine whose key is gone") } } @@ -482,8 +527,8 @@ func TestAPassBeforeTheKeysAreLoadedRemovesAndDeletesNothing(t *testing.T) { if got := strings.Join(r.forge.titles("nox-agent"), ","); got != "mesh:g14.nox,my own" { t.Fatalf("keys = %s", got) } - if !r.bus.has("g14.ssh-client.forge-account") { - t.Fatal("the entry was deleted before the keys were read") + if !r.bus.serves("g14.ssh-client.forge-account", provisions.Waits) { + t.Fatal("the entry stopped serving before the keys were read; it waits and keeps serving") } if len(out) != 1 || !strings.Contains(out[0].Why, "not read yet") { t.Fatalf("the wait should be said: %+v", out) @@ -493,7 +538,7 @@ func TestAPassBeforeTheKeysAreLoadedRemovesAndDeletesNothing(t *testing.T) { r.svc.loaded = true r.svc.mu.Unlock() r.pass() - if got := strings.Join(r.forge.titles("nox-agent"), ","); got != "my own" || r.bus.has("g14.ssh-client.forge-account") { + if got := strings.Join(r.forge.titles("nox-agent"), ","); got != "my own" || r.bus.serves("g14.ssh-client.forge-account") { t.Fatalf("keys = %s", got) } } @@ -515,7 +560,7 @@ func TestKeepRunningStartsTheProviderAgainAfterARefusedWatch(t *testing.T) { r.svc.KeepRunning(ctx, func(ctx context.Context) error { calls++ if calls <= 2 { - return errors.New(`watching mesh-controller_assignments: gitea reads no state called "mesh-controller.assignments"`) + return errors.New(`watching mesh-provisioner_plans: gitea reads no state called "mesh-provisioner.plans"`) } close(serving) <-ctx.Done() @@ -535,7 +580,7 @@ func TestKeepRunningStartsTheProviderAgainAfterARefusedWatch(t *testing.T) { if len(pauses) != 2 || pauses[0] != 5*time.Second || pauses[1] != 10*time.Second { t.Fatalf("the backoff is %v", pauses) } - if len(said) == 0 || !strings.Contains(said[0], "mesh-controller.assignments") { + if len(said) == 0 || !strings.Contains(said[0], "mesh-provisioner.plans") { t.Fatalf("a refusal is said loudly: %v", said) } loud := 0 diff --git a/modules/gitea/cmd/forge-account-provider/watch_test.go b/modules/gitea/cmd/forge-account-provider/watch_test.go index 2fb60b97..60618da4 100644 --- a/modules/gitea/cmd/forge-account-provider/watch_test.go +++ b/modules/gitea/cmd/forge-account-provider/watch_test.go @@ -142,7 +142,7 @@ func TestABuildModulesStateThatNeverAnswersHoldsBackOnlyItsOwnEntries(t *testing } // 1482/AC4 (the provider's half): the manifest reads the states of both build modules beside ssh-client's and the -// controller's assignments; the provider watches exactly the modules the code names as builders. +// seat mesh-provisioner's plans; the provider watches exactly the modules the code names as builders. func TestTheManifestReadsTheBuildModulesKeys(t *testing.T) { raw, err := os.ReadFile("../../module.json") if err != nil { @@ -154,7 +154,7 @@ func TestTheManifestReadsTheBuildModulesKeys(t *testing.T) { if err := json.Unmarshal(raw, &m); err != nil { t.Fatal(err) } - for _, state := range []string{"mesh-controller.assignments", "ssh-client.keys", "mesh-build-go.keys", "mesh-build-node.keys", "mesh-build-image.keys"} { + for _, state := range []string{"mesh-provisioner.plans", "mesh-provisioner.provisions", "ssh-client.keys", "mesh-build-go.keys", "mesh-build-node.keys", "mesh-build-image.keys"} { if !slices.Contains(m.Reads, state) { t.Errorf("gitea's reads %v lack %q", m.Reads, state) } diff --git a/modules/gitea/go.mod b/modules/gitea/go.mod index 6639c1c3..87ac68dc 100644 --- a/modules/gitea/go.mod +++ b/modules/gitea/go.mod @@ -3,7 +3,7 @@ module gitea go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023150321-a2c813a6b468 + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 golang.org/x/crypto v0.57.0 diff --git a/modules/gitea/go.sum b/modules/gitea/go.sum index 2a3e9f34..a2b8e018 100644 --- a/modules/gitea/go.sum +++ b/modules/gitea/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023150321-a2c813a6b468 h1:UcMee+IV2ii5E4dRZdQUaSaeh9IedTFYumuQbBiB0k0= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023150321-a2c813a6b468/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 h1:lKU3HjNXGiVbLW2hUgFjU/TMTUYMlNnnmZxG5T7biQA= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/gitea/module.json b/modules/gitea/module.json index 3d9d00f6..2b197288 100644 --- a/modules/gitea/module.json +++ b/modules/gitea/module.json @@ -400,6 +400,7 @@ "provides": [ { "name": "npm-package-registry", + "credential": {"form": "pair"}, "scope": "mesh", "identity": { "max": 40, @@ -408,6 +409,7 @@ }, { "name": "git", + "credential": {"form": "none"}, "scope": "mesh" }, { @@ -420,6 +422,7 @@ }, { "name": "forge-account", + "credential": {"form": "none"}, "scope": "mesh" } ], @@ -564,7 +567,8 @@ "announced" ], "reads": [ - "mesh-controller.assignments", + "mesh-provisioner.plans", + "mesh-provisioner.provisions", "ssh-client.keys", "mesh-build-go.keys", "mesh-build-node.keys", diff --git a/modules/go/module.json b/modules/go/module.json index 48a6bf57..9783eda4 100644 --- a/modules/go/module.json +++ b/modules/go/module.json @@ -7,7 +7,10 @@ "provides": [ { "name": "go-toolchain", - "reach": "machine" + "reach": "machine", + "credential": { + "form": "none" + } } ], "resources": [ diff --git a/modules/influxdb/module.json b/modules/influxdb/module.json index 53389a5f..13a7905f 100644 --- a/modules/influxdb/module.json +++ b/modules/influxdb/module.json @@ -7,6 +7,9 @@ "scope": "mesh", "identity": { "in": "an InfluxDB v1 authorization" + }, + "credential": { + "form": "pair" } } ], diff --git a/modules/keycloak/cmd/keycloak-provider/grant_moved_test.go b/modules/keycloak/cmd/keycloak-provider/grant_moved_test.go index 20843dcd..80f02f9a 100644 --- a/modules/keycloak/cmd/keycloak-provider/grant_moved_test.go +++ b/modules/keycloak/cmd/keycloak-provider/grant_moved_test.go @@ -15,6 +15,7 @@ import ( "git.novox.be/novox/mesh-sdk/go/provider" "git.novox.be/novox/mesh-sdk/go/provider/harness" + "git.novox.be/novox/mesh-sdk/go/provisions" ) // grantGives writes a grant file (MESH_RECEIVES) naming consumers the plans do not move (issue 937): each one's @@ -38,11 +39,33 @@ func grantGives(t *testing.T, given ...map[string]any) string { return at } -// noOpPlan puts one assignment the bus can read that binds nothing of keycloak's, so the moved path has plans it +// noOpPlan puts one plan the bus can read that binds nothing of keycloak's, so the moved path has plans it // could read (m.asked answers "read", issue 241) even when nothing is moved. func noOpPlan(bus *bucketBus, node, module string) { - raw, _ := json.Marshal(map[string]any{"node": node, "module": module}) - _ = bus.Put(ctx, provider.AssignmentsBucket, node+"."+module, raw) + seatPlan(bus, node, module, map[string]any{}) +} + +// seatPlan puts a module's plan as the seat mesh-provisioner keeps it, and for each of its provisions served by +// keycloak on novox the key the seat composes, asked (novox/hq ADR 0570 clauses 1 to 3). +func seatPlan(bus *bucketBus, node, module string, plan map[string]any) { + raw, _ := json.Marshal(map[string]any{"v": 1, "node": node, "module": module, "plan": plan}) + _ = bus.Put(ctx, provisions.PlansBucket, node+"."+module, raw) + list, _ := plan["provisions"].([]any) + for _, e := range list { + entry := e.(map[string]any) + name, _ := entry["provision"].(string) + local, _ := entry["local"].(string) + key, err := provisions.Key(node, module, name, local) + if err != nil { + panic(err) + } + asked, err := provisions.Encode(provisions.Provision{Provision: name, Local: local, State: provisions.Asked, + Consumer: provisions.End{Node: node, Module: module}, Provider: provisions.End{Node: "novox", Module: "keycloak"}}) + if err != nil { + panic(err) + } + _ = bus.Put(ctx, provisions.ProvisionsBucket, key, asked) + } } // waitUntil polls cond until it is true or the timeout passes. diff --git a/modules/keycloak/cmd/keycloak-provider/live_retire_test.go b/modules/keycloak/cmd/keycloak-provider/live_retire_test.go index 4cbc24a0..0ab5cc34 100644 --- a/modules/keycloak/cmd/keycloak-provider/live_retire_test.go +++ b/modules/keycloak/cmd/keycloak-provider/live_retire_test.go @@ -62,6 +62,14 @@ func TestLiveRetirement(t *testing.T) { if code := authorize(); code == 200 { t.Fatal("a retired client is still served") } + // Its secret is removed (ADR 0570 clause 5): the mesh's no longer authenticates. + if found, err := o.KC.FindClient(ctx, o.Realm, p.As); err != nil || found == nil { + t.Fatal(found, err) + } else if id, _ := found["id"].(string); true { + if secret, err := o.KC.ClientSecretByID(ctx, o.Realm, id); err != nil || secret == p.Password { + t.Fatal("a retired client keeps the mesh's secret", err) + } + } // Retired, so a person may delete it. if err := o.Delete(ctx, p.As); err != nil { t.Fatal(err) diff --git a/modules/keycloak/cmd/keycloak-provider/moved_test.go b/modules/keycloak/cmd/keycloak-provider/moved_test.go index f69aa363..dd5371dd 100644 --- a/modules/keycloak/cmd/keycloak-provider/moved_test.go +++ b/modules/keycloak/cmd/keycloak-provider/moved_test.go @@ -155,10 +155,9 @@ func TestBothPathsLeaveTheSameClientInKeycloak(t *testing.T) { t.Fatal(err) } bus := &bucketBus{buckets: map[string]map[string][]byte{}} - plan, _ := json.Marshal(map[string]any{"node": "home", "module": "grafana", "provisioning": map[string]any{ + seatPlan(bus, "home", "grafana", map[string]any{ "provisions": []any{map[string]any{"provision": ProvisionName, "provider": "novox", - "provider-module": "keycloak", "contributes": grafanaContributes, "names": grafanaNames}}}}) - _ = bus.Put(ctx, provider.AssignmentsBucket, "home.grafana", plan) + "provider-module": "keycloak", "contributes": grafanaContributes, "names": grafanaNames}}}) pr := provider.New(provider.Options{Node: "novox", Module: "keycloak", Bus: bus, Vault: provider.ConsumersFile{Paths: []string{file}}, Offers: map[string]provider.Offer{ProvisionName: {Credential: provider.PerConsumer, diff --git a/modules/keycloak/cmd/keycloak-provider/oidc.go b/modules/keycloak/cmd/keycloak-provider/oidc.go index c7815462..ca702b0a 100644 --- a/modules/keycloak/cmd/keycloak-provider/oidc.go +++ b/modules/keycloak/cmd/keycloak-provider/oidc.go @@ -24,15 +24,18 @@ package main import ( "context" + "crypto/rand" + "encoding/hex" "encoding/json" "errors" "fmt" - "git.novox.be/novox/mesh-sdk/go/provider/harness" "net/url" "regexp" "sort" "strings" "time" + + "git.novox.be/novox/mesh-sdk/go/provider/harness" ) // Mark is the attribute marking a client as the mesh's own work. @@ -307,8 +310,10 @@ func (o OidcClients) Holds(ctx context.Context, p harness.Provision) (bool, erro return secret == p.Password, nil } -// Retire disables a consumer's client — only one the mesh made — and marks it with when and why. -// Answers what happened: "retired", "absent" or "not ours". +// Retire, on the operator's yes (novox/hq ADR 0570 clause 5), removes the client's secret — replaced by one nobody +// is given, so the mesh's credential no longer authenticates — and disables and marks the client with when and why. +// The client and its settings are kept; asked for again, Ensure sets the mesh's secret and enables it. Only a client +// the mesh made. Answers what happened: "retired", "absent" or "not ours". func (o OidcClients) Retire(ctx context.Context, as, why string, at time.Time) (string, error) { found, err := o.KC.FindClient(ctx, o.Realm, as) if err != nil { @@ -332,6 +337,11 @@ func (o OidcClients) Retire(ctx context.Context, as, why string, at time.Time) ( attrs[MarkRetiredWhy] = why merged["attributes"] = attrs merged["enabled"] = false + unknown := make([]byte, 32) + if _, err := rand.Read(unknown); err != nil { + return "", err + } + merged["secret"] = hex.EncodeToString(unknown) // never kept, never said id, _ := found["id"].(string) return "retired", o.KC.UpdateClient(ctx, o.Realm, id, merged) } diff --git a/modules/keycloak/cmd/keycloak-provider/oidc_test.go b/modules/keycloak/cmd/keycloak-provider/oidc_test.go index 45c95ae3..4d8f52d0 100644 --- a/modules/keycloak/cmd/keycloak-provider/oidc_test.go +++ b/modules/keycloak/cmd/keycloak-provider/oidc_test.go @@ -186,7 +186,7 @@ func TestAClientTheMeshDidNotMakeIsRefusedAndLeftAlone(t *testing.T) { } } -// Retired is disabled and marked, everything else kept; asked for again it is enabled as it was; only +// Retired is its secret removed, disabled and marked, everything else kept (ADR 0570 clause 5); asked for again it is enabled as it was; only // a deletion removes it, and never while enabled (novox/hq ADR 0230). func TestARetiredClientIsDisabledKeptAndEnabledAgainAsItWas(t *testing.T) { f, o := oidcWorld(t) @@ -204,7 +204,7 @@ func TestARetiredClientIsDisabledKeptAndEnabledAgainAsItWas(t *testing.T) { t.Fatal(done, err) } c := f.clients[id] - if c["enabled"] != false || c["secret"] != "s" || attributes(c)[MarkRetired] != "2026-10-06T12:00:00Z" || + if c["enabled"] != false || c["secret"] == "s" || c["secret"] == "" || attributes(c)[MarkRetired] != "2026-10-06T12:00:00Z" || attributes(c)[MarkRetiredWhy] != "the mesh stopped asking for it" || c["description2"] == nil { t.Fatalf("%v", c) } @@ -219,7 +219,7 @@ func TestARetiredClientIsDisabledKeptAndEnabledAgainAsItWas(t *testing.T) { t.Fatal(err) } c = f.clients[id] - if c["enabled"] != true || attributes(c)[MarkRetired] != nil || attributes(c)[MarkRetiredWhy] != nil || c["description2"] == nil { + if c["enabled"] != true || c["secret"] != "s" || attributes(c)[MarkRetired] != nil || attributes(c)[MarkRetiredWhy] != nil || c["description2"] == nil { t.Fatalf("%v", c) } mustHold(t, o, p, true) diff --git a/modules/keycloak/cmd/keycloak-provider/provisioner.go b/modules/keycloak/cmd/keycloak-provider/provisioner.go index e0428a4e..6bf4aef6 100644 --- a/modules/keycloak/cmd/keycloak-provider/provisioner.go +++ b/modules/keycloak/cmd/keycloak-provider/provisioner.go @@ -62,7 +62,8 @@ func (a provisioner) Create(ctx context.Context, p harness.Provision) error { return nil } -// Retire disables the consumer's client and marks it, never deletes it (novox/hq ADR 0230). +// Retire removes the consumer's client secret and disables and marks the client, never deletes it (novox/hq ADR 0570 +// clause 5, ADR 0230). func (a provisioner) Retire(ctx context.Context, as string, _ map[string]any, why string, at time.Time) error { if err := a.refused(); err != nil { return err @@ -75,7 +76,7 @@ func (a provisioner) Retire(ctx context.Context, as string, _ map[string]any, wh case "not ours": a.log("[provisioner:oidc-client] %s: a client of that id exists that the mesh did not make — left alone", as) case "retired": - a.log("[provisioner:oidc-client] retired client %s in realm %s: disabled, kept, marked to delete", as, a.clients.Realm) + a.log("[provisioner:oidc-client] retired client %s in realm %s: its secret removed, disabled, kept, marked to delete", as, a.clients.Realm) a.announce("client.retired", map[string]any{"realm": a.clients.Realm, "clientId": as}) } return nil diff --git a/modules/keycloak/go.mod b/modules/keycloak/go.mod index 28481201..64fffc79 100644 --- a/modules/keycloak/go.mod +++ b/modules/keycloak/go.mod @@ -2,4 +2,4 @@ module keycloak go 1.26.0 -require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021114014-66e8fdaa62e2 +require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 diff --git a/modules/keycloak/go.sum b/modules/keycloak/go.sum index 6162079d..f8bfbd2b 100644 --- a/modules/keycloak/go.sum +++ b/modules/keycloak/go.sum @@ -1,2 +1,2 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021114014-66e8fdaa62e2 h1:Cdl5N56QPfmFBoLHrDfvvyc5Q+PUnTjf3fgYD7imcco= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021114014-66e8fdaa62e2/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 h1:lKU3HjNXGiVbLW2hUgFjU/TMTUYMlNnnmZxG5T7biQA= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= diff --git a/modules/keycloak/module.json b/modules/keycloak/module.json index 847d5f4f..84c08f10 100644 --- a/modules/keycloak/module.json +++ b/modules/keycloak/module.json @@ -9,6 +9,7 @@ "provides": [ { "name": "oidc-client", + "credential": {"form": "pair"}, "scope": "mesh", "identity": { "max": 255, @@ -240,6 +241,7 @@ "bound" ], "reads": [ - "mesh-controller.assignments" + "mesh-provisioner.plans", + "mesh-provisioner.provisions" ] } diff --git a/modules/mailu/module.json b/modules/mailu/module.json index 93d72adb..0faccc58 100644 --- a/modules/mailu/module.json +++ b/modules/mailu/module.json @@ -665,7 +665,14 @@ "path": "${dir:mesh-state}/config.json", "mode": "0600", "template": "files/config.json", - "takes-config": ["domain", "proxy-address", "sitename", "website", "url", "container"], + "takes-config": [ + "domain", + "proxy-address", + "sitename", + "website", + "url", + "container" + ], "merge": "json" }, { @@ -758,6 +765,9 @@ "identity": { "max": 64, "in": "a mailbox's local part" + }, + "credential": { + "form": "pair" } } ], diff --git a/modules/mesh-build-image/.gitignore b/modules/mesh-build-image/.gitignore new file mode 100644 index 00000000..00eb03a8 --- /dev/null +++ b/modules/mesh-build-image/.gitignore @@ -0,0 +1 @@ +mesh-build-image diff --git a/modules/mesh-build-image/go.mod b/modules/mesh-build-image/go.mod index 701791be..cb908ec3 100644 --- a/modules/mesh-build-image/go.mod +++ b/modules/mesh-build-image/go.mod @@ -3,7 +3,7 @@ module meshbuildimage go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029233535-78c9ec881771 + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-build-image/go.sum b/modules/mesh-build-image/go.sum index 9dde7f0c..a2b8e018 100644 --- a/modules/mesh-build-image/go.sum +++ b/modules/mesh-build-image/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029233535-78c9ec881771 h1:Tt16HJk8JZRce1i3JzzDSNdCWOrv7dw8FQ/bckNQ5Ig= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029233535-78c9ec881771/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 h1:lKU3HjNXGiVbLW2hUgFjU/TMTUYMlNnnmZxG5T7biQA= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/mesh-build-node/.gitignore b/modules/mesh-build-node/.gitignore new file mode 100644 index 00000000..f7604aaa --- /dev/null +++ b/modules/mesh-build-node/.gitignore @@ -0,0 +1 @@ +mesh-build-node diff --git a/modules/mesh-build-node/go.mod b/modules/mesh-build-node/go.mod index f8b463c0..cd83b38b 100644 --- a/modules/mesh-build-node/go.mod +++ b/modules/mesh-build-node/go.mod @@ -3,7 +3,7 @@ module meshbuildnode go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029233535-78c9ec881771 + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 github.com/distribution/distribution/v3 v3.1.2 github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-build-node/go.sum b/modules/mesh-build-node/go.sum index eee5d03f..5f0e39b2 100644 --- a/modules/mesh-build-node/go.sum +++ b/modules/mesh-build-node/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029233535-78c9ec881771 h1:Tt16HJk8JZRce1i3JzzDSNdCWOrv7dw8FQ/bckNQ5Ig= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029233535-78c9ec881771/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 h1:lKU3HjNXGiVbLW2hUgFjU/TMTUYMlNnnmZxG5T7biQA= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/alicebob/miniredis/v2 v2.35.0 h1:QwLphYqCEAo1eu1TqPRN2jgVMPBweeQcR21jeqDCONI= github.com/alicebob/miniredis/v2 v2.35.0/go.mod h1:TcL7YfarKPGDAthEtl5NBeHZfeUQj6OXMm/+iu5cLMM= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= diff --git a/modules/mesh-catalog/heard.ts b/modules/mesh-catalog/heard.ts index eaa8f882..619639ed 100644 --- a/modules/mesh-catalog/heard.ts +++ b/modules/mesh-catalog/heard.ts @@ -45,13 +45,72 @@ export interface Built { branches?: string[]; } -/** The events a build outcome reaches this module on: the build seat's, as it happens, and the control plane's replay. */ -export const BUILD_EVENTS = [ - // The build seat node-build-agent and the retired mesh-build-machine went with build-agent (novox/hq ADR 0524 §5, - // ADR 0525 §10): no build seat says `built` since R4g, so what the mesh holds is asked when this module starts. - // What the mesh already held, when this module asks what it missed (novox/hq ADR 0134). - "mesh-controller.built-before", -] as const; +/** + * Where a build outcome comes from now: the builder's log (novox/hq ADR 0571 clauses 1 and 9). The module asks the + * builder's tool for the builds that ended since a time, and again whenever the builder's state of done keys changes. + */ +export const BUILDER = "mesh-delivery-builder"; +export const BUILDS_SINCE_TOOL = "builder_builds_since"; +export const BUILDS_STATE = "mesh-delivery-builder.builds"; +export const MANIFESTS_STATE = "mesh-delivery-indexer.manifests"; + +/** One entry of the builder's log, an ended build (mesh-sdk delivery.BuildLogEntry). */ +export interface LogEntry { + event: string; + module: string; + fp: string; + repository: string; + directory: string; + commit: string; + ref?: string; + node?: string; + at: string; + /** done, failed or queued. */ + outcome?: string; + /** The digest of the manifest built, read from the indexer's state of manifests. */ + manifest?: string; + against?: string[]; + made?: Made[]; +} + +/** The answer of the builder's tool: every ended build since a time, oldest first, and whether the log held more. */ +export interface BuildsSince { + since: string; + builds: LogEntry[]; + more?: boolean; +} + +/** + * What a tool answers, whether bare or wrapped as the runtime wraps a tool's reply ({output} or {content:[{text}]}). + */ +export function answerOf(raw: unknown): T { + const r = raw as { output?: unknown; content?: { text?: string }[] } | null; + if (r && typeof r.output === "string") return JSON.parse(r.output) as T; + if (r && Array.isArray(r.content) && typeof r.content[0]?.text === "string") return JSON.parse(r.content[0].text) as T; + if (typeof raw === "string") return JSON.parse(raw) as T; + return raw as T; +} + +/** + * A log entry as the build outcome the catalogue places: a build that is done is placed with its manifest, what it + * made and what it was built against; one that failed is passed over (placeable says so); one still queued has not + * ended and is not taken in. `replay` is set by the caller for history. + */ +export function builtFrom(e: LogEntry, manifest: unknown, replay: boolean): Built | undefined { + if (e.outcome === "queued") return undefined; + return { + module: e.module, + commit: e.commit, + repository: e.repository, + path: e.directory, + ref: e.ref ?? "", + manifest, + made: e.made ?? [], + against: e.against ?? [], + replay, + ...(e.outcome === "failed" ? { failed: "the builder's log says the build failed" } : {}), + }; +} /** * Whether a build outcome is a module-version to place in the graph, and why not when it is not. diff --git a/modules/mesh-catalog/index.ts b/modules/mesh-catalog/index.ts index 13a98d2d..cc034002 100644 --- a/modules/mesh-catalog/index.ts +++ b/modules/mesh-catalog/index.ts @@ -9,9 +9,15 @@ // and appear the next time round, once whatever they were waiting for is registered. A chain and a // diamond need no special handling, and nothing holds a plan. -import { on, emit } from "@novox/mesh-sdk/events"; +import { emit } from "@novox/mesh-sdk/events"; +import { broker } from "@novox/mesh-sdk/messaging"; +import { invokeTool } from "@novox/mesh-sdk/tools"; +import { state } from "@novox/mesh-sdk/state"; import { Graph } from "./store.js"; -import { BUILD_EVENTS, placeable, type Built } from "./heard.js"; +import { + answerOf, builtFrom, placeable, BUILDER, BUILDS_SINCE_TOOL, BUILDS_STATE, MANIFESTS_STATE, + type Built, type BuildsSince, type LogEntry, +} from "./heard.js"; const graph = Graph.fromEnv(); @@ -77,18 +83,74 @@ const placeTheBuild = async (event: { body: unknown }): Promise => { } }; -// As it happens — from the build seat that builds now, and the retired one while it still answers — and -// what the mesh already held when this module asked what it missed (novox/hq issue 712). -for (const heard of BUILD_EVENTS) await on(heard, placeTheBuild); +// **From the builder's log** (novox/hq ADR 0571 clauses 1 and 9). The builder keeps every build's end in a log of its +// own and answers the builds that ended since a time by its tool. This module asks at its start for what it missed (all +// the log holds, taken in as history: registered, announced to nobody), and again whenever the builder's state of done +// keys changes, for what ended since the last build it took in. Registering a build already held changes nothing, and a +// build is taken in once per run of this module. +const taken = new Set(); +let since = new Date(0).toISOString(); +let caughtUp = false; +let asking: Promise | undefined; +let askAgain = false; -// **And ask for what was built before this catalogue existed** (novox/hq 04-ISSUES/050). -// -// The queue above is durable, so nothing is missed once this is running. What it cannot have is -// what was announced before it first ran — and on a fresh mesh that is never arbitrary: the shared -// base, the store this runs on, and this module itself are each necessarily built BEFORE a -// catalogue exists to hear about them. The graph's foundation is the part it never sees. -// -// Asked on every start, not only the first. A catalogue cannot tell whether it has a gap, and the -// answer is idempotent: registering a build already held changes nothing and announces nothing. -// Asked AFTER subscribing, so a build arriving during the replay is not lost between the two. +const manifestOf = async (digest: string | undefined): Promise => { + if (!digest) return {}; + const held = await state<{ manifest?: string }>(MANIFESTS_STATE).get(digest); + if (!held?.value.manifest) return {}; + return JSON.parse(held.value.manifest); +}; + +const takeIn = async (e: LogEntry, replay: boolean): Promise => { + const key = `${e.module}/${e.fp}`; + if (taken.has(key)) return; + const built = builtFrom(e, await manifestOf(e.manifest), replay); + if (!built) return; + await placeTheBuild({ body: built }); + taken.add(key); +}; + +const catchUp = async (): Promise => { + for (;;) { + const answer = answerOf( + await invokeTool(broker(), BUILDER, BUILDS_SINCE_TOOL, { since }), + ); + for (const e of answer.builds ?? []) { + await takeIn(e, !caughtUp); + if (e.at > since) since = e.at; + } + if (!answer.more) break; + } + caughtUp = true; +}; + +/** One ask at a time; a change heard while asking is asked for again once, so none is missed. */ +const ask = (): Promise => { + if (asking) { + askAgain = true; + return asking; + } + asking = (async () => { + try { + do { + askAgain = false; + await catchUp(); + } while (askAgain); + } catch (err) { + console.error("mesh-catalog: the builder's builds since", since, "could not be taken in:", err); + } finally { + asking = undefined; + } + })(); + return asking; +}; + +// Asked AFTER watching begins, so a build ending while the first ask runs asks again rather than being missed: the watch +// hands every current key first (the start's ask), then every change. +await state(BUILDS_STATE).watch(async (change) => { + if (!change.current) await ask(); +}); +await ask(); + +// **And say that this catalogue has caught up** (novox/hq 04-ISSUES/050). await emit("catching-up", {}); diff --git a/modules/mesh-catalog/module.json b/modules/mesh-catalog/module.json index b8f45096..c5ad61a4 100644 --- a/modules/mesh-catalog/module.json +++ b/modules/mesh-catalog/module.json @@ -25,9 +25,7 @@ "secrets": { "postgres-database": "${dir:state}/database.secret" }, - "consumes": [ - "mesh-controller.built-before" - ], + "consumes": [], "emits": [ "registered", "upgraded", @@ -92,5 +90,12 @@ "from": "files" } ] - } + }, + "invokes": [ + "mesh-delivery-builder.builder_builds_since" + ], + "reads": [ + "mesh-delivery-builder.builds", + "mesh-delivery-indexer.manifests" + ] } diff --git a/modules/mesh-catalog/test/heard.test.ts b/modules/mesh-catalog/test/heard.test.ts index 84b91ac4..391cb5e9 100644 --- a/modules/mesh-catalog/test/heard.test.ts +++ b/modules/mesh-catalog/test/heard.test.ts @@ -2,7 +2,7 @@ import { test } from "node:test"; import assert from "node:assert/strict"; import { readFile } from "node:fs/promises"; -import { BUILD_EVENTS, placeable } from "../heard.js"; +import { answerOf, builtFrom, placeable, BUILDER, BUILDS_SINCE_TOOL, BUILDS_STATE, MANIFESTS_STATE } from "../heard.js"; // A walk's build as the build seat says it (mesh-controller's link.BuildResult), here mailu's of novox/hq issue 712. const walkBuild = { @@ -20,14 +20,25 @@ const walkBuild = { made: [{ name: "code", kind: "bundle", reference: "artifact-store://mailu/code/blobs/sha256:d041" }], }; -test("the catalogue hears no retired build seat, and what the mesh held (issue 712, issue 1887 E2)", async () => { - for (const retired of ["node-build-agent.built", "mesh-build-machine.built"]) { - assert.ok(!(BUILD_EVENTS as readonly string[]).includes(retired), `${retired} is still heard`); - } - assert.ok(BUILD_EVENTS.includes("mesh-controller.built-before")); - // Subscribing is not enough: the mesh grants and routes only what the manifest consumes. +test("the catalogue takes builds from the builder's log, and hears no controller event (ADR 0571 clause 9)", async () => { const manifest = JSON.parse(await readFile(new URL("../module.json", import.meta.url), "utf8")); - for (const heard of BUILD_EVENTS) assert.ok(manifest.consumes.includes(heard), `${heard} is not consumed`); + assert.deepEqual(manifest.consumes, []); + assert.ok(manifest.invokes.includes(`${BUILDER}.${BUILDS_SINCE_TOOL}`), "the builder's tool is not invoked"); + for (const read of [BUILDS_STATE, MANIFESTS_STATE]) assert.ok(manifest.reads.includes(read), `${read} is not read`); + // A log entry is placed as a build: done with its manifest, made and against; failed passed over; queued not taken in. + const entry = { event: "build.ended", module: "mailu", fp: "fp1", repository: walkBuild.repository, directory: "modules/mailu", + commit: walkBuild.commit, ref: "main", at: "2026-10-29T20:00:00Z", outcome: "done", manifest: "ab", against: ["img@sha256:1"], + made: walkBuild.made }; + const done = builtFrom(entry, walkBuild.manifest, false); + assert.deepEqual(placeable(done!), { placed: "yes", module: "mailu", commit: walkBuild.commit }); + assert.deepEqual([done!.path, done!.against, done!.replay], ["modules/mailu", ["img@sha256:1"], false]); + assert.equal(placeable(builtFrom({ ...entry, outcome: "failed" }, {}, true)!).placed, "no"); + assert.equal(builtFrom({ ...entry, outcome: "queued" }, {}, true), undefined); + // The tool's answer is read bare or as the runtime wraps it. + const bare = { since: "x", builds: [entry] }; + for (const raw of [bare, { output: JSON.stringify(bare) }, { content: [{ text: JSON.stringify(bare) }] }]) { + assert.equal(answerOf(raw).builds[0].module, "mailu"); + } }); test("a walk's build is placed, under its module and commit", () => { diff --git a/modules/mesh-conditions/cmd/mesh-conditions/compare.go b/modules/mesh-conditions/cmd/mesh-conditions/compare.go new file mode 100644 index 00000000..e04e805a --- /dev/null +++ b/modules/mesh-conditions/cmd/mesh-conditions/compare.go @@ -0,0 +1,221 @@ +package main + +// **Built dark, compared, then switched by the operator** (novox/hq ADR 0564 clause 9; issue 1952 C5, C8): `compare` +// lists the open keys of the keeper and of the controller, each difference with its reason; the kinds that end with the +// controller, the kinds only the keeper raises and the keys held while their module is delivered are listed apart and +// not counted. `take-over` raises an approve ask under the keeper's own name (ADR 0259), refused while compare shows a +// difference, and on the warrant writes the take-over key naming the keeper. `hand-back` writes it back to the +// controller until the lease write, and is refused after it. + +import ( + "encoding/json" + "errors" + "fmt" + "sort" + "strings" + "time" + + stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/asks" + "git.novox.be/novox/mesh-sdk/go/health" +) + +// Difference is one key the two sides do not agree on, or one listed apart. +type Difference struct { + Key string `json:"key"` + Kind string `json:"kind"` + Reason string `json:"reason"` +} + +// Comparison is compare's answer. +type Comparison struct { + Differences []Difference `json:"differences"` + // Apart are the keys not counted: of kinds that end with the controller, raised only by the keeper, or held. + Apart []Difference `json:"apart"` + Same int `json:"same"` + At time.Time `json:"at"` +} + +// Empty says nothing differs. +func (c Comparison) Empty() bool { return len(c.Differences) == 0 } + +// Compare is the keeper's open conditions against the controller's. +func (k *Keeper) Compare() (Comparison, error) { + mine, err := k.Open() + if err != nil { + return Comparison{}, err + } + k.mu.Lock() + k.init() + theirs := map[string]health.Condition{} + for key, c := range k.ctrl { + theirs[key] = c + } + heldNow := map[string]held{} + for key, h := range k.held { + heldNow[key] = h + } + k.mu.Unlock() + out := Comparison{Differences: []Difference{}, Apart: []Difference{}, At: k.Now().UTC()} + open := map[string]health.Condition{} + for _, c := range mine { + open[c.Key] = c + } + keys := map[string]bool{} + for key := range open { + keys[key] = true + } + for key := range theirs { + keys[key] = true + } + for key := range heldNow { + keys[key] = true + } + sorted := make([]string, 0, len(keys)) + for key := range keys { + sorted = append(sorted, key) + } + sort.Strings(sorted) + for _, key := range sorted { + c, inMine := open[key] + t, inTheirs := theirs[key] + kind := c.Kind + if !inMine { + kind = t.Kind + } + home, homed := k.homeOf(kind) + if h, ok := heldNow[key]; ok { + out.Apart = append(out.Apart, Difference{Key: key, Kind: kind, + Reason: fmt.Sprintf("held while %s is delivered on %s (version %s)", h.Module, h.Node, short(h.Version))}) + continue + } + switch { + case homed && (home.Carried || home.Replaced): + why := "ends with the controller: carried from its bucket until the lease write" + if home.Replaced { + why = "ends with the controller: its fault is observed otherwise" + } + if home.Why != "" { + why += " (" + home.Why + ")" + } + out.Apart = append(out.Apart, Difference{Key: key, Kind: kind, Reason: why}) + case inMine && !inTheirs && (!homed || !home.Controller): + out.Apart = append(out.Apart, Difference{Key: key, Kind: kind, Reason: "keeper only: the controller never raises this kind"}) + case inMine && !inTheirs: + out.Differences = append(out.Differences, Difference{Key: key, Kind: kind, + Reason: "open at the keeper, not at the controller"}) + case !inMine && !homed: + out.Differences = append(out.Differences, Difference{Key: key, Kind: kind, + Reason: fmt.Sprintf("open at the controller; no home is decided for the kind %s (ADR 0564's gap)", kind)}) + case !inMine: + out.Differences = append(out.Differences, Difference{Key: key, Kind: kind, + Reason: fmt.Sprintf("open at the controller; %s has not written its finding", home.Observer)}) + case c.Severity != t.Severity: + out.Differences = append(out.Differences, Difference{Key: key, Kind: kind, + Reason: fmt.Sprintf("%s at the keeper, %s at the controller", c.Severity, t.Severity)}) + default: + out.Same++ + } + } + return out, nil +} + +// Asks is the operator channel as the keeper asks it: an approve ask under its own name, and where one stands. +type Asks interface { + Ask(a asks.Ask) error +} + +// TakeOver raises the approve ask of a switch to the keeper, refused while compare differs or after the lease write. +func (k *Keeper) TakeOver(a Asks, by, why string) (map[string]any, error) { + if strings.TrimSpace(why) == "" { + return nil, errors.New("a take-over says why") + } + if k.LeaseAway() { + return nil, errors.New("the controller's lease is handed away: the keeper is the one source already") + } + cmp, err := k.Compare() + if err != nil { + return nil, err + } + if !cmp.Empty() { + return nil, fmt.Errorf("refused: compare shows %d difference(s), the first %s (%s); the take-over waits for none", + len(cmp.Differences), cmp.Differences[0].Key, cmp.Differences[0].Reason) + } + now := k.Now().UTC() + id := fmt.Sprintf("take-over-%d", now.UnixNano()) + act := asks.Act{"verb": Seat + ".take-over", "module": Module, "ask": id} + bind, err := asks.ActDigest(act) + if err != nil { + return nil, err + } + ask := asks.Ask{ID: id, Headline: "Switch the mesh's alarms to the new keeper?", Who: asks.Operator, + Explanation: fmt.Sprintf("The new condition keeper holds the same %d open alarms as the controller. Your yes makes "+ + "the alarm messages come from the keeper, with the same words. It can be handed back until the controller "+ + "gives up its lease.", cmp.Same), + Options: []asks.Option{{ID: "yes", Label: "Yes, switch", Does: "Alarms come from the keeper.", Level: asks.Approve, Binds: bind}, + {ID: "no", Label: "No, leave it", Does: "Alarms keep coming from the controller.", Level: asks.Acknowledge}}, + Expires: now.Add(asks.ApproveLasts - time.Minute), OnExpiry: "Alarms keep coming from the controller.", + About: Module + "-take-over", Details: "Asked by " + by + ": " + why} + if err := a.Ask(ask); err != nil { + return nil, err + } + return map[string]any{"asked": id, "same": cmp.Same, "apart": len(cmp.Apart)}, nil +} + +// Warranted is the act on the operator's answer: a yes, compare still empty, writes the take-over key naming the keeper. +func (k *Keeper) Warranted(o asks.Option, w asks.Warrant) error { + if o.ID != "yes" { + k.Logf("the take-over %s was answered %s: alarms stay with the controller", w.Ask, o.ID) + return nil + } + cmp, err := k.Compare() + if err != nil { + return err + } + if !cmp.Empty() { + return fmt.Errorf("the take-over %s was approved, and compare now shows %d difference(s): not switched", w.Ask, + len(cmp.Differences)) + } + return k.writeTakeover(health.Takeover{Follow: health.FollowKeeper, At: k.Now().UTC(), By: w.Says(), Ask: w.Ask}) +} + +// HandBack switches the readers back to the controller, until the lease write. +func (k *Keeper) HandBack(by, why string) (map[string]any, error) { + if strings.TrimSpace(why) == "" { + return nil, errors.New("a hand-back says why") + } + if k.LeaseAway() { + return nil, errors.New("refused: the controller's lease is handed away, so there is no controller to hand back to") + } + t := health.Takeover{Follow: health.FollowController, At: k.Now().UTC(), By: by, Why: why} + if err := k.writeTakeover(t); err != nil { + return nil, err + } + return map[string]any{"follow": t.Follow, "at": t.At}, nil +} + +// writeTakeover writes the take-over key, and keeps it in history as the record of the switch. +func (k *Keeper) writeTakeover(t health.Takeover) error { + raw, err := health.EncodeTakeover(t) + if err != nil { + return err + } + _, rev, found, err := k.Store.Get(TakeoverState, health.TakeoverKey) + if err != nil { + return err + } + if found { + _, err = k.Store.Update(TakeoverState, health.TakeoverKey, json.RawMessage(raw), rev) + } else { + _, err = k.Store.Create(TakeoverState, health.TakeoverKey, json.RawMessage(raw)) + } + if errors.Is(err, stdio.ErrChanged) { + return fmt.Errorf("the take-over key moved under this write: read it again and ask again") + } + if err != nil { + return err + } + k.Takeover(t) + k.Logf("the readers of conditions follow the %s from now (%s)", t.Follow, t.By) + return nil +} diff --git a/modules/mesh-conditions/cmd/mesh-conditions/hold.go b/modules/mesh-conditions/cmd/mesh-conditions/hold.go new file mode 100644 index 00000000..8491812e --- /dev/null +++ b/modules/mesh-conditions/cmd/mesh-conditions/hold.go @@ -0,0 +1,340 @@ +package main + +// **A module being delivered raises no condition until its delivery ends** (novox/hq ADR 0564 clause 13 and Limits 12; +// issue 1952 C7, C8). The keeper reads the assigner's wants and the nodes' haves, and judges a module's delivery on a +// node with mesh-sdk's delivery.MotionOf, as the view does. While it is delivered there, a finding about it there, or +// about what it serves, and a condition carried from the controller's bucket on those subjects, is kept in the state +// `held` and raises nothing. When the delivery ends, healthy at the want or failed, what still stands is raised once, +// saying it happened during the delivery; what went is dropped with a log line. The hold lasts at most the delivery's +// bound, a key of the state `bounds` whose time to live is the bound: its expiry raises what is held, saying the delivery +// did not end. A finding about another module or about the machine itself is never held. A condition already open is +// not held: it is no news. + +import ( + "encoding/json" + "errors" + "fmt" + "sort" + "strings" + "time" + + stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/health" +) + +// The modules whose delivery holds what they serve beyond their own findings (Limits 12). +const ( + controllerModule = "mesh-controller" + runtimeModule = "node-tools" +) + +// controllerServes are the sources of what the controller serves while it is delivered: its listening, its lease, its +// writes and its facts (ADR 0505 §2 (a)), matched to mesh-controller whatever its key names. +var controllerServes = map[string]bool{"S4": true, "S7": true, "S12": true, "S13": true, "S14": true, "S25": true} + +// about is a node and a module a finding or a condition is about, for the hold. +type about struct{ Node, Module string } + +// aboutOf are the modules on nodes a condition's subject is about (Limits 12): a module on a machine, a provider on its +// node, a seat of the same name as its module on a machine, the controller's own work wherever mesh-controller is +// delivered, and every module's tools on a machine whose tool runner is delivered. Never the machine itself. +func (k *Keeper) aboutOf(kind, source string, s health.Subject) []about { + var out []about + parts := strings.Split(s.ID, ".") + switch s.Scope { + case health.ScopeModule: + if len(parts) >= 2 { + out = append(out, about{Node: parts[len(parts)-1], Module: strings.Join(parts[:len(parts)-1], ".")}) + } + case health.ScopeProvider: + if len(parts) >= 2 { + out = append(out, about{Node: parts[1], Module: parts[0]}) + } + case health.ScopeSeat: + // The module the placement says holds the seat on that node, or the seat's own name where none is read. + if len(parts) >= 2 { + node, seat := parts[len(parts)-1], strings.Join(parts[:len(parts)-1], ".") + module := k.holders[node+"."+seat] + if module == "" { + module = seat + } + out = append(out, about{Node: node, Module: module}) + } + case health.ScopeMachine: + // The machine itself (presence, network, resolver, disk) is never held; its tools are, while its tool runner is + // delivered. + if source == "S11" || kind == "tools-silent" { + out = append(out, about{Node: s.Machine, Module: runtimeModule}) + if s.Machine == "" && len(parts) > 0 { + out[len(out)-1].Node = parts[0] + } + } + return out + } + if controllerServes[source] || (s.Scope == health.ScopeCore && strings.HasPrefix(s.ID, "controller")) { + for hk := range k.wants { + if node, module := splitNodeModule(hk); module == controllerModule { + out = append(out, about{Node: node, Module: module}) + } + } + } + if s.Machine != "" && s.Scope != health.ScopeMachine { + out = append(out, about{Node: s.Machine, Module: runtimeModule}) + } + return out +} + +func splitNodeModule(hk string) (string, string) { + node, module, _ := strings.Cut(hk, ".") + return node, module +} + +// motion is the delivery of module on node, from its want and have. +func (k *Keeper) motion(node, module string) (delivery.Motion, delivery.Want) { + w, wanted := k.wants[node+"."+module] + h, has := k.haves[node+"."+module] + return delivery.MotionOf(w, wanted, h, has, "", false), w +} + +// holdFor says whether key is held now, and as what. Called with mu held. +func (k *Keeper) holdFor(key string, f *health.Finding, carried *health.Condition, open bool) (held, bool) { + if open || (f == nil && carried == nil) { + return held{}, false + } + var kind, source string + var subject health.Subject + if f != nil { + kind, source, subject = f.Kind, f.Source, f.Subject + } else { + kind, source, subject = carried.Kind, carried.Source, carried.Subject + } + if _, raisedOut := k.outOfHold[key]; raisedOut { + return held{}, false + } + for _, a := range k.aboutOf(kind, source, subject) { + m, w := k.motion(a.Node, a.Module) + if !m.Delivering || m.Ended() || k.passed[a.Node+"."+a.Module] == w.FP { + continue + } + since := k.Now().UTC() + if was, ok := k.held[key]; ok { + since = was.Since + } + return held{Key: key, Node: a.Node, Module: a.Module, Since: since, Version: w.FP}, true + } + return held{}, false +} + +// keepHeld keeps a held key in the state `held`, and starts its delivery's bound once. +func (k *Keeper) keepHeld(h held) error { + k.mu.Lock() + _, was := k.held[h.Key] + k.held[h.Key] = h + bound := k.bound(h.Module) + k.mu.Unlock() + if was { + return nil + } + if _, err := k.Store.Create(HeldState, h.Key, h); err != nil && !errors.Is(err, stdio.ErrChanged) { + return fmt.Errorf("keeping %s held: %w", h.Key, err) + } + b := boundKey(h.Node, h.Module) + err := k.Store.CreateTTL(BoundsState, b, map[string]any{"node": h.Node, "module": h.Module, "version": h.Version, + "bound-seconds": int(bound / time.Second)}, bound) + if err != nil && !errors.Is(err, stdio.ErrChanged) { + return fmt.Errorf("starting the bound of %s's delivery on %s: %w", h.Module, h.Node, err) + } + k.Logf("held %s while %s is delivered on %s (version %s), at most %s", h.Key, h.Module, h.Node, h.Version, bound) + return nil +} + +// bound is a module's delivery bound: its manifest names none yet, so the default (clause 13 (d)). Each key of the state +// `bounds` is written with it as its own time to live, and its expiry is the bound passing. +func (k *Keeper) bound(string) time.Duration { + if k.Bound > 0 { + return k.Bound + } + return delivery.DefaultDeliveryBound +} + +func boundKey(node, module string) string { return node + "." + module } + +// dropHeld lets a held key go: raised, or gone with why said. +func (k *Keeper) dropHeld(key, why string) { + k.mu.Lock() + h, ok := k.held[key] + delete(k.held, key) + k.mu.Unlock() + if !ok { + return + } + if err := k.Store.Delete(HeldState, key); err != nil { + k.Logf("the held %s could not be removed from held: %v", key, err) + } + if why != "" { + k.Logf("dropped %s, held while %s was delivered on %s: %s", key, h.Module, h.Node, why) + } +} + +// Want takes one change of the assigner's wants; Have one of the nodes' haves. Each may end a delivery, which raises or +// drops what it held. +func (k *Keeper) Want(key string, value []byte, deleted bool) error { + node, module, err := delivery.ParseWantKey(key) + if err != nil { + return nil // not a want of a module on a node (a held fingerprint's key) + } + k.mu.Lock() + k.init() + if deleted { + delete(k.wants, node+"."+module) + } else if w, err := delivery.DecodeWant(value); err == nil { + k.wants[node+"."+module] = w + } else { + k.Logf("the want %s is not readable: %v", key, err) + } + k.mu.Unlock() + return k.judge(node, module, "") +} + +// Have takes one change of a node's have. +func (k *Keeper) Have(key string, value []byte, deleted bool) error { + node, module, isNode, err := delivery.ParseHaveKey(key) + if err != nil || isNode { + return nil + } + k.mu.Lock() + k.init() + if deleted { + delete(k.haves, node+"."+module) + } else if h, err := delivery.DecodeHave(value); err == nil { + k.haves[node+"."+module] = h + } else { + k.Logf("the have %s is not readable: %v", key, err) + } + k.mu.Unlock() + return k.judge(node, module, "") +} + +// Placement takes one change of the placement: which module holds which seat on a node. +func (k *Keeper) Placement(key string, value []byte, deleted bool) { + node, err := delivery.ParsePlacementKey(key) + if err != nil { + return + } + k.mu.Lock() + defer k.mu.Unlock() + k.init() + for hk := range k.holders { + if strings.HasPrefix(hk, node+".") { + delete(k.holders, hk) + } + } + if deleted { + return + } + p, err := delivery.DecodePlacement(value) + if err != nil { + k.Logf("the placement of %s is not readable, so its seats' holders are not known: %v", node, err) + return + } + for _, h := range p.Holds { + k.holders[node+"."+h.Claim] = h.Module + } +} + +// BoundPassed is a delivery's bound key expiring: what it holds is raised, saying the delivery did not end. +func (k *Keeper) BoundPassed(key string) error { + node, module := splitNodeModule(key) + k.mu.Lock() + k.init() + _, w := k.motion(node, module) + holding := false + for _, h := range k.held { + holding = holding || (h.Node == node && h.Module == module) + } + if !holding { + k.mu.Unlock() + return nil + } + k.passed[key] = w.FP + k.mu.Unlock() + return k.judge(node, module, fmt.Sprintf("Its delivery of version %s on %s did not end within %s.", short(w.FP), node, + k.bound(module))) +} + +// judge raises or drops what node and module hold when their delivery ended, or with line past its bound. +func (k *Keeper) judge(node, module, line string) error { + k.mu.Lock() + m, w := k.motion(node, module) + var keys []string + for key, h := range k.held { + if h.Node == node && h.Module == module { + keys = append(keys, key) + } + } + if line == "" && !m.Ended() && m.Delivering { + k.mu.Unlock() + return nil + } + if line == "" { + line = fmt.Sprintf("It happened during the delivery of version %s of %s on %s.", short(w.FP), module, node) + if m.Failed { + line = fmt.Sprintf("It happened as the delivery of version %s of %s on %s failed.", short(w.FP), module, node) + } + delete(k.passed, node+"."+module) + } + for _, key := range keys { + k.outOfHold[key] = line + } + k.mu.Unlock() + sort.Strings(keys) + var problems []string + for _, key := range keys { + k.dropHeld(key, "") + if err := k.Settle(key); err != nil { + problems = append(problems, err.Error()) + } + k.mu.Lock() + delete(k.outOfHold, key) + k.mu.Unlock() + } + if len(problems) > 0 { + return errors.New(strings.Join(problems, "; ")) + } + return nil +} + +func short(fp string) string { + if _, rest, ok := strings.Cut(fp, ":"); ok { + fp = rest + } + if len(fp) > 12 { + return fp[:12] + } + return fp +} + +// Held is every key held now, for the read tool and compare. +func (k *Keeper) Held() []held { + k.mu.Lock() + defer k.mu.Unlock() + out := make([]held, 0, len(k.held)) + for _, h := range k.held { + out = append(out, h) + } + sort.Slice(out, func(i, j int) bool { return out[i].Key < out[j].Key }) + return out +} + +// restoreHeld takes the state `held` as read at the start. +func (k *Keeper) restoreHeld(key string, value []byte) { + var h held + if json.Unmarshal(value, &h) != nil { + return + } + k.mu.Lock() + k.init() + k.held[key] = h + k.mu.Unlock() +} diff --git a/modules/mesh-conditions/cmd/mesh-conditions/hold_test.go b/modules/mesh-conditions/cmd/mesh-conditions/hold_test.go new file mode 100644 index 00000000..fa668544 --- /dev/null +++ b/modules/mesh-conditions/cmd/mesh-conditions/hold_test.go @@ -0,0 +1,129 @@ +package main + +import ( + "encoding/json" + "strings" + "testing" + + "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/health" +) + +// The fingerprints and composition of mesh-controller on novox, copied from its have (mesh-delivery-view.node_haves, +// 2026-10-29): the delivery replayed moves it from the running fingerprint to the next, through the node-engine's phases +// (delivery.go Phase*). No tool reads a have's or a want's history yet, so the sequence is the node-engine's documented +// order, not a capture; the values are captured. +const ( + ctrlRunning = "src2:fcaa7d7ccb35ea2beed5b9e4b87f4c24c99452244d49f4786d4022e5c543f39c" + ctrlComposed = "sha256:e9a4de75c9ca70a32cef3137cb0e6692b0f9558b699013caaee399d88dd1bac0" + ctrlNext = "src2:0d1e2f3a4b5c6d7e8f90a1b2c3d4e5f60718293a4b5c6d7e8f9001a2b3c4d5e6" + ctrlNextComp = "sha256:1f2e3d4c5b6a79881726354453627180918a7b6c5d4e3f2a1b0c9d8e7f605142" +) + +// ADR 0564 clause 13 and Limits 12 (issue 1952 C7, C8): a delivery of mesh-controller on novox replayed want by have. +// Its own findings and the carried controller-deaf raise nothing during it and none after when they went; one still +// standing at the end is raised once with its during-the-delivery line; one past the bound with its did-not-end line; a +// failed delivery raises what stands at once; another module's and the machine's own are raised at once; compare lists a +// held key as held. +func TestADeliveryHoldsItsOwnFindingsUntilItEnds(t *testing.T) { + want := func(fp, composed string) []byte { + raw, _ := json.Marshal(delivery.Want{V: delivery.Version, FP: fp, Composed: composed}) + return raw + } + have := func(fp, composed string, phase delivery.Phase, h delivery.Health) []byte { + raw, _ := json.Marshal(delivery.Have{V: delivery.Version, FP: fp, Composed: composed, Phase: phase, Health: h}) + return raw + } + deaf, _ := json.Marshal(health.Condition{Key: "core.controller.deaf", Kind: "controller-deaf", Source: "S4", + Subject: health.Subject{Scope: health.ScopeCore, ID: "controller"}, Severity: health.Urgent, Summary: "deaf", Resolver: "self"}) + for _, c := range []struct { + name string + // end is the have the delivery ends with, or nil for a delivery past its bound. + end []byte + standing bool + said string + line string + }{ + {name: "gone by its end", end: have(ctrlNext, ctrlNextComp, delivery.PhaseRunning, delivery.HealthOK)}, + {name: "still standing at its end", end: have(ctrlNext, ctrlNextComp, delivery.PhaseRunning, delivery.HealthOK), + standing: true, said: "raised module.mesh-controller.novox.unhealthy", line: "during the delivery of version 0d1e2f3a4b5c"}, + {name: "past its bound", standing: true, said: "raised module.mesh-controller.novox.unhealthy", line: "did not end within"}, + {name: "failed", end: have(ctrlNext, ctrlNextComp, delivery.PhaseFailed, delivery.HealthFailed), standing: true, + said: "raised module.mesh-controller.novox.unhealthy", line: "as the delivery of version 0d1e2f3a4b5c"}, + } { + r := newRig() + _ = r.k.Want("want.novox.mesh-controller", want(ctrlRunning, ctrlComposed), false) + _ = r.k.Have("novox.mesh-controller", have(ctrlRunning, ctrlComposed, delivery.PhaseRunning, delivery.HealthOK), false) + // The delivery starts: the assigner wants the next fingerprint; the node applies it. + _ = r.k.Want("want.novox.mesh-controller", want(ctrlNext, ctrlNextComp), false) + _ = r.k.Have("novox.mesh-controller", have(ctrlNext, ctrlNextComp, delivery.PhaseApplying, delivery.HealthUnknown), false) + own, ownRaw := finding(t, watchman, health.ScopeModule, "mesh-controller.novox", "unhealthy", "health", health.Warning, r.now) + other, otherRaw := finding(t, watchman, health.ScopeModule, "postgres.novox", "unhealthy", "health", health.Warning, r.now) + machine, machineRaw := finding(t, watchman, health.ScopeMachine, "novox", "silent", "S1", health.Urgent, r.now) + for _, f := range []struct { + key string + raw []byte + }{{own, ownRaw}, {other, otherRaw}, {machine, machineRaw}} { + if err := r.k.Finding(watchman, f.key, f.raw, false); err != nil { + t.Fatal(err) + } + } + if err := r.k.Controller("core.controller.deaf", deaf, false); err != nil { + t.Fatal(err) + } + // The delivery's bound is its own key's time to live, the module's bound (the default where it names none). + if r.st.ttls["bounds/novox.mesh-controller"] != delivery.DefaultDeliveryBound { + t.Errorf("%s: the bound key lives %s", c.name, r.st.ttls["bounds/novox.mesh-controller"]) + } + if got := r.changes(); got != "raised "+other+", raised "+machine { + t.Errorf("%s: during the delivery said %q: another module's and the machine's at once, nothing of its own", c.name, got) + } + // C8: compare lists the held keys as held, never as a difference. + cmp, _ := r.k.Compare() + held := 0 + for _, d := range cmp.Apart { + if strings.HasPrefix(d.Reason, "held while mesh-controller is delivered on novox") { + held++ + } + } + if held != 2 { + t.Errorf("%s: compare lists %d held, %+v", c.name, held, cmp) + } + // The carried condition about the controller goes within the delivery; its own finding goes unless it stands. + _ = r.k.Controller("core.controller.deaf", nil, true) + if !c.standing { + _ = r.k.Finding(watchman, own, nil, true) + } + if c.end != nil { + _ = r.k.Have("novox.mesh-controller", c.end, false) + } else { + r.now = r.now.Add(delivery.DefaultDeliveryBound) + _ = r.k.BoundPassed("novox.mesh-controller") + } + if got := r.changes(); got != c.said { + t.Errorf("%s: at its end said %q, want %q", c.name, got, c.said) + } + if co, open := r.open(own); c.standing != open || (open && !strings.Contains(co.Explanation, c.line)) { + t.Errorf("%s: its own condition is %v: %q", c.name, open, co.Explanation) + } + if _, open := r.open("core.controller.deaf"); open || len(r.k.Held()) != 0 { + t.Errorf("%s: the carried deaf condition was raised, or something is still held: %+v", c.name, r.k.Held()) + } + } + // A seat's finding is about the module the placement says holds the seat on that node (the lead, 2026-10-29): held + // while the assigner is delivered on ace, though the seat is named mesh-placement. + r := newRig() + placed, err := delivery.EncodePlacement(delivery.Placement{Node: "ace", Modules: []delivery.PlacedModule{{Module: "mesh-delivery-assigner"}}, + Holds: []delivery.Holding{{Claim: "mesh-placement", Scope: "mesh", Module: "mesh-delivery-assigner"}}}) + if err != nil { + t.Fatal(err) + } + r.k.Placement("placement.ace", placed, false) + _ = r.k.Want("want.ace.mesh-delivery-assigner", want(ctrlNext, ctrlNextComp), false) + _ = r.k.Have("ace.mesh-delivery-assigner", have(ctrlRunning, ctrlComposed, delivery.PhaseRunning, delivery.HealthOK), false) + seat, seatRaw := finding(t, selfCheck, health.ScopeSeat, "mesh-placement.ace", "holder-silent", "D3", health.Warning, r.now) + _ = r.k.Finding(selfCheck, seat, seatRaw, false) + if got := r.changes(); got != "" || len(r.k.Held()) != 1 || r.k.Held()[0].Module != "mesh-delivery-assigner" { + t.Errorf("a seat's finding while its holder is delivered: said %q, held %+v", got, r.k.Held()) + } +} diff --git a/modules/mesh-conditions/cmd/mesh-conditions/homes.go b/modules/mesh-conditions/cmd/mesh-conditions/homes.go new file mode 100644 index 00000000..06ad8187 --- /dev/null +++ b/modules/mesh-conditions/cmd/mesh-conditions/homes.go @@ -0,0 +1,160 @@ +package main + +// **Where each condition kind lives after the take-over** (novox/hq ADR 0564 clauses 4, 6, 7 and 9, the Context's probe +// table; issue 1952 C3, C5): data, so a kind's home is a row and never code. `compare` reads it to tell a difference from +// a kind that ends with the controller or that the controller never raises, and the keeper reads it to carry, until the +// lease write, the controller's conditions of the kinds that end with it. +// +// The rows are ADR 0564's and the lead's table of 2026-10-29, by the controller's source of each kind. A kind with no +// row is named by `compare` as "no home decided". + +// Home is where one condition kind lives. +type Home struct { + // Source is the controller's row or probe that raises it today, for whoever reads the table. + Source string + // Controller says the controller raises this kind today: a kind it never raises is the keeper's alone and is no + // difference in compare. + Controller bool + // Observer is the module whose findings raise it after the take-over; empty for a kind that ends. + Observer string + // Carried says it ends with the controller's lease and is carried from the controller's bucket until then + // (clause 9: S4, S7, S12, S13, S14, S25, D5, D9, H-controller). + Carried bool + // Replaced says the kind itself ends and another observes its fault (D1, D10, the stuck probe): listed apart in + // compare and not carried. + Replaced bool + // Why is why a kind is carried when the record's table gave it another home: said by compare beside it. + Why string +} + +// Dropped is the reason of the probes the operator dropped from this release (D2, D3, D4, D12, D-heard, D-root, D11, +// and D13's data held twice): their kinds end with the controller and are carried until the lease write. +const Dropped = "dropped 2026-10-30 by the operator; the owning module declares its own check later (research 074)" + +// The kinds' homes, by kind: ADR 0564's rows, and the 63 kinds the controller's code raises that the record left without +// one, as the lead settled them under the operator's yes on 2026-10-29 (.work/47-release/kinds-section.html, with the +// lead's three changes: lease-split and image-missing do not end, and the data rows read their readings from disk-load). +var Homes = map[string]Home{ + // The watchman's watchdogs (clause 4). + "silent": {Source: "S1", Controller: true, Observer: watchman}, + "max-deliveries": {Source: "S9", Controller: true, Observer: watchman}, + "slow-consumer": {Source: "S9", Controller: true, Observer: watchman}, + "refused": {Source: "S9", Controller: true, Observer: watchman}, + "connection-closed": {Source: "S9", Controller: true, Observer: watchman}, + "bus-faults-unread": {Source: "S9", Controller: true, Observer: watchman}, + "tools-silent": {Source: "S11", Controller: true, Observer: watchman}, + "apply-failing": {Source: "S21", Controller: true, Observer: watchman}, + "module-unhealthy": {Source: "health", Controller: true, Observer: watchman}, + "stuck": {Source: "DM", Controller: true, Observer: watchman}, + "provider-failing": {Source: "S8", Controller: true, Observer: watchman}, + "provider-silent": {Source: "S8", Controller: true, Observer: watchman}, + "self-check-silent": {Source: "S10", Controller: true, Observer: watchman}, + // From the provision keys in waits (ADR 0570 §7), a setting's wait from the node's own health report. + "needs-operator": {Source: "health", Controller: true, Observer: watchman}, + // From the vault's record of each secret given (ADR 0570 §7). + "secret-given": {Source: "desk", Controller: true, Observer: watchman}, + // The watchman reading each node's own health report key (a4's, mesh-host). + "machine-network": {Source: "node", Controller: true, Observer: watchman}, + "network-rewritten": {Source: "node", Controller: true, Observer: watchman}, + "network-unreachable": {Source: "node", Controller: true, Observer: watchman}, + "machine-units": {Source: "node", Controller: true, Observer: watchman}, + "relogin-needed": {Source: "health", Controller: true, Observer: watchman}, + "directory-used-as-found": {Source: "health", Controller: true, Observer: watchman}, + // The self-check's probes (clause 6, the Context's table). + "lease-split": {Source: "D5", Controller: true, Observer: selfCheck}, // a seat held once answers from two holders + "consumer-wrong": {Source: "D6", Controller: true, Observer: selfCheck}, + "consumer-lost": {Source: "D6", Controller: true, Observer: selfCheck}, // and the watchman's S9 + "consumer-behind": {Source: "D6", Controller: true, Observer: selfCheck}, + "stream-wrong": {Source: "D7", Controller: true, Observer: selfCheck}, + "own-address-banned": {Source: "D8", Controller: true, Observer: selfCheck}, + "data-unmeasured": {Source: "D13", Controller: true, Observer: selfCheck}, + "data-shrank": {Source: "D13", Controller: true, Observer: selfCheck}, // readings from disk-load (L7) + "empty-replacement": {Source: "D13", Controller: true, Observer: selfCheck}, + "data-quiet": {Source: "D13", Controller: true, Observer: selfCheck}, + "backup-stale": {Source: "D13", Controller: true, Observer: selfCheck}, + "data-missing": {Source: "D13", Controller: true, Observer: selfCheck}, + "array-degraded": {Source: "D13", Controller: true, Observer: selfCheck}, + "protection-missing": {Source: "D13", Controller: true, Observer: selfCheck}, + "bus-reconnects": {Source: "D15", Controller: true, Observer: selfCheck}, + "bucket-or-log-filling": {Source: "fill", Controller: true, Observer: selfCheck}, + "bucket-or-log-unread": {Source: "fill", Controller: true, Observer: selfCheck}, + "watchdogs-silent": {Source: "DW", Controller: true, Observer: selfCheck}, + "core-unhealthy": {Source: "H-bus", Controller: true, Observer: selfCheck}, + "agent-can-become-root": {Source: "DA", Controller: true, Observer: selfCheck}, + "probe-failed": {Source: "doctor", Controller: true, Observer: selfCheck}, + // The healers' own (clause 7, Limits 2). + "healer-wanted": {Source: "S15", Controller: true, Observer: healers}, + "healers-braked": {Source: "healers", Controller: true, Observer: healers}, + // The assigner's, as its own findings (the bus step, ADR 0565 §9). + "bus-maintenance": {Source: "bus", Controller: true, Observer: assigner}, + "bus-upgrade-failed": {Source: "DB", Controller: true, Observer: assigner}, + // The provisioner's (ADR 0570 §5): its own ask replaces retire-waiting; a declined key stays its finding. + "retire-waiting": {Source: "retire", Controller: true, Observer: provisioner}, + "retire-rejected": {Source: "retire", Controller: true, Observer: provisioner}, + // The router's own: no channel delivered a question. + "asks-undelivered": {Source: "asks", Controller: true, Observer: messenger}, + // The seat mesh-artifact-sweep (L7): an image a want names is missing from the store, artifact-store-tools' finding. + "image-missing": {Source: "S25", Controller: true, Observer: artifactStore}, + // The kinds that end with the controller and are carried until its lease write (clauses 4, 9). + "controller-deaf": {Source: "S4", Controller: true, Carried: true}, + "lease-lost": {Source: "S12", Controller: true, Carried: true}, + "call-hung": {Source: "S7", Controller: true, Carried: true}, + "stale-writer": {Source: "S13", Controller: true, Carried: true}, + "facts-stale": {Source: "S14", Controller: true, Carried: true}, + "status-slow": {Source: "D9", Controller: true, Carried: true}, + "vault-silent": {Source: "send", Controller: true, Carried: true}, + "declaration-files-not-put": {Source: "send", Controller: true, Carried: true}, + "config-not-followed": {Source: "config", Controller: true, Carried: true}, + "assignment-not-made": {Source: "assign", Controller: true, Carried: true}, + "builds-not-laid-out": {Source: "start", Controller: true, Carried: true}, + "sent-builds-not-copied": {Source: "start", Controller: true, Carried: true}, + "fingerprint-not-composed": {Source: "done keys", Controller: true, Carried: true}, + "take-in-not-consumed": {Source: "take-in", Controller: true, Carried: true}, + "take-in-stalled": {Source: "take-in", Controller: true, Carried: true}, + "new-module-waits": {Source: "register", Controller: true, Carried: true}, + "left-out": {Source: "send", Controller: true, Carried: true}, + "declaration-refused": {Source: "D1", Controller: true, Carried: true}, + "unknown-field": {Source: "read", Controller: true, Carried: true}, + "bus-objects-unasserted": {Source: "send", Controller: true, Carried: true}, + // The probes the operator dropped from this release (D2 resolvers, D3 holders, D4 archives, D12 bindings, D-heard + // withheld hearings, D-root, D11 retired data, and D13's data held twice): their kinds are carried until the lease + // write. + "resolver-wrong": {Source: "D2", Controller: true, Carried: true, Why: Dropped}, + "holder-silent": {Source: "D3", Controller: true, Carried: true, Why: Dropped}, + "archives-missing": {Source: "D4", Controller: true, Carried: true, Why: Dropped}, + "archives-unheld": {Source: "D4", Controller: true, Carried: true, Why: Dropped}, + "binding-kept": {Source: "D12", Controller: true, Carried: true, Why: Dropped}, + "binding-moved": {Source: "D12", Controller: true, Carried: true, Why: Dropped}, + "binding-moving": {Source: "D12", Controller: true, Carried: true, Why: Dropped}, + "grant-composes-nothing": {Source: "withheld", Controller: true, Carried: true, Why: Dropped}, + "root-not-free": {Source: "D-root", Controller: true, Carried: true, Why: Dropped}, + "cleanup-waiting": {Source: "D11", Controller: true, Carried: true, Why: Dropped}, + "data-held-twice": {Source: "D13", Controller: true, Carried: true, Why: Dropped}, + // The lead, 2026-10-29: no delivery plans since R6 (wants replace them), and a stalled delivery is the watchman's + // stuck finding seen directly. + "plan-failed": {Source: "S23", Controller: true, Carried: true}, + "stalled": {Source: "D14", Controller: true, Carried: true}, + // The kinds that end, their faults observed otherwise (the Context's table). + "core-behind": {Source: "D10", Controller: true, Replaced: true}, +} + +// The observers, as the table names them. +const ( + watchman = "mesh-delivery-watchman" + selfCheck = "mesh-self-check" + healers = "mesh-healers" + assigner = "mesh-delivery-assigner" + provisioner = "mesh-provisioner" + messenger = "messenger" + artifactStore = "artifact-store-tools" +) + +// homeOf is the home of a condition's kind, and whether the table has one. +func (k *Keeper) homeOf(kind string) (Home, bool) { + homes := k.Homes + if homes == nil { + homes = Homes + } + h, ok := homes[kind] + return h, ok +} diff --git a/modules/mesh-conditions/cmd/mesh-conditions/keeper.go b/modules/mesh-conditions/cmd/mesh-conditions/keeper.go new file mode 100644 index 00000000..a167631c --- /dev/null +++ b/modules/mesh-conditions/cmd/mesh-conditions/keeper.go @@ -0,0 +1,668 @@ +package main + +// The condition keeper (novox/hq ADR 0564 clauses 2, 3, 7, 8, 9 and 13; issue 1952 C2, C3, C7): the one writer of the +// mesh's open conditions. It watches every observer's `findings`, the healers' `attempts`, the controller's lease and, +// until the lease write, the controller's `conditions`, and the assigner's wants and the nodes' haves for the hold. From +// all of them it derives, per key, the condition that should be open, and writes the difference by compare-and-set, +// saying each transition as the controller says it (go/health) and keeping it in `history`. Nothing clears a condition +// but its finding going (ADR 0231 §14). + +import ( + "encoding/json" + "errors" + "fmt" + "sort" + "strings" + "sync" + "time" + + stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/health" +) + +// The module, its seat, and the states it keeps. +const ( + Module = health.KeeperModule + Seat = "mesh-conditions" + + ConditionsState = health.ConditionsState + TakeoverState = health.TakeoverState + HistoryState = "history" + HeldState = "held" + BoundsState = "bounds" + // AskedState keeps the keeper's asks of the operator (go/asker). + AskedState = "asked" +) + +// HistoryKeptFor is how long a transition is kept (clause 3). +const HistoryKeptFor = 90 * 24 * time.Hour + +// Observers are the modules whose `findings` the keeper reads (clause 2; ADR 0565 §8 for the assigner's; the lead's +// table of 2026-10-29 for the provisioner's and the router's; ADR 0571 clause 2 for the seat artifact-store's holder, +// which observes S25, image-missing). +var Observers = []string{health.WatchmanModule, health.SelfCheckModule, health.HealersModule, delivery.AssignerModule, + "mesh-provisioner", "messenger", artifactStore} + +// The other states it reads, as `.`. +const ( + AttemptsRead = health.HealersModule + "." + health.AttemptsState + ControllerConditions = "mesh-controller.conditions" + ControllerLease = "mesh-controller.lease" + ControllerLeaseKey = "holder" + WantsRead = delivery.AssignerModule + ".wants" + // PlacementRead is the mesh-placement seat's placement, read for which module holds a seat on a node (hold.go; the + // lead, 2026-10-29, for ADR 0564 clause 2's inputs). + PlacementRead = delivery.PlacementRead + HavesRead = "node-engine.have" + ConditionWrites = 8 +) + +// Store is the bus as the keeper uses it: its own states, written by compare-and-set, and a key with its own time to +// live for a hold's bound. +type Store interface { + Get(state, key string) (value []byte, revision uint64, found bool, err error) + Create(state, key string, value any) (uint64, error) + Update(state, key string, value any, revision uint64) (uint64, error) + Delete(state, key string) error + Keys(state string) ([]string, error) + CreateTTL(state, key string, value any, ttl time.Duration) error +} + +// held is a finding or a carried condition kept while its module is delivered (clause 13 (b)). +type held struct { + Key string `json:"key"` + Node string `json:"node"` + Module string `json:"module"` + Since time.Time `json:"since"` + Version string `json:"version"` + Wanted json.RawMessage `json:"-"` +} + +// clearedMemo is what a cleared condition leaves for ReopenWithin: the same fault again is a reopening. +type clearedMemo struct { + at time.Time + count int + began time.Time + gaps []health.Gap + silenced *health.Silence + tried []health.Tried +} + +// Keeper is the condition keeper's state and its writes. +type Keeper struct { + Store Store + // Say publishes one event on the seat; History keeps one transition. + Say func(e health.Event) error + Now func() time.Time + Logf func(format string, a ...any) + // Homes says, by kind, where each controller kind lives after the take-over (homes.go); Homes when nil. + Homes map[string]Home + // Bound is a delivery's hold bound; delivery.DefaultDeliveryBound when zero (hold.go). + Bound time.Duration + + mu sync.Mutex + findings map[string]map[string]health.Finding + blind map[string]string + attempts map[string]health.Attempt + ctrl map[string]health.Condition + leaseOut bool + takeover health.Takeover + wants map[string]delivery.Want + haves map[string]delivery.Have + // holders are, by `.`, the module the placement says holds the seat there. + holders map[string]string + stuck map[string]string + held map[string]held + // passed are the deliveries whose bound passed, by node and module, at the fingerprint wanted then; outOfHold the + // line a key raised out of a hold says (hold.go). + passed map[string]string + outOfHold map[string]string + cleared map[string]clearedMemo + seq uint64 +} + +func (k *Keeper) init() { + if k.findings == nil { + k.findings, k.blind, k.attempts, k.ctrl = map[string]map[string]health.Finding{}, map[string]string{}, + map[string]health.Attempt{}, map[string]health.Condition{} + k.wants, k.haves, k.stuck, k.held, k.cleared = map[string]delivery.Want{}, map[string]delivery.Have{}, + map[string]string{}, map[string]held{}, map[string]clearedMemo{} + k.takeover = health.Takeover{V: health.TakeoverVersion, Follow: health.FollowController} + k.passed, k.outOfHold, k.holders = map[string]string{}, map[string]string{}, map[string]string{} + } + if k.Now == nil { + k.Now = time.Now + } + if k.Logf == nil { + k.Logf = func(string, ...any) {} + } +} + +// Finding takes one change of an observer's finding: a key put raises or changes its condition, a key deleted clears +// it (clause 2). A value that is not a finding is the observer's fault, said and left as it was. +func (k *Keeper) Finding(observer, key string, value []byte, deleted bool) error { + k.mu.Lock() + k.init() + if k.findings[observer] == nil { + k.findings[observer] = map[string]health.Finding{} + } + if deleted { + delete(k.findings[observer], key) + } else { + f, err := health.DecodeFinding(value) + if err != nil { + k.mu.Unlock() + k.Logf("%s's finding %s is not one the keeper can raise, so it is left as it stood: %v", observer, key, err) + return nil + } + if f.Key != key { + k.mu.Unlock() + k.Logf("%s's finding under %s names the key %s, so it is left as it stood", observer, key, f.Key) + return nil + } + k.findings[observer][key] = f + } + k.mu.Unlock() + return k.Settle(key) +} + +// Blind says an observer's findings cannot be read: its keys stay as they were, and nothing is cleared for not seeing +// (ADR 0227 §4). Empty why says it is read again. +func (k *Keeper) Blind(observer, why string) { + k.mu.Lock() + defer k.mu.Unlock() + k.init() + if why == "" { + delete(k.blind, observer) + } else { + k.blind[observer] = why + } +} + +// Attempt takes one change of the healers' `attempts`: the condition it names gets its tried, resolver and escalation +// (clause 7). +func (k *Keeper) Attempt(key string, value []byte, deleted bool) error { + k.mu.Lock() + k.init() + var condition string + if old, ok := k.attempts[key]; ok { + condition = old.Condition + } + if deleted { + delete(k.attempts, key) + } else { + var a health.Attempt + if err := json.Unmarshal(value, &a); err != nil || a.Condition == "" { + k.mu.Unlock() + k.Logf("the healers' attempt %s is not readable, so it is left out: %v", key, err) + return nil + } + k.attempts[key], condition = a, a.Condition + } + k.mu.Unlock() + if condition == "" { + return nil + } + return k.Settle(condition) +} + +// Controller takes one change of the controller's `conditions` bucket (clauses 8, 9): read until the lease write. +func (k *Keeper) Controller(key string, value []byte, deleted bool) error { + k.mu.Lock() + k.init() + if deleted { + delete(k.ctrl, key) + } else { + var c health.Condition + if err := json.Unmarshal(value, &c); err != nil { + k.mu.Unlock() + k.Logf("the controller's condition %s is not readable, so nothing is carried from it: %v", key, err) + return nil + } + k.ctrl[key] = c + } + k.mu.Unlock() + return k.Settle(key) +} + +// Lease takes the controller's lease key as read (clause 8, Limits 8): once it is handed to a holder that is no +// controller, nothing more is carried from the controller's bucket, and the kinds that ended with it clear. +func (k *Keeper) Lease(away bool) error { + k.mu.Lock() + k.init() + was := k.leaseOut + k.leaseOut = away + keys := make([]string, 0, len(k.ctrl)) + for key := range k.ctrl { + keys = append(keys, key) + } + k.mu.Unlock() + if was == away { + return nil + } + return k.settleAll(keys) +} + +// LeaseAway says whether the controller's lease is handed away. +func (k *Keeper) LeaseAway() bool { + k.mu.Lock() + defer k.mu.Unlock() + return k.leaseOut +} + +// Takeover takes the keeper's own take-over key as read. +func (k *Keeper) Takeover(t health.Takeover) { + k.mu.Lock() + defer k.mu.Unlock() + k.init() + k.takeover = t +} + +// Settle derives the condition key should be and writes the difference (clause 2), retried while another write moves +// it under this one. +func (k *Keeper) Settle(key string) error { + var err error + for i := 0; i < ConditionWrites; i++ { + if err = k.settleOnce(key); !errors.Is(err, stdio.ErrChanged) { + return err + } + } + return fmt.Errorf("the condition %s kept moving under this write, %d tries: %w", key, ConditionWrites, err) +} + +func (k *Keeper) settleAll(keys []string) error { + sort.Strings(keys) + var problems []string + for _, key := range keys { + if err := k.Settle(key); err != nil { + problems = append(problems, err.Error()) + } + } + if len(problems) > 0 { + return errors.New(strings.Join(problems, "; ")) + } + return nil +} + +// wanted is what the inputs say key should be: a finding of an observer, or a condition carried from the controller's +// bucket of a kind that ends with the controller, until the lease write (clause 9). Nil when nothing says it. +func (k *Keeper) wanted(key string) (*health.Finding, *health.Condition) { + observers := make([]string, 0, len(k.findings)) + for o := range k.findings { + observers = append(observers, o) + } + sort.Strings(observers) + for _, o := range observers { + if f, ok := k.findings[o][key]; ok { + return &f, nil + } + } + if c, ok := k.ctrl[key]; ok && !k.leaseOut && k.endsWithController(c) { + return nil, &c + } + return nil, nil +} + +func (k *Keeper) endsWithController(c health.Condition) bool { + h, ok := k.homeOf(c.Kind) + return ok && h.Carried +} + +// settleOnce is one try of Settle. +func (k *Keeper) settleOnce(key string) error { + now := k.Now().UTC() + raw, rev, open, err := k.Store.Get(ConditionsState, key) + if err != nil { + return fmt.Errorf("reading the condition %s: %w", key, err) + } + var was health.Condition + if open { + if err := json.Unmarshal(raw, &was); err != nil { + return fmt.Errorf("the condition %s on the bus cannot be read: %w", key, err) + } + } + k.mu.Lock() + k.init() + f, carried := k.wanted(key) + hold, holding := k.holdFor(key, f, carried, open) + k.mu.Unlock() + + if f == nil && carried == nil { + k.dropHeld(key, "its finding went while its module was delivered") + if !open { + return nil + } + return k.clear(key, was, rev, now, "its finding went") + } + if holding { + return k.keepHeld(hold) + } + k.dropHeld(key, "") + next := k.derive(key, was, open, f, carried, now) + events := transitions(was, next, open) + if len(events) == 0 && open && sameBody(was, next) { + return nil + } + if open { + _, err = k.Store.Update(ConditionsState, key, next, rev) + } else { + _, err = k.Store.Create(ConditionsState, key, next) + } + if err != nil { + return err + } + k.mu.Lock() + delete(k.cleared, key) + k.mu.Unlock() + for _, e := range events { + e.Condition, e.At, e.Event = next, now, health.EventFor(e.Change) + e.Show = k.show(next, carried != nil) + k.tell(e) + } + return nil +} + +func (k *Keeper) show(c health.Condition, carried bool) string { + if carried { + return health.ShowController(c.Key) + } + return health.ShowKeeper(c.Key) +} + +// derive is the condition key should be now, from what was open and what the inputs say. +func (k *Keeper) derive(key string, was health.Condition, open bool, f *health.Finding, carried *health.Condition, now time.Time) health.Condition { + var c health.Condition + switch { + case carried != nil: + // Raised and cleared as the controller's bucket says, with its show and actions (Limits 9). + c = *carried + c.Epoch = 0 + c.LooksUnread = "" + case open: + c = was + k.fromFinding(&c, *f, now) + default: + c = health.Condition{Key: key, Raised: now, Count: 1, Resolver: health.ResolverSelf} + k.fromFinding(&c, *f, now) + k.mu.Lock() + if before, ok := k.cleared[key]; ok && now.Sub(before.at) <= health.ReopenWithin { + c.Count = before.count + 1 + c.First = before.began + c.Gaps = append(append([]health.Gap(nil), before.gaps...), health.Gap{Cleared: before.at, Reopened: now}) + if over := len(c.Gaps) - health.KeptGaps; over > 0 { + c.First, c.Gaps = c.Gaps[over-1].Reopened, c.Gaps[over:] + } + if before.silenced != nil && now.Before(before.silenced.Until) { + c.Silenced = before.silenced + } + c.Tried = before.tried + } + k.mu.Unlock() + } + if carried == nil { + k.overlay(&c, now) + } + if !open { + k.mu.Lock() + if line := k.outOfHold[key]; line != "" { + c.Explanation = strings.TrimSpace(c.Explanation + " " + line) + } + k.mu.Unlock() + } + if c.Silenced != nil && !now.Before(c.Silenced.Until) { + c.Silenced = nil + } + return c +} + +// fromFinding is what the condition says of its finding: its words, subject, severity and source, and its evidence +// newest first (clause 2). A healer's escalation is not undone by the finding saying a warning again. +func (k *Keeper) fromFinding(c *health.Condition, f health.Finding, now time.Time) { + escalated := c.Escalated() + c.Kind, c.Subject, c.Summary, c.Source = f.Kind, f.Subject, f.Summary, f.Source + c.Headline, c.Explanation, c.Resolved, c.Needs, c.Actions = f.Headline, f.Explanation, f.Resolved, f.Needs, f.Actions + if !(escalated && f.Severity != health.Urgent) { + c.Severity = f.Severity + } + if !escalated { + c.Resolver = f.Resolver + if c.Resolver == "" { + c.Resolver = health.ResolverSelf + } + } + if len(f.Evidence) > 0 { + c.Evidence = append([]health.Evidence(nil), f.Evidence...) + c.LastObserved = f.Evidence[0].At + } else if c.Evidence == nil { + c.Evidence = []health.Evidence{{At: f.At, Said: f.Summary}} + c.LastObserved = f.At + } + c.Observations = len(c.Evidence) + if c.Actions == nil { + c.Actions = []health.Action{} + } + _ = now +} + +// overlay writes the healers' attempts into the condition (clause 7) and, until the lease write, what the controller's +// bucket holds for the same key (clause 8): its tried, its resolver and escalation, and its silence where the keeper has +// none of its own. +func (k *Keeper) overlay(c *health.Condition, now time.Time) { + k.mu.Lock() + var mine []health.Attempt + for _, a := range k.attempts { + if a.Condition == c.Key { + mine = append(mine, a) + } + } + ctrl, carry := k.ctrl[c.Key] + carry = carry && !k.leaseOut + k.mu.Unlock() + sort.Slice(mine, func(i, j int) bool { return mine[i].Begun.Before(mine[j].Begun) }) + if carry && len(ctrl.Tried) > 0 { + c.Tried = append([]health.Tried(nil), ctrl.Tried...) + if ctrl.Escalated() { + c.Resolver, c.Severity = health.ResolverOperator, health.Urgent + } else if ctrl.Resolver != "" && ctrl.Resolver != health.ResolverSelf { + c.Resolver = ctrl.Resolver + } + } + for _, a := range mine { + t := a.Tried() + if !containsTried(c.Tried, t) { + c.Tried = append(c.Tried, t) + } + switch a.Outcome { + case health.OutcomeEscalated: + c.Resolver, c.Severity = health.ResolverOperator, health.Urgent + default: + if c.Resolver != health.ResolverOperator { + c.Resolver = health.ResolverHealer(a.Healer) + } + } + } + if len(c.Tried) > health.KeptTried { + c.Tried = c.Tried[len(c.Tried)-health.KeptTried:] + } + if carry && c.Silenced == nil && ctrl.Silenced != nil && now.Before(ctrl.Silenced.Until) { + c.Silenced = ctrl.Silenced + } +} + +func containsTried(list []health.Tried, t health.Tried) bool { + for _, x := range list { + if x.At.Equal(t.At) && x.By == t.By && x.What == t.What { + return true + } + } + return false +} + +// transitions are the events between what was open and what is now, in the controller's order: raised or reopened; +// else severity, resolver, silence and words changes. An observation that changes none of these says nothing. +func transitions(was, now health.Condition, open bool) []health.Event { + if !open { + change := health.ChangeRaised + if now.Count > 1 { + change = health.ChangeReopened + } + return []health.Event{{Change: change}} + } + var out []health.Event + if was.Severity != now.Severity { + out = append(out, health.Event{Change: health.ChangeSeverity, Was: string(was.Severity)}) + } + if was.Resolver != now.Resolver { + out = append(out, health.Event{Change: health.ChangeResolver, Was: was.Resolver}) + } + switch { + case was.Silenced == nil && now.Silenced != nil: + out = append(out, health.Event{Change: health.ChangeSilenced, Why: now.Silenced.Why}) + case was.Silenced != nil && now.Silenced == nil: + out = append(out, health.Event{Change: health.ChangeUnsilenced}) + } + if was.Needs != now.Needs || !sameActions(was.Actions, now.Actions) { + out = append(out, health.Event{Change: health.ChangeWords, Was: was.Needs}) + } + return out +} + +func sameActions(a, b []health.Action) bool { + x, _ := json.Marshal(a) + y, _ := json.Marshal(b) + return string(x) == string(y) || (len(a) == 0 && len(b) == 0) +} + +func sameBody(a, b health.Condition) bool { + x, _ := json.Marshal(a) + y, _ := json.Marshal(b) + return string(x) == string(y) +} + +// clear removes an open condition whose finding went, and says it (clause 2). +func (k *Keeper) clear(key string, was health.Condition, rev uint64, now time.Time, why string) error { + // A delete at the revision read: a write that moved it meanwhile is read again. + if _, err := k.Store.Update(ConditionsState, key, json.RawMessage("null"), rev); err != nil { + return err + } + if err := k.Store.Delete(ConditionsState, key); err != nil { + return err + } + k.mu.Lock() + k.cleared[key] = clearedMemo{at: now, count: was.Count, began: was.Began(), gaps: was.Gaps, silenced: was.Silenced, tried: was.Tried} + k.mu.Unlock() + cleared := now + k.tell(health.Event{Condition: was, Event: health.EventCleared, At: now, Change: health.ChangeCleared, Why: why, + Cleared: &cleared, Show: k.show(was, false)}) + return nil +} + +// tell keeps the transition in history and says it on the seat. A transition not said is said in the log: the condition +// and its history still hold it. +func (k *Keeper) tell(e health.Event) { + k.mu.Lock() + k.seq++ + hk := fmt.Sprintf("%020d-%06d", e.At.UnixNano(), k.seq) + k.mu.Unlock() + if _, err := k.Store.Create(HistoryState, hk, e); err != nil { + k.Logf("the %s of %s could not be kept in history: %v", e.Change, e.Key, err) + } + if k.Say == nil { + return + } + if err := k.Say(e); err != nil { + k.Logf("the %s of %s could NOT be said on the seat: %v", e.Change, e.Key, err) + } +} + +// Open is every open condition, ordered as `conditions` says them. +func (k *Keeper) Open() ([]health.Condition, error) { + keys, err := k.Store.Keys(ConditionsState) + if err != nil { + return nil, err + } + var out []health.Condition + for _, key := range keys { + raw, _, found, err := k.Store.Get(ConditionsState, key) + if err != nil { + return nil, err + } + if !found || string(raw) == "null" { + continue + } + var c health.Condition + if err := json.Unmarshal(raw, &c); err != nil { + return nil, fmt.Errorf("the condition %s on the bus cannot be read: %w", key, err) + } + out = append(out, c) + } + health.Order(out) + return out, nil +} + +// Resync settles every key any input names or any open condition holds (clause 12): what differs is raised or cleared. +func (k *Keeper) Resync() (int, error) { + open, err := k.Open() + if err != nil { + return 0, err + } + seen := map[string]bool{} + for _, c := range open { + seen[c.Key] = true + } + k.mu.Lock() + k.init() + for o, fs := range k.findings { + if k.blind[o] != "" { + continue + } + for key := range fs { + seen[key] = true + } + } + for key := range k.ctrl { + seen[key] = true + } + k.mu.Unlock() + keys := make([]string, 0, len(seen)) + for key := range seen { + keys = append(keys, key) + } + return len(keys), k.settleAll(keys) +} + +// Silence stops a condition's messages for a while, a hand act with why (clause 3, Limits 1): kept as its transition in +// history, naming who and why. +func (k *Keeper) Silence(key string, d time.Duration, by, why string) (health.Condition, error) { + if strings.TrimSpace(why) == "" { + return health.Condition{}, errors.New("a silence says why") + } + if d <= 0 || d > health.MaxSilence { + return health.Condition{}, fmt.Errorf("a silence lasts more than nothing and at most %s, not %s", health.MaxSilence, d) + } + for i := 0; i < ConditionWrites; i++ { + raw, rev, found, err := k.Store.Get(ConditionsState, key) + if err != nil { + return health.Condition{}, err + } + if !found || string(raw) == "null" { + return health.Condition{}, fmt.Errorf("no condition %s is open", key) + } + var c health.Condition + if err := json.Unmarshal(raw, &c); err != nil { + return health.Condition{}, err + } + now := k.Now().UTC() + c.Silenced = &health.Silence{Until: now.Add(d), By: by, Why: why, Since: now} + if _, err := k.Store.Update(ConditionsState, key, c, rev); errors.Is(err, stdio.ErrChanged) { + continue + } else if err != nil { + return health.Condition{}, err + } + k.tell(health.Event{Condition: c, Event: health.EventChanged, At: now, Change: health.ChangeSilenced, + Why: why + " (" + by + ")", Show: health.ShowKeeper(key)}) + return c, nil + } + return health.Condition{}, fmt.Errorf("the condition %s kept moving under the silence", key) +} diff --git a/modules/mesh-conditions/cmd/mesh-conditions/keeper_test.go b/modules/mesh-conditions/cmd/mesh-conditions/keeper_test.go new file mode 100644 index 00000000..475f8ec5 --- /dev/null +++ b/modules/mesh-conditions/cmd/mesh-conditions/keeper_test.go @@ -0,0 +1,341 @@ +package main + +import ( + "encoding/json" + "fmt" + "sort" + "strings" + "sync" + "testing" + "time" + + stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/health" +) + +// fakeStore is the bus's states in memory, with revisions, as the runtime answers them. +type fakeStore struct { + mu sync.Mutex + vals map[string][]byte + revs map[string]uint64 + next uint64 + // ttls are the times to live the keys were created with. + ttls map[string]time.Duration +} + +func newFakeStore() *fakeStore { + return &fakeStore{vals: map[string][]byte{}, revs: map[string]uint64{}} +} + +func (f *fakeStore) Get(state, key string) ([]byte, uint64, bool, error) { + f.mu.Lock() + defer f.mu.Unlock() + v, ok := f.vals[state+"/"+key] + return v, f.revs[state+"/"+key], ok, nil +} + +func (f *fakeStore) put(state, key string, value any) uint64 { + raw, _ := json.Marshal(value) + f.next++ + f.vals[state+"/"+key], f.revs[state+"/"+key] = raw, f.next + return f.next +} + +func (f *fakeStore) Create(state, key string, value any) (uint64, error) { + f.mu.Lock() + defer f.mu.Unlock() + if _, ok := f.vals[state+"/"+key]; ok { + return 0, fmt.Errorf("%s: %w", key, stdio.ErrChanged) + } + return f.put(state, key, value), nil +} + +func (f *fakeStore) Update(state, key string, value any, rev uint64) (uint64, error) { + f.mu.Lock() + defer f.mu.Unlock() + if f.revs[state+"/"+key] != rev { + return 0, fmt.Errorf("%s: %w", key, stdio.ErrChanged) + } + return f.put(state, key, value), nil +} + +func (f *fakeStore) Delete(state, key string) error { + f.mu.Lock() + defer f.mu.Unlock() + delete(f.vals, state+"/"+key) + delete(f.revs, state+"/"+key) + return nil +} + +func (f *fakeStore) Keys(state string) ([]string, error) { + f.mu.Lock() + defer f.mu.Unlock() + var out []string + for k := range f.vals { + if s, key, _ := strings.Cut(k, "/"); s == state { + out = append(out, key) + } + } + sort.Strings(out) + return out, nil +} + +func (f *fakeStore) CreateTTL(state, key string, value any, ttl time.Duration) error { + f.mu.Lock() + if f.ttls == nil { + f.ttls = map[string]time.Duration{} + } + f.ttls[state+"/"+key] = ttl + f.mu.Unlock() + _, err := f.Create(state, key, value) + return err +} + +// rig is a keeper over a fake store, its clock and what it said. +type rig struct { + k *Keeper + st *fakeStore + now time.Time + said []health.Event +} + +func newRig() *rig { + r := &rig{st: newFakeStore(), now: time.Date(2026, 10, 29, 12, 0, 0, 0, time.UTC)} + r.k = &Keeper{Store: r.st, Now: func() time.Time { return r.now }, Say: func(e health.Event) error { + r.said = append(r.said, e) + return nil + }} + return r +} + +func (r *rig) changes() string { + var out []string + for _, e := range r.said { + out = append(out, e.Change+" "+e.Key) + } + r.said = nil + return strings.Join(out, ", ") +} + +func (r *rig) open(key string) (health.Condition, bool) { + raw, _, ok, _ := r.st.Get(ConditionsState, key) + var c health.Condition + if ok { + _ = json.Unmarshal(raw, &c) + } + return c, ok +} + +// finding is an observer's finding as mesh-sdk's layout writes it (health.EncodeFinding). +func finding(t *testing.T, observer, scope, id, kind, source string, sev health.Severity, at time.Time) (string, []byte) { + t.Helper() + f := health.Finding{Key: health.Key(scope, id, kind), Kind: kind, Subject: health.Subject{Scope: scope, ID: id}, + Severity: sev, Summary: kind + " of " + id, Headline: kind, Explanation: "It is " + kind + ".", Source: source, + Observer: observer, At: at, Evidence: []health.Evidence{{At: at, Said: kind}}} + raw, err := health.EncodeFinding(f) + if err != nil { + t.Fatal(err) + } + return f.Key, raw +} + +// ADR 0564 clauses 2, 3, 7, 8 (issue 1952 C2, C3): a finding raises, changes and clears its condition; a blind observer +// clears nothing; the healers' attempts and the controller's escalation reach the condition; a reopening within ten +// minutes is the same fault; a silence is a transition in history. +func TestTheKeeperRaisesChangesAndClearsFromFindings(t *testing.T) { + r := newRig() + key, raw := finding(t, watchman, health.ScopeMachine, "g14", "silent", "S1", health.Warning, r.now) + steps := []struct { + name string + do func() error + said string + check func(c health.Condition, open bool) string + }{ + {name: "a finding written raises", do: func() error { return r.k.Finding(watchman, key, raw, false) }, + said: "raised machine.g14.silent", check: func(c health.Condition, open bool) string { + if !open || c.Severity != health.Warning || c.Resolver != health.ResolverSelf || c.Epoch != 0 || c.Count != 1 { + return fmt.Sprintf("%+v", c) + } + return "" + }}, + {name: "the same finding again says nothing", do: func() error { return r.k.Finding(watchman, key, raw, false) }}, + {name: "an observer gone blind clears nothing", do: func() error { r.k.Blind(watchman, "refused"); _, err := r.k.Resync(); return err }, + check: func(c health.Condition, open bool) string { + if !open { + return "cleared by blindness" + } + return "" + }}, + {name: "a healer's attempt reaches tried and the resolver", do: func() error { + a, _ := json.Marshal(health.Attempt{Healer: "H3", Condition: key, Act: "asserted", Outcome: health.OutcomeActed, Begun: r.now}) + return r.k.Attempt("h3-1", a, false) + }, said: "resolver machine.g14.silent", check: func(c health.Condition, _ bool) string { + if c.Resolver != "healer:H3" || len(c.Tried) != 1 || c.Tried[0].By != "healer H3" { + return fmt.Sprintf("%+v", c) + } + return "" + }}, + {name: "an escalation makes it urgent, the operator's", do: func() error { + a, _ := json.Marshal(health.Attempt{Healer: "H3", Condition: key, Act: "handed over", Outcome: health.OutcomeEscalated, + Begun: r.now.Add(time.Minute)}) + return r.k.Attempt("h3-2", a, false) + }, said: "severity machine.g14.silent, resolver machine.g14.silent", check: func(c health.Condition, _ bool) string { + if c.Severity != health.Urgent || c.Resolver != health.ResolverOperator || len(c.Tried) != 2 { + return fmt.Sprintf("%+v", c) + } + return "" + }}, + {name: "a silence is a transition with who and why", do: func() error { + _, err := r.k.Silence(key, time.Hour, "jochen", "known, a cable") + return err + }, said: "silenced machine.g14.silent"}, + {name: "the finding deleted clears", do: func() error { return r.k.Finding(watchman, key, nil, true) }, + said: "cleared machine.g14.silent", check: func(_ health.Condition, open bool) string { + if open { + return "still open" + } + return "" + }}, + {name: "raised again within ten minutes is a reopening", do: func() error { + r.now = r.now.Add(5 * time.Minute) + return r.k.Finding(watchman, key, raw, false) + }, said: "reopened machine.g14.silent", check: func(c health.Condition, _ bool) string { + if c.Count != 2 || len(c.Gaps) != 1 { + return fmt.Sprintf("%+v", c) + } + return "" + }}, + // Until the lease write the controller's escalation for the same key reaches the condition (clause 8). + {name: "the controller's escalation is carried", do: func() error { + ctrl, _ := json.Marshal(health.Condition{Key: key, Kind: "silent", Severity: health.Urgent, Resolver: health.ResolverOperator, + Tried: []health.Tried{{At: r.now, What: "x", Outcome: "escalated", By: "healer H1"}}}) + _ = r.k.Attempt("h3-1", nil, true) + _ = r.k.Attempt("h3-2", nil, true) + return r.k.Controller(key, ctrl, false) + }, check: func(c health.Condition, _ bool) string { + if c.Resolver != health.ResolverOperator || c.Severity != health.Urgent { + return fmt.Sprintf("%+v", c) + } + return "" + }}, + } + for _, s := range steps { + if err := s.do(); err != nil { + t.Fatalf("%s: %v", s.name, err) + } + if got := r.changes(); s.said != "" && got != s.said || s.said == "" && got != "" && s.name != "the controller's escalation is carried" { + t.Errorf("%s: said %q, want %q", s.name, got, s.said) + } + if s.check != nil { + c, open := r.open(key) + if why := s.check(c, open); why != "" { + t.Errorf("%s: %s", s.name, why) + } + } + } + // Every transition is kept in history, and each event carries the keeper's show. + keys, _ := r.st.Keys(HistoryState) + if len(keys) < 7 { + t.Errorf("history keeps %d transitions", len(keys)) + } + raw0, _, _, _ := r.st.Get(HistoryState, keys[0]) + if !strings.Contains(string(raw0), `"show":"mesh-conditions.conditions key=machine.g14.silent"`) { + t.Errorf("the first transition is %s", raw0) + } + // A carried kind (it ends with the controller) is raised and cleared as the controller's bucket says, until the + // lease is handed away, with the controller's show (clause 9, Limits 9). + deaf, _ := json.Marshal(health.Condition{Key: "core.controller.deaf", Kind: "controller-deaf", Source: "S4", + Subject: health.Subject{Scope: health.ScopeCore, ID: "controller"}, Severity: health.Urgent, Summary: "deaf", Resolver: "self", Epoch: 9}) + if err := r.k.Controller("core.controller.deaf", deaf, false); err != nil { + t.Fatal(err) + } + if c, open := r.open("core.controller.deaf"); !open || c.Epoch != 0 || r.said[0].Show != "mesh-controller.conditions key=core.controller.deaf" { + t.Errorf("a carried kind is %+v, said %+v", c, r.said) + } + r.said = nil + if err := r.k.Lease(true); err != nil { + t.Fatal(err) + } + if _, open := r.open("core.controller.deaf"); open || r.changes() != "cleared core.controller.deaf" { + t.Error("a carried kind outlived the lease write") + } +} + +// ADR 0564 clause 9 (issue 1952 C5, C8; the lead 2026-10-29): compare counts only what both sides can raise. A key held +// while its module is delivered, a kind that ends with the controller and a kind the controller never raises are listed +// apart; a kind with no home decided is a difference naming the gap; take-over is refused while compare differs, and +// hand-back after the lease write. +func TestCompareCountsOnlyWhatBothSidesRaise(t *testing.T) { + r := newRig() + put := func(key, kind string) []byte { + raw, _ := json.Marshal(health.Condition{Key: key, Kind: kind, Severity: health.Warning, Source: "x"}) + return raw + } + key, raw := finding(t, watchman, health.ScopeMachine, "g14", "silent", "S1", health.Warning, r.now) + _ = r.k.Finding(watchman, key, raw, false) + _ = r.k.Controller(key, put(key, "silent"), false) + only, onlyRaw := finding(t, delivery.AssignerModule, health.ScopeSeat, "mesh-placement.ace", "states-unbound", "say", health.Warning, r.now) + _ = r.k.Finding(delivery.AssignerModule, only, onlyRaw, false) + _ = r.k.Controller("core.controller.deaf", put("core.controller.deaf", "controller-deaf"), false) + _ = r.k.Controller("delivery.x.mystery", put("delivery.x.mystery", "mystery"), false) + _ = r.k.Controller("bus.x.consumer-wrong", put("bus.x.consumer-wrong", "consumer-wrong"), false) + _ = r.k.Controller("seat.vault.ace.holder-silent", put("seat.vault.ace.holder-silent", "holder-silent"), false) + cmp, err := r.k.Compare() + if err != nil { + t.Fatal(err) + } + reasons := map[string]string{} + for _, d := range append(append([]Difference(nil), cmp.Differences...), cmp.Apart...) { + reasons[d.Key] = d.Reason + } + for key, says := range map[string]string{ + only: "keeper only", + "core.controller.deaf": "carried", + "delivery.x.mystery": "no home is decided for the kind mystery", + "bus.x.consumer-wrong": "mesh-self-check has not written its finding", + // A kind of a probe the operator dropped (2026-10-30) is carried, never missing. + "seat.vault.ace.holder-silent": "carried from its bucket until the lease write (dropped 2026-10-30 by the operator", + } { + if !strings.Contains(reasons[key], says) { + t.Errorf("%s: %q, want it to say %q", key, reasons[key], says) + } + } + if cmp.Same != 1 || len(cmp.Differences) != 2 { + t.Errorf("compare: %+v", cmp) + } + // The lead's table of 2026-10-29: each kind has its home, lease-split and image-missing do not end, and the 22 + // that end are carried; with the 11 kinds of the probes the operator dropped on 2026-10-30, 33. + carried := 0 + for _, h := range Homes { + if h.Carried { + carried++ + } + } + for kind, observer := range map[string]string{"needs-operator": watchman, "machine-network": watchman, "lease-split": selfCheck, + "data-shrank": selfCheck, "bus-upgrade-failed": assigner, "retire-waiting": provisioner, "asks-undelivered": messenger, + "image-missing": artifactStore} { + if h, ok := Homes[kind]; !ok || h.Observer != observer || h.Carried { + t.Errorf("%s's home is %+v", kind, h) + } + } + for _, kind := range []string{"resolver-wrong", "holder-silent", "archives-missing", "archives-unheld", "binding-kept", + "binding-moved", "binding-moving", "grant-composes-nothing", "root-not-free", "cleanup-waiting", "data-held-twice"} { + if h := Homes[kind]; !h.Carried || h.Observer != "" || h.Why != Dropped { + t.Errorf("the dropped probe's kind %s is %+v", kind, h) + } + } + if len(Homes) != 84 || carried != 33 { // 19 of ADR 0564, the 63 of the table, plan-failed and stalled + t.Errorf("the table holds %d kinds, %d carried", len(Homes), carried) + } + if _, err := r.k.TakeOver(nil, "jochen", "now"); err == nil || !strings.Contains(err.Error(), "2 difference") { + t.Errorf("take-over while compare differs: %v", err) + } + if _, err := r.k.HandBack("jochen", "back"); err != nil { + t.Errorf("hand-back before the lease write: %v", err) + } + _ = r.k.Lease(true) + if _, err := r.k.HandBack("jochen", "back"); err == nil { + t.Error("hand-back after the lease write was taken") + } +} diff --git a/modules/mesh-conditions/cmd/mesh-conditions/main.go b/modules/mesh-conditions/cmd/mesh-conditions/main.go new file mode 100644 index 00000000..ac597d4b --- /dev/null +++ b/modules/mesh-conditions/cmd/mesh-conditions/main.go @@ -0,0 +1,164 @@ +// mesh-conditions: the condition keeper, holder of the seat mesh-conditions (novox/hq ADR 0564 clauses 1-3, 9, 13). A +// Go bundle served by a runtime of its own, on its own bus account (`runs-as`), so that its account alone writes its +// states: `conditions` (one key per open condition), `history` (every transition, ninety days), `takeover` (the key the +// readers follow), `held` and `bounds` (what a delivery holds, and for how long). It watches every observer's +// `findings`, the healers' `attempts`, the assigner's wants, the nodes' haves and the controller's lease and, until the +// lease write, its `conditions`; it says every transition on its seat with the controller's body. stdout is the MCP +// channel; what this module says, it says on stderr. +package main + +import ( + "fmt" + "os" + "time" + + stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/asker" + "git.novox.be/novox/mesh-sdk/go/asks" + "git.novox.be/novox/mesh-sdk/go/buscompose" + "git.novox.be/novox/mesh-sdk/go/health" +) + +func logf(format string, a ...any) { fmt.Fprintf(os.Stderr, format+"\n", a...) } + +// runtimeStore is the bus, through the runtime that launched this bundle. +type runtimeStore struct{} + +func (runtimeStore) Get(state, key string) ([]byte, uint64, bool, error) { + e, err := stdio.State(state).Get(key) + if err != nil || e == nil { + return nil, 0, false, err + } + return e.Value, e.Revision, true, nil +} + +func (runtimeStore) Create(state, key string, value any) (uint64, error) { + return stdio.State(state).Create(key, value) +} + +func (runtimeStore) Update(state, key string, value any, revision uint64) (uint64, error) { + return stdio.State(state).Update(key, value, revision) +} + +func (runtimeStore) Delete(state, key string) error { return stdio.State(state).Delete(key) } + +func (runtimeStore) Keys(state string) ([]string, error) { return stdio.State(state).Keys() } + +// CreateTTL creates the key with its own time to live (`bounds` is key-ttl); its expiry is handed over as OpExpire. +func (runtimeStore) CreateTTL(state, key string, value any, ttl time.Duration) error { + _, err := stdio.State(state).CreateTTL(key, value, ttl) + return err +} + +// sayOnSeat says a transition on the seat, as the controller says it on its own (ADR 0564 clause 3). +func sayOnSeat(e health.Event) error { + return stdio.SeatPublish("mesh.seat."+Seat+".event."+e.Event, e, "") +} + +// NewKeeper is the keeper over the runtime. +func NewKeeper() *Keeper { + return &Keeper{Store: runtimeStore{}, Say: sayOnSeat, Now: time.Now, Logf: logf} +} + +// watch keeps a watch of one state, said blind while it cannot be read: a state not on the bus yet (an observer not +// delivered) is read again when it is. +func watch(k *Keeper, state, observer string, handle func(stdio.StateChange) error) { + go func() { + for wait := 5 * time.Second; ; wait = min(wait*2, 5*time.Minute) { + err := stdio.State(state).Watch("", handle) + if err == nil { + if observer != "" { + k.Blind(observer, "") + } + return + } + if observer != "" { + k.Blind(observer, err.Error()) + } + logf("%s cannot be watched (%v); watched again in %s", state, err, wait) + time.Sleep(wait) + } + }() +} + +// Keep starts every watch the keeper reads by (clauses 2, 8, 9, 13). +func Keep(k *Keeper) { + for _, s := range []string{HeldState} { + keys, _ := stdio.State(s).Keys() + for _, key := range keys { + if e, err := stdio.State(s).Get(key); err == nil && e != nil { + k.restoreHeld(key, e.Value) + } + } + } + if e, err := stdio.State(TakeoverState).Get(health.TakeoverKey); err == nil { + var raw []byte + if e != nil { + raw = e.Value + } + if t, err := health.DecodeTakeover(raw, e != nil); err == nil { + k.Takeover(t) + } + } + said := func(err error) error { + if err != nil { + logf("%v", err) + } + return nil + } + watch(k, ControllerLease, "", func(c stdio.StateChange) error { + if c.Key != ControllerLeaseKey { + return nil + } + away, why := buscompose.LeaseHandedAway(c.Value, c.Op == "put", nil) + logf("the controller's lease: %s", why) + return said(k.Lease(away)) + }) + watch(k, WantsRead, "", func(c stdio.StateChange) error { return said(k.Want(c.Key, c.Value, c.Op != "put")) }) + watch(k, HavesRead, "", func(c stdio.StateChange) error { return said(k.Have(c.Key, c.Value, c.Op != "put")) }) + watch(k, AttemptsRead, health.HealersModule+" attempts", func(c stdio.StateChange) error { + return said(k.Attempt(c.Key, c.Value, c.Op != "put")) + }) + watch(k, ControllerConditions, "", func(c stdio.StateChange) error { + if k.LeaseAway() { + return nil + } + return said(k.Controller(c.Key, c.Value, c.Op != "put")) + }) + for _, o := range Observers { + watch(k, o+"."+health.FindingsState, o, func(c stdio.StateChange) error { + return said(k.Finding(o, c.Key, c.Value, c.Op != "put")) + }) + } + watch(k, BoundsState, "", func(c stdio.StateChange) error { + if c.Op == stdio.OpExpire { + return said(k.BoundPassed(c.Key)) + } + return nil + }) + watch(k, PlacementRead, "", func(c stdio.StateChange) error { + k.Placement(c.Key, c.Value, c.Op != stdio.OpPut) + return nil + }) +} + +func main() { + k := NewKeeper() + client := asker.New(AskedState) + client.Module = Module + go func() { + <-stdio.Served() + Keep(k) + if err := client.Listen(k.Warranted, func(h asker.Handled, w asks.Warrant, err error) { + if err != nil { + logf("the answer to %s (%s): %v", w.Ask, h, err) + } + }); err != nil { + logf("the operator's answers cannot be heard: %v", err) + } + }() + if err := stdio.Serve("", Tools(k, client)); err != nil { + logf("%v", err) + os.Exit(1) + } +} diff --git a/modules/mesh-conditions/cmd/mesh-conditions/manifest_test.go b/modules/mesh-conditions/cmd/mesh-conditions/manifest_test.go new file mode 100644 index 00000000..94b1801c --- /dev/null +++ b/modules/mesh-conditions/cmd/mesh-conditions/manifest_test.go @@ -0,0 +1,69 @@ +package main + +import ( + "os" + "path/filepath" + "slices" + "testing" + + "git.novox.be/novox/mesh-sdk/go/catalogue" + "git.novox.be/novox/mesh-sdk/go/health" +) + +// ADR 0564 clauses 1, 2, 3 and 12 (issue 1952 C1): the manifest says what the code does. It claims the seat +// mesh-conditions once for the mesh with the verbs mesh-sdk defines for it; reads every observer's findings, the healers' +// attempts, the controller's conditions and lease, the wants and the haves, and never the healers' would; keeps its +// states, history ninety days and each delivery's bound as long as the bound; and every state has a tool that reads it, +// every act a tool that starts it. +func TestTheManifestSaysWhatTheCodeDoes(t *testing.T) { + raw, err := os.ReadFile(filepath.Join("..", "..", "module.json")) + if err != nil { + t.Fatal(err) + } + m, err := catalogue.ParseManifest(raw) + if err != nil { + t.Fatalf("the manifest is refused: %v", err) + } + seat, _ := catalogue.SeatNamed(Seat) + if m.Module != Module || m.RunsAs != Module || len(m.Claims) != 1 || m.Claims[0].Name != Seat || + !slices.Equal(Verbs, catalogue.VerbNames(seat.Serves)) { + t.Errorf("module %s runs as %s and claims %+v; the seat serves %v", m.Module, m.RunsAs, m.Claims, catalogue.VerbNames(seat.Serves)) + } + reads := []string{ControllerConditions, ControllerLease, WantsRead, HavesRead, AttemptsRead, PlacementRead} + for _, o := range Observers { + reads = append(reads, o+"."+health.FindingsState) + } + slices.Sort(reads) + got := slices.Clone(m.Reads) + slices.Sort(got) + if !slices.Equal(got, reads) || slices.Contains(m.Reads, health.HealersModule+"."+health.WouldState) { + t.Errorf("reads %v, want %v and never the healers' would", got, reads) + } + ttl, keyTTL := map[string]int{}, map[string]bool{} + for _, s := range m.State { + ttl[s.Name], keyTTL[s.Name] = s.TTLSeconds, s.KeyTTL + } + // Each bound is its own key's time to live (the runtime's per-key TTL), never the bucket's age. + if ttl[HistoryState] != int(HistoryKeptFor.Seconds()) || ttl[BoundsState] != 0 || !keyTTL[BoundsState] { + t.Errorf("history is kept %d s, a bound's bucket ages %d s, its keys have their own time to live: %v", + ttl[HistoryState], ttl[BoundsState], keyTTL[BoundsState]) + } + // Clause 12: each state with the tool that reads it, each act with the tool that starts it. + readBy := map[string]string{ConditionsState: Seat + ".conditions", HistoryState: HistoryTool, TakeoverState: TakeoverTool, + HeldState: HeldTool, BoundsState: HeldTool, AskedState: Seat + ".take-over", "a resync": Seat + ".resync", + "what it reads": InputsTool} + tools := map[string]bool{} + for _, tool := range Tools(&Keeper{}, nil) { + tools[tool.Name] = true + } + for name := range ttl { + if readBy[name] == "" { + t.Errorf("the state %s has no tool that reads it", name) + } + } + for what, tool := range readBy { + if !tools[tool] { + t.Errorf("%s is read or started by %s, which is not served", what, tool) + } + } +} diff --git a/modules/mesh-conditions/cmd/mesh-conditions/reads-outside b/modules/mesh-conditions/cmd/mesh-conditions/reads-outside new file mode 100644 index 00000000..c20a67a5 --- /dev/null +++ b/modules/mesh-conditions/cmd/mesh-conditions/reads-outside @@ -0,0 +1,2 @@ +# What these tests read outside their package, as paths from the repository's root (issue 1936). +modules/mesh-conditions/module.json diff --git a/modules/mesh-conditions/cmd/mesh-conditions/tools.go b/modules/mesh-conditions/cmd/mesh-conditions/tools.go new file mode 100644 index 00000000..330e153e --- /dev/null +++ b/modules/mesh-conditions/cmd/mesh-conditions/tools.go @@ -0,0 +1,252 @@ +package main + +// The seat's verbs and the keeper's own tools (novox/hq ADR 0564 clauses 3, 9 and 12; issue 1952 C4): `conditions` +// answers as the controller's verb answers, so a caller changes only the seat it asks; and a read of every state the +// keeper keeps and of what it reads, so a person sees all it holds and a recovery is a call. + +import ( + "encoding/json" + "fmt" + "slices" + "strconv" + "strings" + "time" + + stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/health" +) + +// Verbs are the seat's verbs, as the manifest claims them. +var Verbs = []string{"conditions", "silence", "compare", "take-over", "hand-back", "resync"} + +// The keeper's own read tools. +const ( + HistoryTool = "conditions_history" + HeldTool = "conditions_held" + TakeoverTool = "conditions_takeover" + InputsTool = "conditions_inputs" +) + +// briefEvidence is how many observations a condition carries in a list: its newest (the controller's, issue 314). +const briefEvidence = 1 + +// inBrief is a condition in a list: its newest evidence, with how many it keeps and how to read it whole. +type inBrief struct { + health.Condition + EvidenceKept int `json:"evidence-kept,omitempty"` + More string `json:"more,omitempty"` +} + +func obj(props map[string]any, required ...string) map[string]any { + out := map[string]any{"type": "object", "properties": props} + if len(required) > 0 { + out["required"] = required + } + return out +} + +func str(description string) map[string]any { + return map[string]any{"type": "string", "description": description} +} + +// Tools are what the runtime serves. +func Tools(k *Keeper, a Asks) []stdio.Tool { + return []stdio.Tool{ + {Name: Seat + ".conditions", Description: "What is wrong with the mesh now: every open condition, urgent first, then " + + "oldest, raised from the observers' findings and cleared when the finding goes, never by hand. Given a key, that " + + "one whole; with history, every transition lately.", + Input: obj(map[string]any{"key": str("a condition's key"), "scope": str("only this scope"), + "severity": str("urgent or warning"), "machine": str("only those about this machine"), + "history": str("\"true\": every transition lately"), "days": str("with history: days back (7, at most 90)")}), + Run: func(args map[string]any) (any, error) { return k.ConditionsVerb(args) }}, + {Name: Seat + ".silence", Description: "Stop one condition's messages for a while, a hand act with why, kept as its " + + "transition in history; it stays open.", + Input: obj(map[string]any{"key": str("the condition's key"), "for": str("30m, 4h, 2d; at most 7d"), + "why": str("why"), "cause": str("the cause in a word")}, "key", "for", "why"), + Run: func(args map[string]any) (any, error) { + d, err := duration(strArg(args, "for")) + if err != nil { + return nil, err + } + return k.Silence(strArg(args, "key"), d, stdio.CallerOf(args), strArg(args, "why")) + }}, + {Name: Seat + ".compare", Description: "The keeper's open conditions against the controller's, each difference with " + + "its reason; what ends with the controller, what only the keeper raises and what is held listed apart.", + Run: func(map[string]any) (any, error) { return k.Compare() }}, + {Name: Seat + ".take-over", Description: "Ask the operator's Approve to switch the readers of conditions to the " + + "keeper; refused while compare differs.", + Input: obj(map[string]any{"why": str("why now")}, "why"), + Run: func(args map[string]any) (any, error) { + return k.TakeOver(a, stdio.CallerOf(args), strArg(args, "why")) + }}, + {Name: Seat + ".hand-back", Description: "Switch the readers of conditions back to the controller, until its lease " + + "is handed away.", + Input: obj(map[string]any{"why": str("why")}, "why"), + Run: func(args map[string]any) (any, error) { return k.HandBack(stdio.CallerOf(args), strArg(args, "why")) }}, + {Name: Seat + ".resync", Description: "Read every finding again and raise and clear what differs.", + Run: func(map[string]any) (any, error) { + n, err := k.Resync() + return map[string]any{"settled": n}, err + }}, + {Name: HistoryTool, Description: "The keeper's history: every transition it said, newest first, at most limit.", + Input: obj(map[string]any{"limit": str("how many (default 50)"), "key": str("only this condition")}), + Run: func(args map[string]any) (any, error) { + return k.History(strArg(args, "key"), intArg(args, "limit", 50)) + }}, + {Name: HeldTool, Description: "What the keeper holds while its module is delivered (ADR 0564 clause 13): each key " + + "with its module, node, version and since when, and the deliveries' bounds.", + Run: func(map[string]any) (any, error) { + bounds, err := k.Store.Keys(BoundsState) + return map[string]any{"held": k.Held(), "bounds": bounds}, err + }}, + {Name: TakeoverTool, Description: "The take-over key: which source the router, the grants part and mesh-watcher hear.", + Run: func(map[string]any) (any, error) { + raw, _, found, err := k.Store.Get(TakeoverState, health.TakeoverKey) + if err != nil { + return nil, err + } + return health.DecodeTakeover(raw, found) + }}, + {Name: InputsTool, Description: "What the keeper reads, as it stands: each observer's findings counted, an observer it " + + "cannot read and why, the healers' attempts, the controller's conditions and whether its lease is handed away.", + Run: func(map[string]any) (any, error) { return k.Inputs(), nil }}, + } +} + +// ConditionsVerb is `conditions`: the list, one whole, or the history. +func (k *Keeper) ConditionsVerb(args map[string]any) (any, error) { + key := strArg(args, "key") + if strArg(args, "history") == "true" { + days := intArg(args, "days", 7) + if days < 1 || days > 90 { + return nil, fmt.Errorf("days is from 1 to 90, not %d", days) + } + return k.HistorySince(key, k.Now().Add(-time.Duration(days)*24*time.Hour), days) + } + if key != "" { + raw, _, found, err := k.Store.Get(ConditionsState, key) + if err != nil { + return nil, err + } + if !found || string(raw) == "null" { + return nil, fmt.Errorf("no condition %s is open", key) + } + var c health.Condition + return c, json.Unmarshal(raw, &c) + } + scope, severity, machine := strArg(args, "scope"), strArg(args, "severity"), strArg(args, "machine") + if severity != "" && severity != string(health.Urgent) && severity != string(health.Warning) { + return nil, fmt.Errorf("a severity is urgent or warning, not %q", severity) + } + open, err := k.Open() + if err != nil { + return nil, err + } + list := []inBrief{} + counted := map[string]int{} + for _, c := range open { + if (scope != "" && c.Subject.Scope != scope) || (severity != "" && string(c.Severity) != severity) || + (machine != "" && !concerns(c, machine)) { + continue + } + b := inBrief{Condition: c} + if len(c.Evidence) > briefEvidence { + b.EvidenceKept, b.Evidence, b.More = len(c.Evidence), c.Evidence[:briefEvidence:briefEvidence], health.ShowKeeper(c.Key) + } + list = append(list, b) + counted[c.Source+" "+c.Kind]++ + } + return map[string]any{"conditions": list, "open": len(open), "counted": counted, + "note": "urgent first, then oldest first; a condition clears when observation says so, never by hand; each with " + + "its newest evidence — `conditions key=` gives one whole"}, nil +} + +// concerns says whether a condition is about a machine: it names it, or its key does. +func concerns(c health.Condition, machine string) bool { + return c.Subject.Machine == machine || slices.Contains(c.Subject.Also, machine) || + slices.Contains(strings.Split(c.Key, "."), machine) +} + +// History is the newest transitions, at most limit, of one key or all. +func (k *Keeper) History(key string, limit int) ([]health.Event, error) { + all, err := k.HistorySince(key, time.Time{}, 0) + if err != nil { + return nil, err + } + events := all["history"].([]health.Event) + slices.Reverse(events) + if len(events) > limit { + events = events[:limit] + } + return events, nil +} + +// HistorySince is every transition since a moment, oldest first, as the controller's verb answers it. +func (k *Keeper) HistorySince(key string, since time.Time, days int) (map[string]any, error) { + keys, err := k.Store.Keys(HistoryState) + if err != nil { + return nil, err + } + events := []health.Event{} + for _, hk := range keys { + raw, _, found, err := k.Store.Get(HistoryState, hk) + if err != nil || !found { + continue + } + var e health.Event + if json.Unmarshal(raw, &e) != nil || e.At.Before(since) || (key != "" && e.Key != key) { + continue + } + events = append(events, e) + } + return map[string]any{"history": events, "days": days}, nil +} + +// Inputs is what the keeper reads, as it stands. +func (k *Keeper) Inputs() map[string]any { + k.mu.Lock() + defer k.mu.Unlock() + k.init() + findings := map[string]int{} + for o, fs := range k.findings { + findings[o] = len(fs) + } + return map[string]any{"findings": findings, "blind": k.blind, "attempts": len(k.attempts), + "controller-conditions": len(k.ctrl), "lease-handed-away": k.leaseOut, "follow": k.takeover.Follow, + "wants": len(k.wants), "haves": len(k.haves)} +} + +func strArg(a map[string]any, name string) string { + switch v := a[name].(type) { + case string: + return strings.TrimSpace(v) + case bool: + return strconv.FormatBool(v) + case float64: + return strconv.FormatFloat(v, 'f', -1, 64) + } + return "" +} + +func intArg(a map[string]any, name string, def int) int { + if n, err := strconv.Atoi(strArg(a, name)); err == nil { + return n + } + return def +} + +// duration reads 30m, 4h or 2d. +func duration(s string) (time.Duration, error) { + if n, ok := strings.CutSuffix(s, "d"); ok { + days, err := strconv.Atoi(n) + if err != nil { + return 0, fmt.Errorf("%q is not a duration: 30m, 4h or 2d", s) + } + return time.Duration(days) * 24 * time.Hour, nil + } + d, err := time.ParseDuration(s) + if err != nil { + return 0, fmt.Errorf("%q is not a duration: 30m, 4h or 2d", s) + } + return d, nil +} diff --git a/modules/mesh-conditions/cmd/mesh-conditions/whole_integration_test.go b/modules/mesh-conditions/cmd/mesh-conditions/whole_integration_test.go new file mode 100644 index 00000000..f8881730 --- /dev/null +++ b/modules/mesh-conditions/cmd/mesh-conditions/whole_integration_test.go @@ -0,0 +1,335 @@ +//go:build integration + +package main + +// The condition keeper whole against a real bus (novox/hq ADR 0564 clauses 2 and 3, issue 1952 C6): the built binary, +// played over its real stdin and stdout as the node's tool runtime plays it (tools/call; mesh/state.* answered from a +// real nats-server's key-value buckets, watches handed over as the runtime hands them, mesh/seat.publish onto its +// subjects). An observer's finding written to its bucket raises a condition the keeper keeps in its bucket and says as +// condition-raised with the controller's body; the seat's verb lists it; the finding deleted clears it, said as +// condition-cleared. + +import ( + "bufio" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" + + "git.novox.be/novox/mesh-sdk/go/health" + "git.novox.be/novox/mesh-sdk/go/testbus" +) + +type wire struct { + ID json.RawMessage `json:"id,omitempty"` + Method string `json:"method,omitempty"` + Params json.RawMessage `json:"params,omitempty"` + Result json.RawMessage `json:"result,omitempty"` + Error *struct { + Message string `json:"message"` + } `json:"error,omitempty"` +} + +// player is the node's tool runtime, as far as the keeper needs it, over a real bus. +type player struct { + ctx context.Context + nc *nats.Conn + js jetstream.JetStream + mu sync.Mutex + in io.Writer + next int + pending map[string]chan wire +} + +func (p *player) send(m map[string]any) { + raw, _ := json.Marshal(m) + p.mu.Lock() + defer p.mu.Unlock() + _, _ = p.in.Write(append(raw, '\n')) +} + +func (p *player) answer(id json.RawMessage, result any, err error) { + if err != nil { + p.send(map[string]any{"id": id, "error": map[string]any{"code": -32000, "message": err.Error()}}) + return + } + p.send(map[string]any{"id": id, "result": result}) +} + +func (p *player) serve(out io.Reader) { + scanner := bufio.NewScanner(out) + scanner.Buffer(make([]byte, 1<<20), 1<<24) + for scanner.Scan() { + var m wire + if json.Unmarshal(scanner.Bytes(), &m) != nil { + continue + } + if m.Method == "" { + var key string + _ = json.Unmarshal(m.ID, &key) + p.mu.Lock() + ch := p.pending[key] + delete(p.pending, key) + p.mu.Unlock() + if ch != nil { + ch <- m + } + continue + } + go p.handle(m) + } +} + +// kv is a state's bucket: the keeper's own by its local name, another's by `.`. +func (p *player) kv(state string) (jetstream.KeyValue, error) { + if owner, name, read := strings.Cut(state, "."); read { + return p.js.KeyValue(p.ctx, owner+"_"+name) + } + return p.js.KeyValue(p.ctx, Module+"_"+state) +} + +func (p *player) handle(m wire) { + var a struct { + State string `json:"state"` + Key string `json:"key"` + Value json.RawMessage `json:"value"` + Revision uint64 `json:"revision"` + Subject string `json:"subject"` + Body json.RawMessage `json:"body"` + } + _ = json.Unmarshal(m.Params, &a) + kv, kvErr := p.kv(a.State) + switch m.Method { + case "mesh/state.get": + if kvErr != nil { + p.answer(m.ID, nil, kvErr) + return + } + e, err := kv.Get(p.ctx, a.Key) + if errors.Is(err, jetstream.ErrKeyNotFound) { + p.answer(m.ID, nil, nil) + return + } + if err != nil { + p.answer(m.ID, nil, err) + return + } + p.answer(m.ID, map[string]any{"key": e.Key(), "value": json.RawMessage(e.Value()), "revision": e.Revision()}, nil) + case "mesh/state.create", "mesh/state.update", "mesh/state.put": + if kvErr != nil { + p.answer(m.ID, nil, kvErr) + return + } + var rev uint64 + var err error + switch m.Method { + case "mesh/state.create": + rev, err = kv.Create(p.ctx, a.Key, a.Value) + case "mesh/state.update": + rev, err = kv.Update(p.ctx, a.Key, a.Value, a.Revision) + default: + rev, err = kv.Put(p.ctx, a.Key, a.Value) + } + if errors.Is(err, jetstream.ErrKeyExists) || errors.Is(err, jetstream.ErrKeyRevisionMismatch) { + p.send(map[string]any{"id": m.ID, "error": map[string]any{"code": -32010, "message": "changed: " + err.Error()}}) + return + } + p.answer(m.ID, map[string]any{"revision": rev}, err) + case "mesh/state.keys": + if kvErr != nil { + p.answer(m.ID, nil, kvErr) + return + } + keys, err := kv.Keys(p.ctx) + if errors.Is(err, jetstream.ErrNoKeysFound) { + keys, err = []string{}, nil + } + p.answer(m.ID, keys, err) + case "mesh/state.delete": + err := kvErr + if err == nil { + err = kv.Delete(p.ctx, a.Key) + } + p.answer(m.ID, map[string]any{}, err) + case "mesh/state.watch": + if kvErr != nil { + p.answer(m.ID, nil, kvErr) + return + } + w, err := kv.WatchAll(p.ctx) + if err != nil { + p.answer(m.ID, nil, err) + return + } + var watchID string + _ = json.Unmarshal(m.ID, &watchID) + current := true + for e := range w.Updates() { + if e == nil { + current = false + p.answer(m.ID, map[string]any{}, nil) + continue + } + op := "put" + if e.Operation() != jetstream.KeyValuePut { + op = "delete" + } + p.mu.Lock() + p.next++ + id := fmt.Sprintf("w-%d", p.next) + p.mu.Unlock() + p.send(map[string]any{"id": id, "method": "mesh/state", "params": map[string]any{"watch": watchID, + "change": map[string]any{"state": a.State, "key": e.Key(), "op": op, "value": json.RawMessage(nullIfEmpty(e.Value())), + "revision": e.Revision(), "current": current}}}) + } + case "mesh/seat.publish": + p.answer(m.ID, map[string]any{"sequence": 1}, p.nc.Publish(a.Subject, a.Body)) + case "mesh/subscribe": + p.answer(m.ID, map[string]any{}, nil) + default: + p.answer(m.ID, nil, errors.New("nats: no responders available for request")) + } +} + +func nullIfEmpty(b []byte) []byte { + if len(b) == 0 { + return []byte("null") + } + return b +} + +func (p *player) call(name string, args map[string]any) (json.RawMessage, error) { + p.mu.Lock() + p.next++ + id := fmt.Sprintf("c-%d", p.next) + ch := make(chan wire, 1) + p.pending[id] = ch + p.mu.Unlock() + p.send(map[string]any{"id": id, "method": "tools/call", "params": map[string]any{"name": name, "arguments": args, + "_meta": map[string]any{"mesh/caller": "person.jochen"}}}) + select { + case m := <-ch: + if m.Error != nil { + return nil, errors.New(m.Error.Message) + } + var r struct { + Content []struct { + Text string `json:"text"` + } `json:"content"` + IsError bool `json:"isError"` + } + if err := json.Unmarshal(m.Result, &r); err != nil || len(r.Content) != 1 { + return nil, fmt.Errorf("%s answered %s", name, m.Result) + } + if r.IsError { + return nil, errors.New(r.Content[0].Text) + } + return json.RawMessage(r.Content[0].Text), nil + case <-time.After(60 * time.Second): + return nil, fmt.Errorf("%s did not answer in 60s", name) + } +} + +// 1952/C6: an observer's finding raises a condition on the bus, said with the controller's body, and its deletion clears +// it. +func TestTheKeeperWholeRaisesAndClearsOnTheBus(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + bin := filepath.Join(t.TempDir(), Module) + build := exec.Command("go", "build", "-o", bin, ".") + build.Stderr = os.Stderr + if err := build.Run(); err != nil { + t.Fatalf("the keeper does not build: %v", err) + } + nc, err := nats.Connect(testbus.URL(t)) + if err != nil { + t.Fatal(err) + } + t.Cleanup(nc.Close) + js, err := jetstream.New(nc) + if err != nil { + t.Fatal(err) + } + buckets := []string{"mesh-controller_conditions", "mesh-controller_lease", "mesh-delivery-assigner_wants", "node-engine_have", "mesh-delivery-assigner_placement", + "mesh-healers_attempts"} + for _, o := range Observers { + buckets = append(buckets, o+"_findings") + } + for _, s := range []string{ConditionsState, HistoryState, TakeoverState, HeldState, BoundsState, AskedState} { + buckets = append(buckets, Module+"_"+s) + } + for _, b := range buckets { + if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: b}); err != nil { + t.Fatal(err) + } + } + events, err := nc.SubscribeSync("mesh.seat." + Seat + ".event.>") + if err != nil { + t.Fatal(err) + } + cmd := exec.CommandContext(ctx, bin) + cmd.Env = append(os.Environ(), "MESH_NODE=ace", "MESH_MODULE="+Module) + stdin, _ := cmd.StdinPipe() + stdout, _ := cmd.StdoutPipe() + cmd.Stderr = os.Stderr + if err := cmd.Start(); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = stdin.Close(); cancel(); _ = cmd.Wait() }) + p := &player{ctx: ctx, nc: nc, js: js, in: stdin, pending: map[string]chan wire{}} + go p.serve(stdout) + + // The first call serves; the watches start after it. + if _, err := p.call(InputsTool, map[string]any{}); err != nil { + t.Fatal(err) + } + time.Sleep(time.Second) + at := time.Now().UTC() + f := health.Finding{Key: "machine.g14.silent", Kind: "silent", Subject: health.Subject{Scope: health.ScopeMachine, ID: "g14", + Machine: "g14"}, Severity: health.Warning, Summary: "g14 is silent", Headline: "g14 is offline", + Explanation: "g14 said nothing for three minutes.", Resolved: "g14 is back", Source: "S1", + Observer: health.WatchmanModule, At: at, Evidence: []health.Evidence{{At: at, Said: "no presence for 3m"}}} + raw, err := health.EncodeFinding(f) + if err != nil { + t.Fatal(err) + } + findings, _ := js.KeyValue(ctx, health.WatchmanModule+"_findings") + if _, err := findings.Put(ctx, f.Key, raw); err != nil { + t.Fatal(err) + } + msg, err := events.NextMsg(10 * time.Second) + if err != nil { + t.Fatalf("no condition-raised was said: %v", err) + } + var e health.Event + if err := json.Unmarshal(msg.Data, &e); err != nil || msg.Subject != "mesh.seat.mesh-conditions.event.condition-raised" || + e.Key != f.Key || e.Change != health.ChangeRaised || e.Show != health.ShowKeeper(f.Key) || e.Headline != f.Headline { + t.Fatalf("said %s on %s", msg.Data, msg.Subject) + } + listed, err := p.call(Seat+".conditions", map[string]any{}) + if err != nil || !strings.Contains(string(listed), `"key":"machine.g14.silent"`) { + t.Fatalf("conditions answered %s, %v", listed, err) + } + if err := findings.Delete(ctx, f.Key); err != nil { + t.Fatal(err) + } + msg, err = events.NextMsg(10 * time.Second) + if err != nil || msg.Subject != "mesh.seat.mesh-conditions.event.condition-cleared" { + t.Fatalf("no condition-cleared was said: %v", err) + } + conditions, _ := js.KeyValue(ctx, Module+"_"+ConditionsState) + if _, err := conditions.Get(ctx, f.Key); !errors.Is(err, jetstream.ErrKeyNotFound) { + t.Errorf("the cleared condition is still kept: %v", err) + } +} diff --git a/modules/mesh-conditions/go.mod b/modules/mesh-conditions/go.mod new file mode 100644 index 00000000..b941eeac --- /dev/null +++ b/modules/mesh-conditions/go.mod @@ -0,0 +1,22 @@ +module mesh-conditions + +go 1.26.0 + +require ( + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 + github.com/nats-io/nats.go v1.54.0 +) + +require ( + github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op // indirect + github.com/google/go-tpm v0.9.8 // indirect + github.com/klauspost/compress v1.20.0 // indirect + github.com/minio/highwayhash v1.0.4 // indirect + github.com/nats-io/jwt/v2 v2.8.1 // indirect + github.com/nats-io/nats-server/v2 v2.11.17 // indirect + github.com/nats-io/nkeys v0.4.16 // indirect + github.com/nats-io/nuid v1.0.1 // indirect + golang.org/x/crypto v0.57.0 // indirect + golang.org/x/sys v0.48.0 // indirect + golang.org/x/time v0.15.0 // indirect +) diff --git a/modules/mesh-conditions/go.sum b/modules/mesh-conditions/go.sum new file mode 100644 index 00000000..a2b8e018 --- /dev/null +++ b/modules/mesh-conditions/go.sum @@ -0,0 +1,27 @@ +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 h1:lKU3HjNXGiVbLW2hUgFjU/TMTUYMlNnnmZxG5T7biQA= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= +github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= +github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= +github.com/google/go-tpm v0.9.8/go.mod h1:h9jEsEECg7gtLis0upRBQU+GhYVH6jMjrFxI8u6bVUY= +github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA= +github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI= +github.com/minio/highwayhash v1.0.4 h1:asJizugGgchQod2ja9NJlGOWq4s7KsAWr5XUc9Clgl4= +github.com/minio/highwayhash v1.0.4/go.mod h1:GGYsuwP/fPD6Y9hMiXuapVvlIUEhFhMTh0rxU3ik1LQ= +github.com/nats-io/jwt/v2 v2.8.1 h1:V0xpGuD/N8Mi+fQNDynXohVvp7ZztevW5io8CUWlPmU= +github.com/nats-io/jwt/v2 v2.8.1/go.mod h1:nWnOEEiVMiKHQpnAy4eXlizVEtSfzacZ1Q43LIRavZg= +github.com/nats-io/nats-server/v2 v2.11.17 h1:GKEghcFK6A+aFx11Yf1LjgLC3txAwvyhnYzhBIQZA8I= +github.com/nats-io/nats-server/v2 v2.11.17/go.mod h1:B1sFVz4StNosQ903ak4N1G01Fl/9f8e06mXpFIE2K24= +github.com/nats-io/nats.go v1.54.0 h1:vsXoOxjHp/GmPUN+EcI7uOf/uB+iAP+kEsAFNQN0yzA= +github.com/nats-io/nats.go v1.54.0/go.mod h1:y+DZoD1oBOYfZTU681eTUiUjI0vbqYGixNVFHcjHJ0k= +github.com/nats-io/nkeys v0.4.16 h1:rd5oAuLOb8mnAycB0xleuEBNS1pVVnN0fv/FF34Eypg= +github.com/nats-io/nkeys v0.4.16/go.mod h1:llLgWoI0o4z/Q57q2R1kHfmocyhGV6VG/U18Glg1Afs= +github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= +github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= +golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M= +golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA= +golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo= +golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= +golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= +golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= diff --git a/modules/mesh-conditions/module.json b/modules/mesh-conditions/module.json new file mode 100644 index 00000000..d676b3cf --- /dev/null +++ b/modules/mesh-conditions/module.json @@ -0,0 +1,87 @@ +{ + "module": "mesh-conditions", + "version": "1", + "runs-as": "mesh-conditions", + "claims": [ + { + "name": "mesh-conditions", + "scope": "mesh", + "serves": [ + "conditions", + "silence", + "compare", + "take-over", + "hand-back", + "resync" + ] + } + ], + "uses": [ + "operator-channel" + ], + "reads": [ + "mesh-delivery-watchman.findings", + "mesh-self-check.findings", + "mesh-healers.findings", + "mesh-delivery-assigner.findings", + "messenger.findings", + "mesh-provisioner.findings", + "artifact-store-tools.findings", + "mesh-healers.attempts", + "mesh-controller.conditions", + "mesh-controller.lease", + "mesh-delivery-assigner.wants", + "node-engine.have", + "mesh-delivery-assigner.placement" + ], + "state": [ + "conditions", + { + "name": "history", + "ttl-seconds": 7776000 + }, + "takeover", + "held", + { + "name": "bounds", + "key-ttl": true + }, + "asked" + ], + "own-secrets": { + "broker": "${dir:state}/broker" + }, + "secrets-owner": "mesh-conditions", + "resources": [ + { + "id": "account", + "type": "user", + "name": "mesh-conditions", + "shell": "/usr/bin/nologin", + "home": "/var/lib/mesh-conditions" + }, + { + "id": "state", + "type": "directory", + "mode": "0700", + "owner": "mesh-conditions", + "place": ".", + "unknown": "program" + } + ], + "build": { + "artifacts": [ + { + "name": "tools", + "kind": "bundle", + "language": "go", + "system": "arch", + "from": "cmd/mesh-conditions", + "binary": "mesh-conditions", + "loads": [ + "mesh-conditions" + ] + } + ] + } +} diff --git a/modules/mesh-config/README.md b/modules/mesh-config/README.md index 42bd1ed0..72220615 100644 --- a/modules/mesh-config/README.md +++ b/modules/mesh-config/README.md @@ -61,3 +61,21 @@ them, newest first; with `proposal`, one read whole, with whether the router's a Each directory or archive this module declares says how it treats a file it did not write (ADR 0422, ADR 0416). - `${dir:state}`: everything in it is the program's; the mesh moves nothing out (ADR 0422 §2). + +## The switch from the controller's records (novox/hq ADR 0565 §3, §4, §13) + +- `mesh-config_declared`: each placed module's declarations at the trunk, with the manifest and commit they come + from, written only when they differ. +- `mesh-config_takeover`: the key `switch` names the source that holds: none (the interim since mesh-catalog #558: + declarations from the modules' own manifests, the list of predefined keys and pull requests' config from the + controller), `indexer` after the operator's take-over (the list and pull requests' config from the indexer's + `predefined` and `config-candidates`; the controller's `pull-declares` refused), `controller` after a hand-back + (declarations, list and pull requests' config from the controller's records). Each take-over ask is kept beside it. +- `config_compare` lists every difference between the own sources and the controller's records: each module's + declarations, the list, each open pull request's config, and every placed module's value on every node resolved + both ways. `config_take_over` asks the operator's Approve, refused while any difference stands and switched on the + Approve only if none stands then. `config_hand_back` returns to the controller's records until its lease is handed + away. `config_declared` and `config_resync` read and write again what it keeps. +- From the controller's lease write a trusted value is set only through propose and the operator's Approve. +- Its sets, clears and warrants are recorded in the hand-act log of the seat `mesh-hand-acts`, falling back to the + controller's while that seat has no holder and the controller holds its lease. diff --git a/modules/mesh-config/cmd/mesh-config/config.go b/modules/mesh-config/cmd/mesh-config/config.go index 0f2435cc..1928a3df 100644 --- a/modules/mesh-config/cmd/mesh-config/config.go +++ b/modules/mesh-config/cmd/mesh-config/config.go @@ -275,7 +275,7 @@ func (c *Config) change(call Call) (*SetAnswer, error) { return nil, fmt.Errorf("%s is refused for %s at %s: %s", show(value), call.Key, level, strings.Join(problems, "; ")) } - if trusted && call.Caller != ControllerCaller && call.approval == nil { + if trusted && call.approval == nil && !c.atTheTerminal(call) { return nil, fmt.Errorf("%s is trusted (it decides what runs or who can reach what), so it is set only at "+ "the controller's terminal or on the operator's Approve (novox/hq ADR 0348 §8); %s is not the terminal: "+ "propose it instead, with config's action propose", call.Key, call.Caller) @@ -293,7 +293,7 @@ func (c *Config) change(call Call) (*SetAnswer, error) { if !had { return nil, fmt.Errorf("nothing is set for %s at %s, so there is nothing to clear", call.Key, level) } - if trusted && call.Caller != ControllerCaller && call.approval == nil { + if trusted && call.approval == nil && !c.atTheTerminal(call) { return nil, fmt.Errorf("%s is trusted, so it is cleared only at the controller's terminal or on the "+ "operator's Approve (novox/hq ADR 0348 §8); %s is not the terminal: propose the clear instead, with "+ "config's action propose and clear", call.Key, call.Caller) @@ -326,6 +326,13 @@ func (c *Config) change(call Call) (*SetAnswer, error) { return answer, nil } +// atTheTerminal says a call is the controller's relay of its terminal, which holds only until the controller's lease is +// handed away: from the lease write a trusted value is set only through propose and the operator's Approve, whoever +// calls (novox/hq ADR 0565 §13). +func (c *Config) atTheTerminal(call Call) bool { + return call.Caller == ControllerCaller && !c.leaseAway() +} + // declared is the rule a value at a level is held to, and whether it is trusted; refused, naming the declared // keys, for a key the module or the list of predefined keys does not declare (ADR 0348 §10). func (c *Config) declared(s Sources, level Level, key string) (rule, bool, error) { diff --git a/modules/mesh-config/cmd/mesh-config/main.go b/modules/mesh-config/cmd/mesh-config/main.go index 036acd86..c0bba92e 100644 --- a/modules/mesh-config/cmd/mesh-config/main.go +++ b/modules/mesh-config/cmd/mesh-config/main.go @@ -19,6 +19,8 @@ import ( stdio "git.novox.be/novox/mesh-sdk/go" "git.novox.be/novox/mesh-sdk/go/asks" + "git.novox.be/novox/mesh-sdk/go/buscompose" + "git.novox.be/novox/mesh-sdk/go/handacts" ) // Seat is the role this module holds, once for the mesh, and Verb its one verb. @@ -73,9 +75,21 @@ func (runtimeStore) Read(log string) ([]LogEntry, error) { func (runtimeStore) Emit(event string, body any) error { return stdio.Emit(event, body) } -// HandAct records a change in the controller's hand-act log (to-be 45 §7) through its seat's verb. +// recorder records into the hand-act log of the seat mesh-hand-acts, falling back to the controller's while the seat +// has no holder that answers and the controller still holds its lease (novox/hq ADR 0565 §11). +var recorder = handacts.Recorder{Ask: stdio.Ask, LeaseAway: func() bool { + e, err := stdio.State(LeaseRead).Get(buscompose.ControllerLeaseKey) + var raw []byte + if e != nil { + raw = e.Value + } + away, _ := buscompose.LeaseHandedAway(raw, e != nil, err) + return away +}} + +// HandAct records a change in the hand-act log (to-be 45 §7). func (runtimeStore) HandAct(what, why, cause string) error { - _, err := stdio.Ask("seat:mesh-controller.hand-act", map[string]any{"what": what, "why": why, "cause": cause}) + _, err := recorder.Record(handacts.Request{What: what, Why: why, Cause: cause}) return err } @@ -90,7 +104,7 @@ func (runtimeAsks) Publish(subject string, body any, id string) error { func (runtimeAsks) Record(id string) (*stdio.StateEntry, error) { return stdio.SeatRecord("", id) } func (runtimeAsks) Kinds() ([]stdio.KindHeld, error) { return stdio.SeatKinds() } func (runtimeAsks) Warranted(ask string) error { - _, err := stdio.Ask("seat:mesh-controller.warranted", map[string]any{"asker": Module, "ask": ask}) + _, err := recorder.RecordWarrant(Module, ask) return err } @@ -118,7 +132,8 @@ func resolving(c *Config) { } // A build's manifest is written before a want names the build, so a want, a trunk module or a placement moving is // every change of what a module declares where (issue 1953); the builds themselves are read, not watched. - for _, read := range []string{PlacementRead, WantsRead, ModulesRead, PredefinedRead} { + for _, read := range []string{PlacementRead, WantsRead, ModulesRead, PredefinedRead, IndexerPredefinedRead, + CandidatesRead, MirrorDeclaredRead, TakeoverState} { go func(read string) { for wait := 2 * time.Second; ; wait = min(wait*2, time.Minute) { time.Sleep(wait) @@ -221,9 +236,9 @@ func hearing(c *Config) { } } -// Tools is the seat's verb as the runtime serves it. +// Tools is the seat's verb and the module's own tools of the switch, as the runtime serves them. func Tools(c *Config) []stdio.Tool { - return []stdio.Tool{{ + return append(switchTools(c), stdio.Tool{ Name: Seat + "." + Verb, Description: "Set, clear, list or read the history of config values (novox/hq ADR 0348). A value is set at " + "one level: mesh config (no module, no node) or node config (a node) for a predefined key, or a " + @@ -268,7 +283,7 @@ func Tools(c *Config) []stdio.Tool { call.Caller = stdio.CallerOf(args) return c.Do(call) }, - }} + }) } // callOf reads a call's arguments strictly: an argument it does not know is refused, not ignored. diff --git a/modules/mesh-config/cmd/mesh-config/manifest_test.go b/modules/mesh-config/cmd/mesh-config/manifest_test.go new file mode 100644 index 00000000..c68914e4 --- /dev/null +++ b/modules/mesh-config/cmd/mesh-config/manifest_test.go @@ -0,0 +1,73 @@ +package main + +import ( + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" +) + +// Every state, log and read the manifest declares has a tool that reads it, and every act the module takes by itself +// a tool that starts it (novox/hq ADR 0565 §15): the verb config's actions, and the switch's tools. +func TestEveryStateAndActHasItsTool(t *testing.T) { + raw, err := os.ReadFile(filepath.Join("..", "..", "module.json")) + if err != nil { + t.Fatal(err) + } + var m struct { + State []string `json:"state"` + Logs []string `json:"logs"` + Reads []string `json:"reads"` + Tools []string `json:"tools"` + Claims []struct { + Name string `json:"name"` + Serves []string `json:"serves"` + } `json:"claims"` + } + if err := json.Unmarshal(raw, &m); err != nil { + t.Fatal(err) + } + via := map[string]string{ + ValuesState: "config list", ResolvedState: "config list", PredefinedState: "config list", + ProposalsState: "config proposals", PullsState: "config list", KeptState: "config list", HistoryLog: "config history", + DeclaredState: DeclaredTool, TakeoverState: DeclaredTool, + PlacementRead: CompareTool, WantsRead: CompareTool, BuildsRead: CompareTool, ModulesRead: DeclaredTool, + ManifestsRead: DeclaredTool, PredefinedRead: CompareTool, MirrorDeclaredRead: CompareTool, + IndexerPredefinedRead: CompareTool, CandidatesRead: CompareTool, LeaseRead: HandBackTool, + "a resolution": ResyncTool, "the indexer's candidates taken": ResyncTool, "a take-over decided": TakeOverTool, + } + served := map[string]bool{} + for _, tool := range Tools(&Config{}) { + served[tool.Name] = true + } + if len(m.Claims) != 1 || m.Claims[0].Name != Seat || strings.Join(m.Claims[0].Serves, ",") != strings.Join(SeatVerbs, ",") { + t.Errorf("claims %+v, want %s serving %v", m.Claims, Seat, SeatVerbs) + } + for _, verb := range SeatVerbs { + if !served[Seat+"."+verb] { + t.Errorf("the seat's verb %s is not served", verb) + } + } + for _, name := range m.Tools { + if !served[name] { + t.Errorf("the manifest lists the tool %s, which is not served", name) + } + } + for _, list := range [][]string{m.State, m.Logs, m.Reads} { + for _, s := range list { + if _, ok := via[s]; !ok { + t.Errorf("%s has no tool that reads it", s) + } + } + } + for what, tool := range via { + name, _, _ := strings.Cut(tool, " ") + if name == "config" { + name = Seat + "." + Verb + } + if !served[name] { + t.Errorf("%s is read or started by %s, which is not served", what, tool) + } + } +} diff --git a/modules/mesh-config/cmd/mesh-config/proposals.go b/modules/mesh-config/cmd/mesh-config/proposals.go index d6cb304d..18619024 100644 --- a/modules/mesh-config/cmd/mesh-config/proposals.go +++ b/modules/mesh-config/cmd/mesh-config/proposals.go @@ -758,6 +758,7 @@ func (c *Config) Decide() ([]string, error) { for _, r := range ready { said = append(said, c.decide(r.p, r.w)) } + said = append(said, c.decideTakeovers()...) c.recordWarrants(now) return said, nil } diff --git a/modules/mesh-config/cmd/mesh-config/pulls.go b/modules/mesh-config/cmd/mesh-config/pulls.go index f803b5de..8c0647a6 100644 --- a/modules/mesh-config/cmd/mesh-config/pulls.go +++ b/modules/mesh-config/cmd/mesh-config/pulls.go @@ -10,6 +10,8 @@ import ( "strconv" "strings" "time" + + "git.novox.be/novox/mesh-sdk/go/delivery" ) // A config value held for the open pull request that introduces its key (novox/hq ADR 0502, issue 1826). @@ -61,31 +63,22 @@ type PullDecl struct { At time.Time `json:"at"` } -// KeptValue is an approved held value, kept apart from the values mesh-config resolves. -type KeptValue struct { - Pull string `json:"pull"` - Head string `json:"head"` - Digest string `json:"declaration"` - Node string `json:"node"` - Module string `json:"module"` - Key string `json:"key"` - Value json.RawMessage `json:"value"` - By string `json:"by"` - Via string `json:"via"` - Why string `json:"why"` - // Approved is who approved it, as the proposal kept it. - Approved string `json:"approved"` - Proposal string `json:"proposal"` - At time.Time `json:"at"` -} +// KeptValue is an approved held value, kept apart from the values mesh-config resolves: mesh-sdk's layout, which the +// checker and the assigner's composition check read (novox/hq ADR 0565 §5). +type KeptValue = delivery.KeptValue +// keptKey is a held value's key in `kept` (delivery.HeldValueKey), the pull request's key form kept for one that is +// not one. func keptKey(pull, node, module, key string) string { + if k, err := delivery.HeldValueKey(pull, node, module, key); err == nil { + return k + } return pullKey(pull) + "." + node + "." + module + "." + key } -func (k KeptValue) stateKey() string { return keptKey(k.Pull, k.Node, k.Module, k.Key) } +func keptStateKey(k KeptValue) string { return keptKey(k.Pull, k.Node, k.Module, k.Key) } -func (k KeptValue) level() Level { return Level{Kind: LevelModule, Node: k.Node, Module: k.Module} } +func keptLevel(k KeptValue) Level { return Level{Kind: LevelModule, Node: k.Node, Module: k.Module} } // declDigest is the digest of what a declaration holds a value to: a held value stays only while it is the same. func declDigest(d Declaration) string { @@ -151,6 +144,12 @@ func (c *Config) PullDeclares(call Call) (*PullDeclaresAnswer, error) { return nil, fmt.Errorf("pull-declares is the controller's: it reads a pull request's manifests and tells "+ "mesh-config what the head declares; %s is not the controller", call.Caller) } + if sw, err := c.source(); err != nil { + return nil, err + } else if sw.Source == SourceIndexer { + return nil, errors.New("pull-declares is refused: since the take-over mesh-config reads a pull request's config " + + "declarations from the indexer's config-candidates (novox/hq ADR 0565 §4)") + } if _, _, _, err := parsePull(call.Pull); err != nil { return nil, err } @@ -185,6 +184,12 @@ func (c *Config) PullDeclares(call Call) (*PullDeclaresAnswer, error) { pull := strings.TrimSpace(call.Pull) c.mu.Lock() defer c.mu.Unlock() + return c.declarePull(pull, head, declares) +} + +// declarePull keeps what a pull request's head declares and judges again the values held for it; from the +// controller's pull-declares, or after the take-over from the indexer's config candidates. The caller holds the lock. +func (c *Config) declarePull(pull, head string, declares map[string]map[string]Declaration) (*PullDeclaresAnswer, error) { if err := c.Store.Put(PullsState, pullKey(pull), PullDecl{Pull: pull, Head: head, Declares: declares, At: c.now()}); err != nil { return nil, fmt.Errorf("the declarations were not kept: %v", err) @@ -235,7 +240,7 @@ func (c *Config) PullDeclares(call Call) (*PullDeclaresAnswer, error) { // heldChange records a hold, a drop or a merge's set of a held value in the history, the hand-act log and // config-changed; what could not be done is answered, not dropped. func (c *Config) heldChange(kv KeptValue, action, why, approved string) []string { - level := kv.level() + level := keptLevel(kv) ch := Change{Action: action, Level: &level, Key: kv.Key, Value: kv.Value, By: kv.By, Via: kv.Via, Why: why, Approved: approved, Pull: kv.Pull, Head: kv.Head, At: c.now(), Moved: []Moved{}} unsaid := c.record(level.ValueKey(kv.Key), ch) @@ -252,8 +257,8 @@ func (c *Config) heldChange(kv KeptValue, action, why, approved string) []string // dropKept removes a held value, saying why. func (c *Config) dropKept(kv KeptValue, why string) []string { - if err := c.Store.Delete(KeptState, kv.stateKey()); err != nil { - return []string{"the held value " + kv.stateKey() + " was not dropped: " + err.Error()} + if err := c.Store.Delete(KeptState, keptStateKey(kv)); err != nil { + return []string{"the held value " + keptStateKey(kv) + " was not dropped: " + err.Error()} } c.logf("[mesh-config] dropped %s of %s on %s held for %s: %s", kv.Key, kv.Module, kv.Node, kv.Pull, why) return c.heldChange(kv, ActionDrop, "dropped, nothing was set: "+why, "") @@ -408,7 +413,7 @@ func (c *Config) keep(p Proposal) (string, error) { kv := KeptValue{Pull: p.Pull, Head: pd.Head, Digest: p.DeclDigest, Node: p.Level.Node, Module: p.Level.Module, Key: p.Key, Value: p.Value, By: "the operator", Via: p.From, Why: p.Why, Approved: p.Approved, Proposal: p.ID, At: c.now()} - if err := c.Store.Put(KeptState, kv.stateKey(), kv); err != nil { + if err := c.Store.Put(KeptState, keptStateKey(kv), kv); err != nil { return "", fmt.Errorf("the value was not kept: %v", err) } unsaid := c.heldChange(kv, ActionHold, "held for "+p.Pull+" until it merges: "+p.Why, p.Approved) @@ -471,7 +476,7 @@ func (c *Config) settlePull(pull string, merged bool, head string) []string { said = append(said, c.dropKept(kv, kv.Pull+" merged, but "+why)...) continue } - vk := kv.level().ValueKey(kv.Key) + vk := keptLevel(kv).ValueKey(kv.Key) v := Value{Value: compact(kv.Value), By: kv.By, Via: kv.Via, Why: kv.Why + " (held for " + kv.Pull + ", merged)", Approved: kv.Approved, At: c.now()} if err := c.Store.Put(ValuesState, vk, v); err != nil { diff --git a/modules/mesh-config/cmd/mesh-config/reads-outside b/modules/mesh-config/cmd/mesh-config/reads-outside new file mode 100644 index 00000000..ca488b77 --- /dev/null +++ b/modules/mesh-config/cmd/mesh-config/reads-outside @@ -0,0 +1,2 @@ +# What these tests read outside their package, as paths from the repository's root (issue 1936). +modules/mesh-config/module.json diff --git a/modules/mesh-config/cmd/mesh-config/sources.go b/modules/mesh-config/cmd/mesh-config/sources.go index 761706d8..e32b4dda 100644 --- a/modules/mesh-config/cmd/mesh-config/sources.go +++ b/modules/mesh-config/cmd/mesh-config/sources.go @@ -12,10 +12,19 @@ import ( // Reading the sources, writing the resolved values, and publishing the list of predefined keys. -// sources reads everything a resolution reads, from the bus as it is now: which modules are placed where from the -// seat mesh-placement, and each placed module's declarations from its own manifests as built, never from the -// controller (issue 1953, ADR 0565 §3). +// sources reads everything a resolution reads, from the bus as it is now, from the source the switch names +// (takeover.go): which modules are placed where from the seat mesh-placement, and each placed module's declarations +// from its own manifests as built (issue 1953, ADR 0565 §3), or after a hand-back from the controller's mirror. func (c *Config) sources() (Sources, error) { + sw, err := c.source() + if err != nil { + return Sources{}, err + } + return c.sourcesFrom(sw.Source) +} + +// sourcesFrom reads the sources with declarations from one source: the own (interim and indexer) or the mirror. +func (c *Config) sourcesFrom(source string) (Sources, error) { s := Sources{Declared: map[string]ModuleDeclarations{}, OnNode: map[string]ModuleDeclarations{}, Predefined: map[string]Published{}, Values: map[string]Value{}} var problems []string @@ -25,6 +34,9 @@ func (c *Config) sources() (Sources, error) { if err := c.readDeclarations(&s); err != nil { return s, fmt.Errorf("the modules' declarations cannot be read: %w", err) } + if source == SourceController { + c.mirrorDeclarations(&s) + } for _, u := range s.Unresolved { problems = append(problems, fmt.Sprintf("%s on %s: %s", u.Module, u.Node, u.Why)) } @@ -350,6 +362,10 @@ type Resolution struct { Moved []Moved `json:"moved"` Published string `json:"published,omitempty"` Withheld string `json:"withheld,omitempty"` + // Declared are the modules whose `declared` key was written, and Candidates what the indexer's config candidates + // did after the take-over. + Declared []string `json:"declared,omitempty"` + Candidates []string `json:"candidates,omitempty"` } // Reconcile publishes the newest list of predefined keys and brings mesh-config_resolved to what the sources @@ -374,26 +390,39 @@ func (c *Config) Reconcile() (Resolution, error) { c.record("resolved", Change{Action: "resolve", Why: "an assignment, a declaration or the list of predefined " + "keys changed", At: c.now(), Moved: moved}) } + if written, derr := c.writeDeclared(s); derr != nil { + c.logf("[mesh-config] declared is not all written: %v", derr) + } else if len(written) > 0 { + r.Declared = written + } + if sw, serr := c.source(); serr == nil && sw.Source == SourceIndexer { + r.Candidates = c.takeCandidates() + } return r, err } -// publish makes mesh-config_predefined hold the list the controller kept, when it is newer than the one published -// (ADR 0348 §7: the newest trunk build wins). **A list dropping a key that still holds a value is withheld**: the +// publish makes mesh-config_predefined hold the list its source keeps — the indexer's after the take-over, the +// controller's mirror otherwise — when it is newer than the one published (ADR 0348 §7: the newest trunk build wins; +// the indexer never keeps an older merge's, so its list is newer when its commit differs). **A list dropping a key that still holds a value is withheld**: the // list published stays, and why is said in every listing and in the log, because a value for a key no longer on // the list could neither be resolved nor cleared by name (§7: removing a key still holding a value is refused). // It answers what it published, empty when nothing changed. func (c *Config) publish() (string, error) { - e, err := c.Store.Get(PredefinedRead, PredefinedMirrorKey) + sw, err := c.source() if err != nil { - return "", fmt.Errorf("the controller's list cannot be read: %w", err) + return "", err } - if e == nil { + read, whose := c.mirrorList, "the controller's list" + if sw.Source == SourceIndexer { + read, whose = c.indexerList, "the indexer's list" + } + list, found, err := read() + if err != nil { + return "", fmt.Errorf("%s cannot be read: %w", whose, err) + } + if !found { return "", nil // no list kept yet } - var list PredefinedList - if err := json.Unmarshal(e.Value, &list); err != nil || list.Keys == nil { - return "", fmt.Errorf("the controller's list is not one: %s", e.Value) - } current := map[string]Published{} if err := c.each(PredefinedState, func(key string, raw []byte) error { var p Published @@ -405,11 +434,12 @@ func (c *Config) publish() (string, error) { }); err != nil { return "", err } - generation := 0 + generation, sameCommit := 0, len(current) > 0 for _, p := range current { generation = max(generation, p.Generation) + sameCommit = sameCommit && p.Commit == list.Commit } - if len(current) > 0 && generation >= list.Generation { + if len(current) > 0 && (sameCommit || (list.Generation > 0 && generation >= list.Generation)) { c.withheld = "" return "", nil } diff --git a/modules/mesh-config/cmd/mesh-config/switch_tools.go b/modules/mesh-config/cmd/mesh-config/switch_tools.go new file mode 100644 index 00000000..7fbc2ffc --- /dev/null +++ b/modules/mesh-config/cmd/mesh-config/switch_tools.go @@ -0,0 +1,79 @@ +package main + +// The tools of the switch (novox/hq ADR 0565 §4, §15): compare, take-over and hand-back, served as the seat's verbs +// and as the module's own tools beside them, the read of `declared` and resync. + +import ( + "strings" + + stdio "git.novox.be/novox/mesh-sdk/go" +) + +// The switch's tools. +const ( + CompareTool = "config_compare" + TakeOverTool = "config_take_over" + HandBackTool = "config_hand_back" + DeclaredTool = "config_declared" + ResyncTool = "config_resync" +) + +// SeatVerbs are the seat's verbs, as the manifest claims them. +var SeatVerbs = []string{Verb, CompareVerb, TakeOverVerb, HandBackVerb} + +func switchTools(c *Config) []stdio.Tool { + own := ownSwitchTools(c) + byName := map[string]stdio.Tool{} + for _, t := range own { + byName[t.Name] = t + } + for verb, tool := range map[string]string{CompareVerb: CompareTool, TakeOverVerb: TakeOverTool, HandBackVerb: HandBackTool} { + t := byName[tool] + t.Name = Seat + "." + verb + own = append(own, t) + } + return own +} + +func ownSwitchTools(c *Config) []stdio.Tool { + why := map[string]any{"type": "object", "properties": map[string]any{ + "why": map[string]any{"type": "string", "description": "why, in a sentence"}}, "required": []string{"why"}} + whyOf := func(a map[string]any) string { s, _ := a["why"].(string); return strings.TrimSpace(s) } + return []stdio.Tool{ + {Name: CompareTool, + Description: "Every difference between config's own sources and the controller's records (novox/hq ADR 0565 §4): " + + "each placed module's declarations (declared against the mirror declared-config), the list of predefined " + + "keys (the indexer's against the mirror's), each open pull request's config (the indexer's candidate " + + "against the controller's pull-declares), and every placed module's value on every node resolved both " + + "ways; each with its reason. The take-over is refused while any stands.", + Run: func(map[string]any) (any, error) { return c.Compare() }}, + {Name: TakeOverTool, + Description: "Ask the operator's Approve, under mesh-config's own name, to switch config to its own sources: " + + "the indexer's list of predefined keys and pull request config, refusing the controller's pull-declares. " + + "Refused while compare shows a difference; on Approve switched if nothing differs then.", + Input: why, + Run: func(a map[string]any) (any, error) { + return c.TakeOver(Call{Why: whyOf(a), Caller: stdio.CallerOf(a)}) + }}, + {Name: HandBackTool, + Description: "Return config to the controller's records (its mirrors declared-config and predefined, and " + + "pull-declares) until the controller hands its lease away; gone after it. Recorded as a hand act.", + Input: why, + Run: func(a map[string]any) (any, error) { + return c.HandBack(Call{Why: whyOf(a), Caller: stdio.CallerOf(a)}) + }}, + {Name: DeclaredTool, + Description: "Each placed module's declarations as config keeps them (its state declared): the manifest and " + + "commit they come from and how many keys, or one module whole; and which source holds now.", + Input: map[string]any{"type": "object", "properties": map[string]any{ + "module": map[string]any{"type": "string", "description": "one module, whole"}}}, + Run: func(a map[string]any) (any, error) { + m, _ := a["module"].(string) + return c.Declared(strings.TrimSpace(m)) + }}, + {Name: ResyncTool, + Description: "Read every source again and write what differs: the published list of predefined keys, the " + + "resolved values, declared, and after the take-over the indexer's config candidates.", + Run: func(map[string]any) (any, error) { return c.Resync() }}, + } +} diff --git a/modules/mesh-config/cmd/mesh-config/takeover.go b/modules/mesh-config/cmd/mesh-config/takeover.go new file mode 100644 index 00000000..dad88461 --- /dev/null +++ b/modules/mesh-config/cmd/mesh-config/takeover.go @@ -0,0 +1,771 @@ +package main + +// Config's records out of the controller (novox/hq ADR 0565 §3, §4, §13, §15, and the progressive insight of +// 2026-10-29): mesh-config keeps each placed module's declarations in its own state `declared`; `compare` lists every +// difference between its own sources and the controller's mirrors — each module's declarations, the predefined list, +// each open pull request's config declarations, and every placed module's resolved value on every node both ways; +// `take-over` asks the operator's Approve under mesh-config's own name and, refused while anything differs, writes the +// key `takeover.switch` naming the indexer; `hand-back` returns to the mirrors until the lease write. +// +// **Which source holds** is the key `switch` alone, read by mesh-config alone: +// - none written (the interim of mesh-catalog #558): declarations from each placed module's own manifests, the list +// of predefined keys from the controller's mirror, a pull request's declarations from the controller's +// pull-declares; +// - `indexer` (taken over): declarations as before, the list from the indexer's `predefined`, a pull request's from +// the indexer's `config-candidates`; pull-declares refused; +// - `controller` (handed back): declarations from the mirror `declared-config`, the list from the mirror, +// pull-declares taken. + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "sort" + "strings" + "time" + + "git.novox.be/novox/mesh-sdk/go/asks" + "git.novox.be/novox/mesh-sdk/go/buscompose" + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +// The states and reads of the switch. +const ( + // DeclaredState is each placed module's declarations at the trunk, one key per module (ADR 0565 §3). + DeclaredState = "declared" + // TakeoverState holds the key SwitchKey and each take-over ask, `ask.`. + TakeoverState = "takeover" + SwitchKey = "switch" + + // MirrorDeclaredRead is the controller's mirror of every registered module's declarations, one key per module: + // what `compare` holds the own source to, and the source after a hand-back. + MirrorDeclaredRead = "mesh-controller.declared-config" + // IndexerPredefinedRead and CandidatesRead are the indexer's states (ADR 0565 §2). + IndexerPredefinedRead = delivery.IndexerModule + "." + delivery.IndexerPredefinedState + CandidatesRead = delivery.IndexerModule + "." + delivery.IndexerConfigCandidatesState + // LeaseRead is the controller's lease: after its write hand-back is gone and the terminal sets nothing (§13). + LeaseRead = buscompose.ControllerLease +) + +// The sources the switch names. +const ( + SourceInterim = "interim" + SourceIndexer = "indexer" + SourceController = "controller" +) + +// TakeOverVerb and HandBackVerb are the seat's verbs. +const ( + TakeOverVerb = "take-over" + HandBackVerb = "hand-back" + CompareVerb = "compare" +) + +// Switch is the key `takeover.switch`: the source that holds, who chose it, why and on which ask. +type Switch struct { + Source string `json:"source"` + At time.Time `json:"at"` + By string `json:"by"` + Why string `json:"why"` + Ask string `json:"ask,omitempty"` +} + +// source is the source that holds now. +func (c *Config) source() (Switch, error) { + e, err := c.Store.Get(TakeoverState, SwitchKey) + if err != nil { + return Switch{}, fmt.Errorf("the take-over's switch cannot be read: %w", err) + } + if e == nil { + return Switch{Source: SourceInterim}, nil + } + var s Switch + if err := json.Unmarshal(e.Value, &s); err != nil || (s.Source != SourceIndexer && s.Source != SourceController) { + return Switch{}, fmt.Errorf("the take-over's switch is not one: %s", e.Value) + } + return s, nil +} + +// leaseAway says the controller's lease is handed away (ADR 0561 §3's reading). +func (c *Config) leaseAway() bool { + e, err := c.Store.Get(LeaseRead, buscompose.ControllerLeaseKey) + var raw []byte + if e != nil { + raw = e.Value + } + away, _ := buscompose.LeaseHandedAway(raw, e != nil, err) + return away +} + +// ---- declared -------------------------------------------------------------------------------------------- + +// DeclaredKept is one key of `declared`: a module's declarations as the manifest the indexer's `modules.` +// names holds them, with that manifest's digest and commit (ADR 0565 §3). +type DeclaredKept struct { + Module string `json:"module"` + Manifest string `json:"manifest"` + Commit string `json:"commit,omitempty"` + Config map[string]Declaration `json:"settings"` +} + +// trunkKept is a module's declarations at the trunk with the manifest they come from. +func (c *Config) trunkKept(module string) (DeclaredKept, error) { + key, err := delivery.ModulesKey(module) + if err != nil { + return DeclaredKept{}, err + } + e, err := c.Store.Get(ModulesRead, key) + if err != nil || e == nil { + return DeclaredKept{}, fmt.Errorf("the indexer's modules.%s cannot be read (%v)", module, err) + } + var m struct { + Manifest string `json:"manifest"` + Commit string `json:"commit"` + } + if err := json.Unmarshal(e.Value, &m); err != nil || m.Manifest == "" { + return DeclaredKept{}, fmt.Errorf("the indexer's modules.%s names no manifest", module) + } + d, err := c.manifestDeclarations(module, m.Manifest) + if err != nil { + return DeclaredKept{}, err + } + return DeclaredKept{Module: module, Manifest: m.Manifest, Commit: m.Commit, Config: d.Config}, nil +} + +// writeDeclared makes `declared` hold each placed module's trunk declarations, writing a key only when it differs +// (ADR 0289 §1) and removing one of a module placed nowhere. It answers the modules written. +func (c *Config) writeDeclared(s Sources) ([]string, error) { + held := map[string][]byte{} + if err := c.each(DeclaredState, func(key string, raw []byte) error { + held[key] = raw + return nil + }); err != nil { + return nil, err + } + var written []string + var problems []error + placed := map[string]bool{} + for _, module := range modulesOf(s.Assignments) { + placed[module] = true + k, err := c.trunkKept(module) + if err != nil { + continue // unresolved, said by the resolution; its key stays as it was + } + raw, _ := json.Marshal(k) + if bytes.Equal(raw, held[module]) { + continue + } + if err := c.Store.Put(DeclaredState, module, k); err != nil { + problems = append(problems, fmt.Errorf("%s: %w", module, err)) + continue + } + written = append(written, module) + } + for module := range held { + if !placed[module] { + if err := c.Store.Delete(DeclaredState, module); err != nil { + problems = append(problems, fmt.Errorf("%s: %w", module, err)) + } + } + } + return written, errors.Join(problems...) +} + +// mirrorDeclarations makes the sources' declarations the controller's mirror's, for every placed module: the source +// after a hand-back, and the other side of compare. +func (c *Config) mirrorDeclarations(s *Sources) { + s.Declared, s.OnNode, s.Unresolved = map[string]ModuleDeclarations{}, map[string]ModuleDeclarations{}, nil + for _, module := range modulesOf(s.Assignments) { + e, err := c.Store.Get(MirrorDeclaredRead, module) + var d ModuleDeclarations + if err == nil && e != nil { + d, err = ParseModuleDeclarations(e.Value) + } else if err == nil { + err = errors.New("the mirror holds no key for it") + } + if err != nil { + for _, a := range s.Assignments { + if a.Module == module { + s.Unresolved = append(s.Unresolved, Unresolved{Node: a.Node, Module: module, + Why: "the controller's mirror declared-config: " + err.Error()}) + } + } + continue + } + s.Declared[module] = d + } +} + +// ---- the predefined list and the pull requests from the indexer ------------------------------------------ + +// indexerList is the indexer's predefined list as the mirror's list holds one: the keys, and the commit. +func (c *Config) indexerList() (PredefinedList, bool, error) { + e, err := c.Store.Get(IndexerPredefinedRead, delivery.PredefinedListKey) + if err != nil || e == nil { + return PredefinedList{}, false, err + } + var kept delivery.PredefinedKept + if err := json.Unmarshal(e.Value, &kept); err != nil { + return PredefinedList{}, false, fmt.Errorf("the indexer's predefined list is not one: %w", err) + } + list := PredefinedList{Keys: map[string]PredefinedKey{}, Commit: kept.Commit} + if err := json.Unmarshal(kept.List, &list.Keys); err != nil { + return PredefinedList{}, false, fmt.Errorf("the indexer's predefined list is not predefined.json: %w", err) + } + return list, true, nil +} + +// mirrorList is the controller's mirror's list. +func (c *Config) mirrorList() (PredefinedList, bool, error) { + e, err := c.Store.Get(PredefinedRead, PredefinedMirrorKey) + if err != nil || e == nil { + return PredefinedList{}, false, err + } + var list PredefinedList + if err := json.Unmarshal(e.Value, &list); err != nil || list.Keys == nil { + return PredefinedList{}, false, fmt.Errorf("the controller's list is not one: %s", e.Value) + } + return list, true, nil +} + +// candidates is every open pull request's config declarations as the indexer keeps them, by pull key. +func (c *Config) candidates() (map[string]PullDecl, error) { + out := map[string]PullDecl{} + err := c.each(CandidatesRead, func(key string, raw []byte) error { + var cand delivery.ConfigCandidate + if err := json.Unmarshal(raw, &cand); err != nil || cand.Pull == "" { + return fmt.Errorf("the indexer's config candidate %s is not one", key) + } + p := PullDecl{Pull: cand.Pull, Head: cand.Head, Declares: map[string]map[string]Declaration{}, At: cand.At} + for module, keys := range cand.Declares { + raw, _ := json.Marshal(keys) + var d map[string]Declaration + if err := json.Unmarshal(raw, &d); err != nil { + return fmt.Errorf("the indexer's config candidate %s declares %s unreadably", key, module) + } + for k, decl := range d { + if !decl.HasDefault() { + decl.Default = nil + } + d[k] = decl + } + p.Declares[module] = d + } + out[pullKey(cand.Pull)] = p + return nil + }) + return out, err +} + +// takeCandidates is the indexer's config candidates taken in place of pull-declares, after the take-over: each one +// whose head or declarations differ from what `pulls` holds is declared again, its held values judged again (ADR 0502 +// §6). The caller holds the lock. +func (c *Config) takeCandidates() []string { + cands, err := c.candidates() + if err != nil { + return []string{"the indexer's config candidates cannot be read: " + err.Error()} + } + var s Sources + if err := c.readPulls(&s); err != nil { + return []string{err.Error()} + } + var said []string + for _, k := range sortedKeys(cands) { + p := cands[k] + if have, ok := s.Pulls[k]; ok && have.Head == p.Head && declaresDigest(have.Declares) == declaresDigest(p.Declares) { + continue + } + a, err := c.declarePull(p.Pull, p.Head, p.Declares) + if err != nil { + said = append(said, fmt.Sprintf("%s from the indexer: %v", p.Pull, err)) + continue + } + said = append(said, a.Said+" (from the indexer)") + } + return said +} + +// declaresDigest is what a pull request's declarations hold a value to, in one encoding. +func declaresDigest(d map[string]map[string]Declaration) string { + flat := map[string]string{} + for m, keys := range d { + for k, decl := range keys { + flat[m+"."+k] = declDigest(decl) + } + } + raw, _ := json.Marshal(flat) + return string(raw) +} + +// ---- compare ---------------------------------------------------------------------------------------------- + +// Difference is one thing the own sources and the controller's mirrors say differently, and why. +type Difference struct { + What string `json:"what"` + Module string `json:"module,omitempty"` + Node string `json:"node,omitempty"` + Key string `json:"key,omitempty"` + Pull string `json:"pull,omitempty"` + Own json.RawMessage `json:"own,omitempty"` + Mirror json.RawMessage `json:"mirror,omitempty"` + Why string `json:"why"` +} + +// The kinds of difference. +const ( + DiffDeclared = "declared" + DiffPredefined = "predefined" + DiffPull = "pull" + DiffValue = "value" +) + +// Comparison is what compare answers. +type Comparison struct { + Source string `json:"source"` + Differences []Difference `json:"differences"` + Count int `json:"count"` + Said string `json:"said"` +} + +// Compare lists every difference (ADR 0565 §4 and its insight). The caller holds no lock. +func (c *Config) Compare() (*Comparison, error) { + c.mu.Lock() + defer c.mu.Unlock() + return c.compare() +} + +// compare is Compare; the caller holds the lock. +func (c *Config) compare() (*Comparison, error) { + sw, err := c.source() + if err != nil { + return nil, err + } + own, err := c.sourcesFrom(SourceInterim) + if err != nil { + return nil, fmt.Errorf("mesh-config's own sources cannot be read: %w", err) + } + mirror := own + mirror.Declared, mirror.OnNode, mirror.Unresolved = nil, nil, nil + c.mirrorDeclarations(&mirror) + var diffs []Difference + + // Each placed module's declarations: `declared` against the mirror's key for it. + for _, module := range modulesOf(own.Assignments) { + var ownRaw []byte + if e, err := c.Store.Get(DeclaredState, module); err == nil && e != nil { + var k DeclaredKept + if json.Unmarshal(e.Value, &k) == nil { + ownRaw, _ = json.Marshal(normal(k.Config)) + } + } + var mirrorRaw []byte + if d, ok := mirror.Declared[module]; ok { + mirrorRaw, _ = json.Marshal(normal(d.Config)) + } + switch { + case ownRaw == nil && mirrorRaw == nil: + diffs = append(diffs, Difference{What: DiffDeclared, Module: module, Why: "neither side holds its declarations"}) + case ownRaw == nil: + diffs = append(diffs, Difference{What: DiffDeclared, Module: module, Mirror: mirrorRaw, + Why: "the mirror holds its declarations and declared does not"}) + case mirrorRaw == nil: + diffs = append(diffs, Difference{What: DiffDeclared, Module: module, Own: ownRaw, + Why: "declared holds its declarations and the mirror does not"}) + case !bytes.Equal(ownRaw, mirrorRaw): + diffs = append(diffs, Difference{What: DiffDeclared, Module: module, Own: ownRaw, Mirror: mirrorRaw, + Why: "the trunk's manifest declares it otherwise than the mirror"}) + } + } + + // The predefined list: the indexer's against the mirror's. + il, iok, ierr := c.indexerList() + ml, mok, merr := c.mirrorList() + switch { + case ierr != nil || merr != nil: + diffs = append(diffs, Difference{What: DiffPredefined, Why: "a list cannot be read: " + errors.Join(ierr, merr).Error()}) + case !iok || !mok: + if iok != mok { + diffs = append(diffs, Difference{What: DiffPredefined, Why: fmt.Sprintf("the indexer holds a list: %v; the "+ + "mirror holds one: %v", iok, mok)}) + } + default: + for _, k := range unionKeys(il.Keys, ml.Keys) { + a, aok := il.Keys[k] + b, bok := ml.Keys[k] + ar, _ := json.Marshal(a) + br, _ := json.Marshal(b) + switch { + case !aok: + diffs = append(diffs, Difference{What: DiffPredefined, Key: k, Mirror: br, Why: "only the mirror's list holds it"}) + case !bok: + diffs = append(diffs, Difference{What: DiffPredefined, Key: k, Own: ar, Why: "only the indexer's list holds it"}) + case !bytes.Equal(ar, br): + diffs = append(diffs, Difference{What: DiffPredefined, Key: k, Own: ar, Mirror: br, + Why: "the two lists declare it otherwise"}) + } + } + } + + // Each open pull request: the indexer's candidate against what the controller told by pull-declares. + cands, cerr := c.candidates() + if cerr != nil { + diffs = append(diffs, Difference{What: DiffPull, Why: "the indexer's config candidates cannot be read: " + cerr.Error()}) + } + for _, k := range unionKeys(cands, own.Pulls) { + a, aok := cands[k] + b, bok := own.Pulls[k] + switch { + case !aok: + diffs = append(diffs, Difference{What: DiffPull, Pull: b.Pull, Why: "the controller told its declarations and the indexer holds no candidate"}) + case !bok: + diffs = append(diffs, Difference{What: DiffPull, Pull: a.Pull, Why: "the indexer holds a candidate and the controller told nothing"}) + case a.Head != b.Head: + diffs = append(diffs, Difference{What: DiffPull, Pull: a.Pull, Why: fmt.Sprintf("the indexer's candidate is at %s, "+ + "the controller told %s", short(a.Head), short(b.Head))}) + case declaresDigest(a.Declares) != declaresDigest(b.Declares): + diffs = append(diffs, Difference{What: DiffPull, Pull: a.Pull, Why: "the two declare its config otherwise"}) + } + } + + // Every placed module's value on every node, resolved both ways. + ownValues, mirrorValues := Resolve(own), Resolve(mirror) + for _, k := range unionKeys(ownValues, mirrorValues) { + a, aok := ownValues[k] + b, bok := mirrorValues[k] + ar, _ := json.Marshal(a) + br, _ := json.Marshal(b) + node, module, key := a.Node, a.Module, a.Key + if !aok { + node, module, key = b.Node, b.Module, b.Key + } + d := Difference{What: DiffValue, Node: node, Module: module, Key: key} + switch { + case !aok: + d.Mirror, d.Why = br, "only the mirror's declarations resolve it on this node" + case !bok: + d.Own, d.Why = ar, "only mesh-config's own declarations (with this node's build) resolve it" + case !bytes.Equal(ar, br): + d.Own, d.Mirror, d.Why = ar, br, "it resolves to another value or source" + default: + continue + } + diffs = append(diffs, d) + } + for _, u := range mirror.Unresolved { + diffs = append(diffs, Difference{What: DiffValue, Node: u.Node, Module: u.Module, Why: "not resolved from the mirror: " + u.Why}) + } + for _, u := range own.Unresolved { + diffs = append(diffs, Difference{What: DiffValue, Node: u.Node, Module: u.Module, Why: "not resolved from its own source: " + u.Why}) + } + sort.SliceStable(diffs, func(i, j int) bool { + a, b := diffs[i], diffs[j] + return a.What+a.Module+a.Node+a.Key+a.Pull < b.What+b.Module+b.Node+b.Key+b.Pull + }) + if diffs == nil { + diffs = []Difference{} + } + cmp := &Comparison{Source: sw.Source, Differences: diffs, Count: len(diffs)} + if len(diffs) == 0 { + cmp.Said = "nothing differs: the take-over may be asked" + } else { + cmp.Said = fmt.Sprintf("%d difference(s): the take-over is refused while any stands", len(diffs)) + } + return cmp, nil +} + +// normal is declarations in one encoding: a null default as none. +func normal(d map[string]Declaration) map[string]Declaration { + out := map[string]Declaration{} + for k, decl := range d { + if !decl.HasDefault() { + decl.Default = nil + } else { + decl.Default = compact(decl.Default) + } + out[k] = decl + } + return out +} + +func unionKeys[A, B any](a map[string]A, b map[string]B) []string { + seen := map[string]bool{} + for k := range a { + seen[k] = true + } + for k := range b { + seen[k] = true + } + return sortedKeys(seen) +} + +// ---- take-over and hand-back --------------------------------------------------------------------------------- + +// TakeoverAsk is one take-over asked of the operator, `takeover.ask.`: what the ask is composed from again, and +// where it stands. +type TakeoverAsk struct { + ID string `json:"id"` + From string `json:"from"` + Why string `json:"why"` + At time.Time `json:"at"` + State string `json:"state"` + Outcome string `json:"outcome,omitempty"` + Decided time.Time `json:"decided,omitzero"` + Answered string `json:"answered,omitempty"` + Recorded bool `json:"recorded,omitempty"` +} + +func askKey(id string) string { return "ask." + id } + +// act is what an answer to the take-over binds. +func (t TakeoverAsk) act(option string) asks.Act { + return asks.Act{"verb": Seat + "." + TakeOverVerb, "option": option, "ask": t.ID, "from": t.From, + "at": t.At.UTC().Format(time.RFC3339Nano)} +} + +// ask is the take-over's ask, composed from it alone. +func (t TakeoverAsk) ask() asks.Ask { + why, _ := sayable("Why: " + t.Why) + a := asks.Ask{ID: t.ID, Headline: "Switch config to its own sources", + Explanation: strings.Join([]string{ + "mesh-config would read the list of predefined keys and each pull request's config from the delivery's " + + "indexer, no longer from the controller. Nothing any node resolves changes: compare showed no difference.", + point(why)}, "\n"), + Who: asks.Operator, Expires: t.At.Add(ProposalLasts), + OnExpiry: "nothing changes; config keeps reading the controller's records", About: "config.takeover", + Details: "Read what was compared with mesh-config's compare verb, through the mesh MCP server. hand-back returns " + + "to the controller's records until the controller's lease is handed away."} + for _, o := range []struct{ id, label, does string }{ + {OptionApprove, "Approve", "config switches to its own sources at once, if nothing differs then"}, + {OptionDecline, "Decline", "nothing changes"}, + } { + binds, _ := asks.ActDigest(t.act(o.id)) + a.Options = append(a.Options, asks.Option{ID: o.id, Label: o.label, Does: o.does, Level: asks.Approve, Binds: binds}) + } + return a +} + +// TakeoverAnswer is what take-over and hand-back answer. +type TakeoverAnswer struct { + Said string `json:"said"` + Ask string `json:"ask,omitempty"` + Source string `json:"source"` +} + +// TakeOver asks the operator's Approve to switch to the own sources, refused while compare shows a difference. +func (c *Config) TakeOver(call Call) (*TakeoverAnswer, error) { + if strings.TrimSpace(call.Why) == "" { + return nil, errors.New("say why the take-over is asked now: it is shown to the operator") + } + if call.Caller == "" || call.Caller == Unproven { + return nil, errors.New("the call names no proven caller") + } + if c.Asks == nil { + return nil, errors.New("the operator cannot be asked from here, so nothing is switched") + } + c.mu.Lock() + defer c.mu.Unlock() + sw, err := c.source() + if err != nil { + return nil, err + } + if sw.Source == SourceIndexer { + return &TakeoverAnswer{Source: sw.Source, Said: "config reads its own sources already, since " + + sw.At.UTC().Format(time.RFC3339) + " (" + sw.By + ")"}, nil + } + cmp, err := c.compare() + if err != nil { + return nil, err + } + if cmp.Count > 0 { + first := cmp.Differences[0] + return nil, fmt.Errorf("the take-over is refused: compare shows %d difference(s), the first %s %s%s%s%s: %s. "+ + "Nothing was asked", cmp.Count, first.What, first.Module, first.Node, first.Key, first.Pull, first.Why) + } + now := c.now() + var open []string + _ = c.each(TakeoverState, func(key string, raw []byte) error { + var t TakeoverAsk + if strings.HasPrefix(key, "ask.") && json.Unmarshal(raw, &t) == nil && t.State == ProposalWaiting && + now.Before(t.At.Add(ProposalLasts)) { + open = append(open, t.ID) + } + return nil + }) + if len(open) > 0 { + return nil, fmt.Errorf("the take-over is asked already, as %s, and waits for the operator's answer", open[0]) + } + t := TakeoverAsk{ID: "takeover-" + newProposalID(), From: call.Caller, Why: strings.TrimSpace(call.Why), At: now.UTC(), + State: ProposalWaiting} + ask := t.ask() + if err := ask.Check(now); err != nil { + return nil, err + } + if err := c.Store.Put(TakeoverState, askKey(t.ID), t); err != nil { + return nil, fmt.Errorf("the take-over could not be kept, so the operator was not asked: %w", err) + } + if err := c.Asks.Publish(asks.AskSubject(Module), ask, ask.ID); err != nil { + t.State, t.Outcome = ProposalUnasked, "the ask could not be published: "+err.Error() + _ = c.Store.Put(TakeoverState, askKey(t.ID), t) + return nil, fmt.Errorf("the operator could not be asked: %w", err) + } + return &TakeoverAnswer{Ask: t.ID, Source: sw.Source, Said: fmt.Sprintf("the operator is asked %q under "+ + "mesh-config's own name; nothing changes until they approve, and on Approve config switches if nothing differs "+ + "then", ask.Headline)}, nil +} + +// decideTakeovers acts on each take-over ask the router has closed. The caller holds the lock. +func (c *Config) decideTakeovers() []string { + var said []string + var waiting []TakeoverAsk + _ = c.each(TakeoverState, func(key string, raw []byte) error { + var t TakeoverAsk + if strings.HasPrefix(key, "ask.") && json.Unmarshal(raw, &t) == nil && t.State == ProposalWaiting { + waiting = append(waiting, t) + } + return nil + }) + now := c.now() + for _, t := range waiting { + keep := func(state, outcome string) string { + t.State, t.Outcome, t.Decided = state, outcome, now + _ = c.Store.Put(TakeoverState, askKey(t.ID), t) + return "take-over " + t.ID + ": " + outcome + } + e, err := c.Asks.Record(t.ID) + if err != nil || e == nil { + if err == nil && now.After(t.At.Add(ProposalLasts+time.Hour)) { + said = append(said, keep(ProposalEnded, "the router keeps no record of its ask; nothing was switched")) + } + continue + } + var rec routerRecord + if json.Unmarshal(e.Value, &rec) != nil || rec.State == "open" || rec.Warrant == nil { + continue + } + w := *rec.Warrant + if w.Asker != Module || w.Ask != t.ID { + said = append(said, keep(ProposalRefused, "the router's record is for another ask; nothing was switched")) + continue + } + if w.Outcome != asks.OutcomeChosen { + said = append(said, keep(ProposalEnded, "the ask "+string(w.Outcome)+"; nothing was switched")) + continue + } + option, err := w.For(Module, t.ask()) + if err == nil { + err = option.Performs(t.act(option.ID)) + } + if err != nil { + said = append(said, keep(ProposalRefused, "the answer is refused: "+err.Error())) + continue + } + t.Answered = w.Says() + if option.ID != OptionApprove { + said = append(said, keep(ProposalDeclined, "declined by the operator; nothing was switched")) + continue + } + cmp, err := c.compare() + switch { + case err != nil: + said = append(said, keep(ProposalRefused, "approved, and compare could not be read, so nothing was switched: "+err.Error())) + continue + case cmp.Count > 0: + said = append(said, keep(ProposalRefused, fmt.Sprintf("approved, and refused: compare shows %d difference(s) "+ + "now; nothing was switched, ask again once it is empty", cmp.Count))) + continue + } + sw := Switch{Source: SourceIndexer, At: now.UTC(), By: approvedTakeover(w), Why: t.Why, Ask: t.ID} + if err := c.Store.Put(TakeoverState, SwitchKey, sw); err != nil { + said = append(said, fmt.Sprintf("take-over %s: approved, and the switch could not be written (%v); read again "+ + "on the next look", t.ID, err)) + continue + } + if err := c.Asks.Warranted(t.ID); err == nil { + t.Recorded = true + } + said = append(said, keep(ProposalApproved, "approved by the operator: config reads its own sources from now")) + } + return said +} + +func approvedTakeover(w asks.Warrant) string { + if w.By == nil { + return "the operator" + } + return fmt.Sprintf("the %s, as %s identity %s", w.By.Who, w.By.Kind, w.By.Identity) +} + +// HandBack returns to the controller's records until the controller's lease is handed away. +func (c *Config) HandBack(call Call) (*TakeoverAnswer, error) { + if strings.TrimSpace(call.Why) == "" { + return nil, errors.New("say why config hands back: it is kept with the switch") + } + if call.Caller == "" || call.Caller == Unproven { + return nil, errors.New("the call names no proven caller") + } + c.mu.Lock() + defer c.mu.Unlock() + if c.leaseAway() { + return nil, errors.New("hand-back is gone: the controller handed its lease away, and its records end with it " + + "(novox/hq ADR 0565 §4)") + } + sw, err := c.source() + if err != nil { + return nil, err + } + if sw.Source == SourceController { + return &TakeoverAnswer{Source: sw.Source, Said: "config reads the controller's records already"}, nil + } + now := c.now() + back := Switch{Source: SourceController, At: now.UTC(), By: call.Caller, Why: strings.TrimSpace(call.Why)} + if err := c.Store.Put(TakeoverState, SwitchKey, back); err != nil { + return nil, fmt.Errorf("the switch could not be written: %w", err) + } + answer := &TakeoverAnswer{Source: back.Source, Said: "config reads the controller's records again: its mirrors " + + "declared-config and predefined, and pull-declares"} + if err := c.Store.HandAct("handed config back to the controller's records", back.Why, "config-hand-back"); err != nil { + answer.Said += "; the hand-act log does not hold it: " + err.Error() + } + return answer, nil +} + +// Declared is the read of `declared`: every module's key, or one whole. +func (c *Config) Declared(module string) (any, error) { + if module != "" { + e, err := c.Store.Get(DeclaredState, module) + if err != nil { + return nil, err + } + if e == nil { + return nil, fmt.Errorf("declared holds no %s: it is placed nowhere, or its manifest cannot be read", module) + } + return json.RawMessage(e.Value), nil + } + rows := map[string]any{} + err := c.each(DeclaredState, func(key string, raw []byte) error { + var k DeclaredKept + if json.Unmarshal(raw, &k) == nil { + rows[key] = map[string]any{"manifest": k.Manifest, "commit": k.Commit, "keys": len(k.Config)} + } + return nil + }) + sw, _ := c.source() + return map[string]any{"declared": rows, "count": len(rows), "source": sw}, err +} + +// Resync reads every source again and writes what differs: the published list, the resolved values, `declared`, and +// after the take-over the indexer's config candidates (ADR 0565 §15). +func (c *Config) Resync() (any, error) { + r, err := c.Reconcile() + if err != nil { + return r, err + } + c.mu.Lock() + defer c.mu.Unlock() + out := map[string]any{"resolution": r} + if sw, err := c.source(); err == nil && sw.Source == SourceIndexer { + out["candidates"] = c.takeCandidates() + } + return out, nil +} diff --git a/modules/mesh-config/cmd/mesh-config/takeover_test.go b/modules/mesh-config/cmd/mesh-config/takeover_test.go new file mode 100644 index 00000000..c66b21f8 --- /dev/null +++ b/modules/mesh-config/cmd/mesh-config/takeover_test.go @@ -0,0 +1,238 @@ +package main + +import ( + "encoding/json" + "slices" + "strings" + "testing" + "time" + + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +// The trunk's manifests aMesh declares, as the controller's mirror declared-config holds the same declarations. +const ( + claudeDeclared = `{"module":"claude-code","settings":{ + "managed":{"kind":"","default":{},"why":"","type":"object","description":"managed settings","trusted":true,"predefined":"agent-managed-settings"}}}` + oldDeclared = `{"module":"old","settings":{"workers":{"kind":"preference","default":4,"why":"two per core"}}}` +) + +// anAgreeingMesh is aProposingMesh whose controller mirrors say what mesh-config's own sources say: the mirror's +// declarations of every placed module, and the indexer's predefined list equal to the mirror's. +func anAgreeingMesh(t *testing.T) (*Config, *memStore, *standIn) { + t.Helper() + c, m, r := aProposingMesh(t) + m.mirror(t, MirrorDeclaredRead, "dunst", dunstDeclared) + m.mirror(t, MirrorDeclaredRead, "claude-code", claudeDeclared) + m.mirror(t, MirrorDeclaredRead, "old", oldDeclared) + indexerList(t, m, "aaaaaaaaaaaa", `{"agent-managed-settings":{"type":"object","description":"managed settings","trusted":true}, + "font-size":{"type":"number","description":"text size, in points","minimum":6,"maximum":72}}`) + if _, err := c.Reconcile(); err != nil { + t.Fatal(err) + } + return c, m, r +} + +// indexerList is the indexer's `predefined.list` at a commit. +func indexerList(t *testing.T, m *memStore, commit, list string) { + t.Helper() + raw, _ := json.Marshal(delivery.PredefinedKept{V: 1, Commit: commit, List: json.RawMessage(list)}) + m.mirror(t, IndexerPredefinedRead, delivery.PredefinedListKey, string(raw)) +} + +// candidate is the indexer's config candidate of a pull request. +func candidate(t *testing.T, m *memStore, pull, head, declares string) { + t.Helper() + key, err := delivery.ConfigCandidateKey(pull) + if err != nil { + t.Fatal(err) + } + m.mirror(t, CandidatesRead, key, `{"v":1,"pull":"`+pull+`","head":"`+head+`","declares":`+declares+`}`) +} + +func kinds(cmp *Comparison) []string { + var out []string + for _, d := range cmp.Differences { + k := d.What + ":" + d.Module + d.Pull + d.Key + if d.Node != "" { + k += "@" + d.Node + } + if !slices.Contains(out, k) { + out = append(out, k) + } + } + return out +} + +// compare lists every difference with its reason — a module's declarations, the predefined list, a pull request's +// config, and each value a node resolves both ways — and the take-over is refused while one stands (ADR 0565 §4 and +// its insight of 2026-10-29; issue 1953 L5-C4). +func TestCompareListsEveryDifferenceAndTheTakeOverWaitsForNone(t *testing.T) { + cases := []struct { + name string + change func(t *testing.T, m *memStore) + want []string + }{ + {"all agree", func(*testing.T, *memStore) {}, nil}, + {"the mirror declares a default otherwise", func(t *testing.T, m *memStore) { + m.mirror(t, MirrorDeclaredRead, "dunst", strings.Replace(dunstDeclared, `"default":300`, `"default":400`, 1)) + }, []string{"declared:dunst", "value:dunstwidth@g14", "value:dunstwidth@novox"}}, + {"the mirror lacks a placed module", func(t *testing.T, m *memStore) { _ = m.Delete(MirrorDeclaredRead, "old") }, + []string{"declared:old", "value:old@g14", "value:oldworkers@g14"}}, + {"a node's build adds a key the mirror lacks", func(t *testing.T, m *memStore) { + m.want(t, "g14", "dunst", "src2:"+strings.Repeat("ab", 32), strings.Replace(dunstDeclared, `"width"`, `"height":{"type":"number","default":5},"width"`, 1)) + }, []string{"value:dunstheight@g14", "value:dunstheight@novox"}}, // a key only a build declares resolves on every node (#558) + {"the lists differ", func(t *testing.T, m *memStore) { + indexerList(t, m, "bbbbbbbbbbbb", `{"font-size":{"type":"number","description":"text size, in points","minimum":6,"maximum":72}}`) + }, []string{"predefined:agent-managed-settings"}}, + {"a pull request only the indexer holds", func(t *testing.T, m *memStore) { + candidate(t, m, "novox/mesh-catalog#9", "h1", `{"dunst":{"gap":{"kind":"","type":"number","default":2}}}`) + }, []string{"pull:novox/mesh-catalog#9"}}, + } + for _, tc := range cases { + c, m, r := anAgreeingMesh(t) + tc.change(t, m) + cmp, err := c.Compare() + if err != nil { + t.Fatalf("%s: %v", tc.name, err) + } + if got := kinds(cmp); !slices.Equal(got, tc.want) { + t.Errorf("%s: compare lists %v, want %v (%+v)", tc.name, got, tc.want, cmp.Differences) + } + for _, d := range cmp.Differences { + if d.Why == "" { + t.Errorf("%s: a difference without its reason: %+v", tc.name, d) + } + } + a, err := c.TakeOver(Call{Why: "compare is empty", Caller: "person.jochen"}) + if len(tc.want) > 0 { + if err == nil || !strings.Contains(err.Error(), "refused") || len(r.subjects) != 0 { + t.Errorf("%s: the take-over was asked while compare differs: %v", tc.name, err) + } + continue + } + if err != nil || a.Ask == "" || len(r.subjects) != 1 { + t.Errorf("%s: the take-over was not asked: %v", tc.name, err) + } + } +} + +// The operator's Approve switches config to the indexer: pull-declares is refused, the list and the pull requests' +// config come from the indexer; hand-back returns to the mirrors until the lease write and is gone after it; from the +// lease write a trusted value is set only on an Approve; declared is written once per manifest (ADR 0565 §3, §4, §13). +func TestTheTakeOverAndTheHandBack(t *testing.T) { + c, m, r := anAgreeingMesh(t) + steps := []struct { + name string + do func() (string, error) + want string // what the answer or the error says + }{ + {"declared is written when it differs, and once", func() (string, error) { + _ = m.Delete(DeclaredState, "dunst") + first, err := c.Reconcile() + again, _ := c.Reconcile() + return strings.Join(first.Declared, ",") + "|" + strings.Join(again.Declared, ","), err + }, "dunst|"}, + {"pull-declares taken before the take-over", func() (string, error) { + a, err := c.PullDeclares(Call{Caller: ControllerCaller, Pull: "novox/mesh-catalog#7", Head: "h0", + Declares: map[string]any{}}) + return answered(a), err + }, "keeps what"}, + {"the take-over is asked", func() (string, error) { + _ = m.Delete(PullsState, "novox.mesh-catalog.7") // compare holds the told pull request against no candidate + a, err := c.TakeOver(Call{Why: "the release is delivered", Caller: "person.jochen"}) + return answered(a), err + }, "is asked"}, + {"the operator approves: switched to the indexer", func() (string, error) { + r.choose(t, r.lastAsk(t), OptionApprove, nil) + out, err := c.Decide() + return strings.Join(out, "; "), err + }, "reads its own sources from now"}, + {"pull-declares refused after it, naming the indexer", func() (string, error) { + _, err := c.PullDeclares(Call{Caller: ControllerCaller, Pull: "novox/mesh-catalog#8", Head: "h1", + Declares: map[string]any{}}) + return "", err + }, "config-candidates"}, + {"the list comes from the indexer", func() (string, error) { + indexerList(t, m, "cccccccccccc", `{"font-size":{"type":"number","description":"text size","minimum":6,"maximum":72}, + "agent-managed-settings":{"type":"object","description":"managed settings","trusted":true}, + "theme":{"type":"string","description":"the desktop's theme"}}`) + res, err := c.Reconcile() + return res.Published, err + }, "theme"}, + {"a pull request's config comes from the indexer", func() (string, error) { + candidate(t, m, "novox/mesh-catalog#9", "h9", `{"dunst":{"gap":{"kind":"","type":"number","default":2}}}`) + res, err := c.Reconcile() + return strings.Join(res.Candidates, "; "), err + }, "from the indexer"}, + {"hand-back returns to the mirrors", func() (string, error) { + a, err := c.HandBack(Call{Why: "a difference was found after the switch", Caller: "person.jochen"}) + return answered(a), err + }, "controller's records again"}, + {"after it, declarations resolve from the mirror", func() (string, error) { + m.mirror(t, MirrorDeclaredRead, "dunst", strings.Replace(dunstDeclared, `"default":300`, `"default":400`, 1)) + _, err := c.Reconcile() + return string(m.resolved(t, "g14", "dunst", "width").Value), err + }, "400"}, + {"hand-back is gone after the lease write", func() (string, error) { + m.mirror(t, LeaseRead, "holder", `{"handed-to":"the operator's hand-over"}`) + _, err := c.HandBack(Call{Why: "again", Caller: "person.jochen"}) + return "", err + }, "gone"}, + {"the terminal sets no trusted value after the lease write", func() (string, error) { + _, err := c.Change(Call{Action: "set", Key: "managed", Module: "claude-code", Node: "g14", Value: map[string]any{}, + Why: "x", Caller: ControllerCaller, By: "person.jochen"}) + return "", err + }, "propose"}, + } + for _, s := range steps { + got, err := s.do() + text := got + if err != nil { + text = err.Error() + } + if !strings.Contains(text, s.want) { + t.Errorf("%s: said %q, want %q", s.name, text, s.want) + } + } + if !slices.Contains(m.handActs, "handed config back to the controller's records | a difference was found after the switch | config-hand-back") { + t.Errorf("the hand-back is not recorded: %v", m.handActs) + } + if e, _ := m.Get(DeclaredState, "dunst"); e == nil || !strings.Contains(string(e.Value), `"manifest":"`) || !strings.Contains(string(e.Value), `"commit":"c1"`) { + t.Errorf("declared holds no dunst with its manifest: %v", e) + } + // An Approve that finds compare differing at the answer switches nothing. + c2, m2, r2 := anAgreeingMesh(t) + if _, err := c2.TakeOver(Call{Why: "now", Caller: "person.jochen"}); err != nil { + t.Fatal(err) + } + m2.mirror(t, MirrorDeclaredRead, "dunst", strings.Replace(dunstDeclared, `"default":300`, `"default":400`, 1)) + r2.choose(t, r2.lastAsk(t), OptionApprove, nil) + out, _ := c2.Decide() + if sw, _ := c2.source(); sw.Source != SourceInterim || !strings.Contains(strings.Join(out, ""), "difference") { + t.Errorf("an Approve while compare differs switched: %s, %v", sw.Source, out) + } +} + +// lastAsk is the newest ask the router took. +func (r *standIn) lastAsk(t *testing.T) string { + t.Helper() + r.mu.Lock() + defer r.mu.Unlock() + var newest string + var at time.Time + for id, a := range r.asks { + if newest == "" || a.Expires.After(at) { + newest, at = id, a.Expires + } + } + if newest == "" { + t.Fatal("the router took no ask") + } + return newest +} + +func answered(a any) string { + raw, _ := json.Marshal(a) + return string(raw) +} diff --git a/modules/mesh-config/go.mod b/modules/mesh-config/go.mod index 6a0791d7..0c8a0a52 100644 --- a/modules/mesh-config/go.mod +++ b/modules/mesh-config/go.mod @@ -2,4 +2,13 @@ module mesh-config go 1.26.0 -require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261028174713-38dbe1d3a388 +require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 + +require ( + github.com/klauspost/compress v1.20.0 // indirect + github.com/nats-io/nats.go v1.54.0 // indirect + github.com/nats-io/nkeys v0.4.16 // indirect + github.com/nats-io/nuid v1.0.1 // indirect + golang.org/x/crypto v0.57.0 // indirect + golang.org/x/sys v0.48.0 // indirect +) diff --git a/modules/mesh-config/go.sum b/modules/mesh-config/go.sum index 8c9ba7ca..05084f7c 100644 --- a/modules/mesh-config/go.sum +++ b/modules/mesh-config/go.sum @@ -1,2 +1,26 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261028174713-38dbe1d3a388 h1:ycu/sNieTdLpD3bzhZOCXdzcoqXUOxPRmqWV019REpo= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261028174713-38dbe1d3a388/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 h1:lKU3HjNXGiVbLW2hUgFjU/TMTUYMlNnnmZxG5T7biQA= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= +github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= +github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= +github.com/google/go-tpm v0.9.8/go.mod h1:h9jEsEECg7gtLis0upRBQU+GhYVH6jMjrFxI8u6bVUY= +github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA= +github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI= +github.com/minio/highwayhash v1.0.4 h1:asJizugGgchQod2ja9NJlGOWq4s7KsAWr5XUc9Clgl4= +github.com/minio/highwayhash v1.0.4/go.mod h1:GGYsuwP/fPD6Y9hMiXuapVvlIUEhFhMTh0rxU3ik1LQ= +github.com/nats-io/jwt/v2 v2.8.1 h1:V0xpGuD/N8Mi+fQNDynXohVvp7ZztevW5io8CUWlPmU= +github.com/nats-io/jwt/v2 v2.8.1/go.mod h1:nWnOEEiVMiKHQpnAy4eXlizVEtSfzacZ1Q43LIRavZg= +github.com/nats-io/nats-server/v2 v2.11.17 h1:GKEghcFK6A+aFx11Yf1LjgLC3txAwvyhnYzhBIQZA8I= +github.com/nats-io/nats-server/v2 v2.11.17/go.mod h1:B1sFVz4StNosQ903ak4N1G01Fl/9f8e06mXpFIE2K24= +github.com/nats-io/nats.go v1.54.0 h1:vsXoOxjHp/GmPUN+EcI7uOf/uB+iAP+kEsAFNQN0yzA= +github.com/nats-io/nats.go v1.54.0/go.mod h1:y+DZoD1oBOYfZTU681eTUiUjI0vbqYGixNVFHcjHJ0k= +github.com/nats-io/nkeys v0.4.16 h1:rd5oAuLOb8mnAycB0xleuEBNS1pVVnN0fv/FF34Eypg= +github.com/nats-io/nkeys v0.4.16/go.mod h1:llLgWoI0o4z/Q57q2R1kHfmocyhGV6VG/U18Glg1Afs= +github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= +github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= +golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M= +golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA= +golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo= +golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= +golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= +golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= diff --git a/modules/mesh-config/module.json b/modules/mesh-config/module.json index 66db5b60..9554db29 100644 --- a/modules/mesh-config/module.json +++ b/modules/mesh-config/module.json @@ -7,7 +7,10 @@ "name": "mesh-config", "scope": "mesh", "serves": [ - "config" + "config", + "compare", + "take-over", + "hand-back" ] } ], @@ -23,6 +26,8 @@ "gitea.forge.pull-request.closed" ], "invokes": [ + "seat:mesh-hand-acts.hand-act", + "seat:mesh-hand-acts.warranted", "seat:mesh-controller.hand-act", "seat:mesh-controller.warranted", "gitea.gitea_get_pull_request" @@ -33,7 +38,18 @@ "mesh-delivery-builder.builds", "mesh-delivery-indexer.modules", "mesh-delivery-indexer.manifests", - "mesh-controller.predefined" + "mesh-controller.predefined", + "mesh-controller.declared-config", + "mesh-delivery-indexer.predefined", + "mesh-delivery-indexer.config-candidates", + "mesh-controller.lease" + ], + "tools": [ + "config_compare", + "config_take_over", + "config_hand_back", + "config_declared", + "config_resync" ], "state": [ "values", @@ -41,7 +57,9 @@ "predefined", "proposals", "pulls", - "kept" + "kept", + "declared", + "takeover" ], "logs": [ "history" diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/approval.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/approval.go index 2a7ba773..bfbcfae4 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/approval.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/approval.go @@ -330,7 +330,7 @@ func (h *Holder) endStaleAsks() { h.endStaleSettles() for _, d := range h.deliveries { r := d.Ready - if r == nil || r.Merged != nil || d.MergedAs != "" || r.Declined != nil || (r.AskID == "" && r.Approved == nil) || len(r.Heads) == 0 { + if r == nil || r.Merged != nil || d.MergedAs != "" || r.declined() || (r.AskID == "" && r.Approved == nil) || len(r.Heads) == 0 { continue } switch { @@ -418,6 +418,7 @@ func (h *Holder) approval(d *Delivery, now time.Time) (string, func()) { h.runOf(m, now) m.Ready.AskID, m.Ready.Ask, m.Ready.Heads, m.Ready.AskedAt, m.Ready.Asks = id, &ask, heads, now, max(m.Ready.Asks, r.Asks) m.Ready.AskedKey, m.Ready.Asking, m.Ready.AskCheck = id, asking, check.Key + m.Ready.Declined, m.Ready.AnsweredAsk = nil, "" // a new ask: an answer to an earlier one does not stand on it h.keep(m) taken = append(taken, m) reads = append(reads, readOf(m, m.Commit)) @@ -608,7 +609,7 @@ func (h *Holder) pendingAsks() []string { seen := map[string]bool{} var out []string for _, d := range h.deliveries { - if r := d.Ready; r != nil && r.AskID != "" && r.Approved == nil && r.Declined == nil && !seen[r.AskID] { + if r := d.Ready; r != nil && r.AskID != "" && r.Approved == nil && !r.declined() && !seen[r.AskID] { seen[r.AskID] = true out = append(out, r.AskID) } @@ -705,10 +706,10 @@ func (h *Holder) answered(key string, ans sdk.Answer) []func() { h.mergeAgainNow(d) // the operator chose: the merge is asked at once switch ans.Option { case OptionApprove: - r.Approved = &now + r.Approved, r.Declined, r.AnsweredAsk = &now, nil, key h.Logf("[mesh-delivery] %s: the operator approved its merge into main (%s)", d.ID, r.Warrant) case OptionDecline: - r.Declined = &now + r.Declined, r.AnsweredAsk = &now, key h.Logf("[mesh-delivery] %s: the operator declined its merge into main (%s)", d.ID, r.Warrant) said = append(said, h.sayOn(d, fmt.Sprintf("mesh-delivery: **not merged.** The merge of this pull request into main at %s was "+ "declined: %s. A new head asks the operator again.", d.Commit, r.Warrant))) diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/approval_test.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/approval_test.go index fde6b6e7..3057f3d8 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/approval_test.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/approval_test.go @@ -405,3 +405,51 @@ func TestAnAnswerToAnEarlierAskingIsNotTakenAndTheMergeNamesTheAnsweredAsking(t t.Fatalf("the merge is announced %v; wanted the ask id of asking 2, %s", say, sdk.AskingID(key, 2)) } } + +// 2026-10-29 (hq#495): a group declined under one ask, a member pushed, and the group asked and approved again at the new +// heads: the member whose head did not move kept the earlier Decline beside the new Approve, and nothing was handed, with +// no word. An answer stands only for the ask it answers; whichever member is pushed, both merge on the new Approve. +func TestADeclineOfAnEarlierAskDoesNotHoldTheApproveOfTheNext(t *testing.T) { + const ctlHead, catHead, pushed = "c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1c1", "d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2d2", "e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3" + for _, push := range []struct { + repo string + number int + trunk string + }{{"novox/mesh-catalog", 2, "t3t3t3t3t3t3"}, {"novox/mesh-controller", 1, "t1t1t1t1t1t1"}} { + t.Run(push.repo, func(t *testing.T) { + w := newWorld(t) + w.op.auto = false + heads := map[string]string{"novox/mesh-controller": ctlHead, "novox/mesh-catalog": catHead} + paths := map[string]string{"novox/mesh-controller": "internal/a.go", "novox/mesh-catalog": "modules/app/module.json"} + numbers := map[string]int{"novox/mesh-controller": 1, "novox/mesh-catalog": 2} + trunks := map[string]string{"novox/mesh-controller": "t1t1t1t1t1t1", "novox/mesh-catalog": "t3t3t3t3t3t3"} + round := func(check string) { + var open []OpenPull + for _, repo := range []string{"novox/mesh-controller", "novox/mesh-catalog"} { + w.h.PullUpdated(pr(repo, numbers[repo], heads[repo], "feat/x", paths[repo])) + w.h.Checked(judgedAt(repo, numbers[repo], heads[repo], "pass", trunks[repo])) + open = append(open, openPull(repo, numbers[repo], heads[repo])) + } + w.forge.open = open + w.h.SeeOpenPulls() + w.settleAll() + w.h.Checked(CheckedEvent{Group: "feat/x", ID: check, Verdict: "pass", Summary: "both compose"}) + w.settleAll() + } + round("check-1") + w.op.answer(w.op.open()[0], OptionDecline) + w.settleAll() + heads[push.repo] = pushed + round("check-2") + open := w.op.open() + if len(open) != 1 { + t.Fatalf("the group at its new heads is not asked once: %v", open) + } + w.op.answer(open[0], OptionApprove) + w.settleAll() + if len(w.forge.merges) != 2 { + t.Fatalf("approved at the new heads, yet merged %v; stalled %+v", w.forge.merges, w.h.Stalled()) + } + }) + } +} diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/events.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/events.go index b0fc97af..be95a0c7 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/events.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/events.go @@ -96,7 +96,7 @@ func (h *Holder) withdrawForStatus(e ForgeStatusEvent) { done := map[string]bool{} for _, d := range h.deliveries { r := d.Ready - if !strings.EqualFold(d.Repository, repository) || r == nil || r.AskID == "" || r.Approved != nil || r.Declined != nil || done[r.AskID] { + if !strings.EqualFold(d.Repository, repository) || r == nil || r.AskID == "" || r.Approved != nil || r.declined() || done[r.AskID] { continue } head := d.Commit diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/group_merge.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/group_merge.go index 8b2fd6a2..b6e1897c 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/group_merge.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/group_merge.go @@ -208,7 +208,7 @@ func (h *Holder) passes() []pass { for _, id := range ids { d := h.deliveries[id] r := d.Ready - if r == nil || r.Approved == nil || r.Declined != nil || r.AskID == "" || len(r.Heads) == 0 || seen[r.AskID] || + if r == nil || r.Approved == nil || r.declined() || r.AskID == "" || len(r.Heads) == 0 || seen[r.AskID] || d.Base != MergeBranch || h.allMerged(r.Heads) { continue } @@ -335,7 +335,7 @@ func (h *Holder) MergeAgain(id, why string) (string, error) { return "", fmt.Errorf("say why: the merge asked again is the operator's word, kept in the log") } c := h.covering(d) - if c == nil || c.Approved == nil || c.Declined != nil || c.Heads[d.ID] != d.Commit { + if c == nil || c.Approved == nil || c.declined() || c.Heads[d.ID] != d.Commit { return "", fmt.Errorf("%s is not approved at its head %s: nothing to ask again (a new head is asked on its own)", id, shortOf(d.Commit, 12)) } if d.MergedAs != "" || (d.Ready != nil && d.Ready.Merged != nil) { diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/holder.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/holder.go index 79af7cda..8ba98d5c 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/holder.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/holder.go @@ -524,7 +524,7 @@ func (h *Holder) PullEdited(p PullEvent) { if after, _ := reviewWords(d); after == before { return } - if r := h.covering(d); r != nil && r.AskID != "" && r.Approved == nil && r.Declined == nil { + if r := h.covering(d); r != nil && r.AskID != "" && r.Approved == nil && !r.declined() { h.cancels = append(h.cancels, h.withdrawAsk(r.AskID, d.ID+"'s review record changed: "+shortOf(d.ReviewedAt, 12))) } } diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/merges.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/merges.go index ec1b6cec..d2634300 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/merges.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/merges.go @@ -149,11 +149,10 @@ func (h *Holder) handTurn(g *Group, d *Delivery, now time.Time) func() { return nil } for _, m := range order { - mc := h.covering(m) - if m.Ready == nil || m.Ready.Declined != nil || mc == nil || mc.Approved == nil || mc.Heads[m.ID] != m.Commit || h.readyWaits(m) != "" { - return nil // the others are not approved at their heads yet: nothing is merged before they are + if h.notHandable(m) != "" { + return nil // the others are not approved at their heads yet: nothing is merged before they are (said by notHanded) } - reads = append(reads, readOf(m, mc.Heads[m.ID])) + reads = append(reads, readOf(m, h.covering(m).Heads[m.ID])) } } if c == nil || c.Approved == nil || c.Heads[d.ID] == "" { @@ -169,6 +168,66 @@ func (h *Holder) handTurn(g *Group, d *Delivery, now time.Time) func() { return func() { h.hand(gid, name, key, target, reads, approved) } } +// notHandable is why a group member cannot be handed in its group's pass, empty when it can: handTurn's own test, in words. +// Under the lock. +func (h *Holder) notHandable(m *Delivery) string { + mc := h.covering(m) + switch { + case m.Ready == nil: + return m.ID + " has no ready run (" + string(m.State) + ")" + case m.Ready.declined(): + return m.ID + " stands declined since " + m.Ready.Declined.UTC().Format(time.RFC3339) + case mc == nil || mc.Approved == nil: + return m.ID + " is not approved at its head " + shortOf(m.Commit, 12) + case mc.Heads[m.ID] != m.Commit: + return m.ID + "'s head " + shortOf(m.Commit, 12) + " is not the head approved" + } + if w := h.readyWaits(m); w != "" { + return m.ID + " waits: " + w + } + return "" +} + +// notHanded is why the approved merge of d, which waits for nothing, is not handed: its group's pass cannot start, and +// says which member holds it and why. Empty when nothing holds it, or when the pass is under way or its refusal is said +// already. Under the lock. +func (h *Holder) notHanded(g *Group, d *Delivery) string { + if g == nil { + return "" + } + order := h.unmergedLive(g) + if len(order) == 0 { + return "" + } + if first := order[0].Ready; first != nil && (first.merging || !first.LastTry.IsZero() || first.RefusedSince != nil) { + return "" // the pass is under way, or its refusal is said on its own line + } + for _, m := range order { + if why := h.notHandable(m); why != "" { + return fmt.Sprintf("its group %s is not handed to the forge bridge: %s", g.ID, why) + } + } + return "" +} + +// noteNotHanded keeps on d's run why its approved merge is not handed, said once in the log when it changes, and as a +// stalled line meanwhile (readyLines). Under the lock. +func (h *Holder) noteNotHanded(d *Delivery, why string, now time.Time) { + r := d.Ready + if r == nil || r.NotHanded == why { + return + } + r.NotHanded = why + if why == "" { + r.NotHandedSince = nil + } else { + at := now + r.NotHandedSince = &at + h.Logf("[mesh-delivery] %s is approved, waits for nothing, and is NOT handed to merge: %s", d.ID, why) + } + h.keep(d) +} + // hand reads a group's required statuses and, when all are a success, writes the target as the member being merged. func (h *Holder) hand(gid, name, key, target string, reads []memberRead, approved map[string]string) { if gid != "" && !h.groupMergeable(gid, target, reads) { diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/ready.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/ready.go index 94ffc5b8..3f16589d 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/ready.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/ready.go @@ -120,11 +120,13 @@ type ReadyRun struct { // in new words, so it never keeps a claim the check no longer holds. AskCheck string `json:"ask_check,omitempty"` // Approved and Declined are when the operator chose, Warrant what the router's warrant says of who and how; Refused - // why an answer was not taken (it did not hold to the ask). - Approved *time.Time `json:"approved,omitempty"` - Declined *time.Time `json:"declined,omitempty"` - Warrant string `json:"warrant,omitempty"` - Refused string `json:"refused,omitempty"` + // why an answer was not taken (it did not hold to the ask). AnsweredAsk is the ask the answer answers: an answer stands + // only for its own ask, so a Decline of an earlier ask never stands beside the Approve of a later one (declined). + Approved *time.Time `json:"approved,omitempty"` + Declined *time.Time `json:"declined,omitempty"` + AnsweredAsk string `json:"answered_ask,omitempty"` + Warrant string `json:"warrant,omitempty"` + Refused string `json:"refused,omitempty"` // One check per delivery (novox/hq ADR 0548 §2, §4, turns.go): HeldSince is when the approved merge began to wait for the // check of other heads HeldFor names (its verdict key), running on a repository it goes into; HandNow the operator's word @@ -137,6 +139,25 @@ type ReadyRun struct { // (writes.go, issue 1945); CannotWriteSince since when. Said at once as a condition. CannotWrite string `json:"cannot_write,omitempty"` CannotWriteSince *time.Time `json:"cannot_write_since,omitempty"` + + // NotHanded is why an approved merge that waits for nothing is not handed to the forge bridge (a member of its group that + // cannot be handed, and why), NotHandedSince since when: said once in the log and as a stalled line, never silent. + NotHanded string `json:"not_handed,omitempty"` + NotHandedSince *time.Time `json:"not_handed_since,omitempty"` +} + +// declined is whether the operator's Decline stands for this run: it answered the ask the run holds now, or the run holds +// no ask any more. A Decline of an earlier ask, the run asked again since at other heads (a sibling's new head), does not +// stand (2026-10-29: hq#495 declined under one ask and approved under the next was never handed). A run kept before answers +// named their ask stands declined unless an Approve came after the Decline. +func (r *ReadyRun) declined() bool { + if r == nil || r.Declined == nil { + return false + } + if r.AnsweredAsk != "" { + return r.AskID == "" || r.AskID == r.AnsweredAsk + } + return r.Approved == nil || r.Declined.After(*r.Approved) } // MergeBranch is the one branch this owner merges into (novox/hq ADR 0398 §4): a merge into any other is its author's. @@ -197,7 +218,7 @@ func (h *Holder) readyAsks(now time.Time) []func() { asks = append(asks, a) } r := d.Ready - if r == nil || r.Merged != nil || r.Declined != nil { + if r == nil || r.Merged != nil || r.declined() { continue } if d.State == Rejected { @@ -276,9 +297,11 @@ func (h *Holder) readyAsks(now time.Time) []func() { asks = append(asks, approvalAsk) } if waits != "" || r.merging { + h.noteNotHanded(d, "", now) // what it waits for is said by its waits, or it is handed continue } if !r.LastTry.IsZero() { + h.noteNotHanded(d, "", now) continue // refused: asked again at the next event that decides whether it may merge (mergeAgainNow), with no timer } // A group is merged one member at a time, handed to the forge bridge, and the pass stops at the first refusal (ADR @@ -288,7 +311,10 @@ func (h *Holder) readyAsks(now time.Time) []func() { g = nil } if a := h.handTurn(g, d, now); a != nil { + h.noteNotHanded(d, "", now) asks = append(asks, a) + } else { + h.noteNotHanded(d, h.notHanded(g, d), now) // a merge never handed says why (2026-10-29: silent for an hour) } } asks = append(asks, h.turnWrites()...) // the merge turns, after the holds of this look (ADR 0548 §3) @@ -575,6 +601,11 @@ func (h *Holder) readyLines(now time.Time) []StalledLine { Bound: "0s", H2: "none: a new head, which asks the operator again — the operator's", Says: fmt.Sprintf("%s#%d at %s is not merged: the operator's answer %s", d.Repository, d.Number, shortOf(r.Head, 12), r.Refused)}) + case r.NotHanded != "" && r.NotHandedSince != nil && r.Approved != nil: + out = append(out, StalledLine{ID: d.ID, Number: d.Number, State: d.State, + For: now.Sub(*r.NotHandedSince).Round(time.Second).String(), Bound: "0s", + H2: "none: the member named is approved at its head and ready, or its group is asked anew by a new head — the operator's", + Says: fmt.Sprintf("%s#%d at %s is approved and %s", d.Repository, d.Number, shortOf(r.Head, 12), r.NotHanded)}) case h.answering(d, now): // The trunk moved and its check was asked again: the gate is answering it, so it is no stall yet (ADR 0505 // §4). `show` lists the refusal meanwhile. diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/turns.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/turns.go index 72b3ab85..48a68aac 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/turns.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/turns.go @@ -110,7 +110,7 @@ func (h *Holder) approvedMerges() []approvedMerge { for _, id := range ids { d := h.deliveries[id] r := d.Ready - if r == nil || r.Approved == nil || r.Declined != nil || r.AskID == "" || len(r.Heads) == 0 || seen[r.AskID] || + if r == nil || r.Approved == nil || r.declined() || r.AskID == "" || len(r.Heads) == 0 || seen[r.AskID] || d.State.Final() || d.State == Rejected || d.Base != MergeBranch || r.RefusedSince != nil || h.allMerged(r.Heads) { continue } @@ -382,7 +382,10 @@ type TurnLine struct { func (h *Holder) TurnsRead() map[string]any { h.mu.Lock() defer h.mu.Unlock() - want := h.wantedTurns() + want := map[string]sdk.Turn{} + if h.checkGates() { + want = h.wantedTurns() // as turnWrites: while the check gates no merge, no turn is written by design + } repos := make([]string, 0, len(want)) for repo := range want { repos = append(repos, repo) @@ -404,6 +407,9 @@ func (h *Holder) TurnsRead() map[string]any { lines = append(lines, l) } out := map[string]any{"turns": lines, "check-gates-merge": h.checkGates(), "bound": turnBound.String()} + if !h.checkGates() { + out["why-none"] = "the check gates no merge (check-gates-merge false): no merge waits for a check, so no turn is written" + } if h.Turns != nil { out["written"] = h.turnsLoaded var unwritten []string diff --git a/modules/mesh-delivery-gate/go.mod b/modules/mesh-delivery-gate/go.mod index 57a6c550..3e3538db 100644 --- a/modules/mesh-delivery-gate/go.mod +++ b/modules/mesh-delivery-gate/go.mod @@ -3,7 +3,7 @@ module mesh-delivery-gate go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261028174713-38dbe1d3a388 + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-delivery-gate/go.sum b/modules/mesh-delivery-gate/go.sum index 6245202d..a2b8e018 100644 --- a/modules/mesh-delivery-gate/go.sum +++ b/modules/mesh-delivery-gate/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261028174713-38dbe1d3a388 h1:ycu/sNieTdLpD3bzhZOCXdzcoqXUOxPRmqWV019REpo= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261028174713-38dbe1d3a388/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 h1:lKU3HjNXGiVbLW2hUgFjU/TMTUYMlNnnmZxG5T7biQA= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/mesh-hand-acts/.gitignore b/modules/mesh-hand-acts/.gitignore new file mode 100644 index 00000000..2a60d0a0 --- /dev/null +++ b/modules/mesh-hand-acts/.gitignore @@ -0,0 +1,2 @@ +/mesh-hand-acts +/cmd/mesh-hand-acts/mesh-hand-acts diff --git a/modules/mesh-hand-acts/README.md b/modules/mesh-hand-acts/README.md new file mode 100644 index 00000000..b435e08d --- /dev/null +++ b/modules/mesh-hand-acts/README.md @@ -0,0 +1,26 @@ +# mesh-hand-acts + +Holds the seat `mesh-hand-acts` (novox/hq ADR 0565 §10, §11): the hand-act log, moved out of the controller. Go, one +binary (`cmd/mesh-hand-acts`), served by a runtime of its own on its own bus account (`runs-as`), so that its account +alone writes the log. + +## What it keeps + +- the log `acts` (the stream `LOG_mesh-hand-acts_acts`, a module log of ADR 0297, as ADR 0565 §10 has it a stream): + every act appended in its order under its key `act--` (`warrant--` for a warrant), in the layout + of the controller's log. The log keeps each act 400 days (`max-age-days`) and refuses a new act when + full rather than drop one; `hand-acts` says how full it is against its cap. Each act is said as `act-recorded`. +- `mesh-hand-acts_warrants`: one key `.` per warrant, made once by create before its act is appended, so of + several instances one records it; a warrant whose act was lost is written again from the router's record. + +## Its verbs + +- `mesh-hand-acts.hand-act`: what, why, cause, the condition addressed, and the act's verb (a repair when not given; + `hand-act drill` for a drill, whose cause is always `drill`). Who did it is the caller the runtime proves. +- `mesh-hand-acts.warranted`: asker and ask. Read from the router's own record of that ask (`messenger.asks`), never + from the caller, and recorded once per ask; an ask still open, ended without a choice or another asker's is refused. +- `mesh-hand-acts.hand-acts`: both logs, this one and the controller's `mesh-controller_hand-acts` until its lease + ends, newest first, with the causes repaired by hand twice within a fortnight; `warrants=true` lists the warrants. + +No act is written to both logs: every recorder calls this seat from its delivery, and the controller keeps its own +acts in its own bucket until it loses its lease. diff --git a/modules/mesh-hand-acts/cmd/mesh-hand-acts/acts.go b/modules/mesh-hand-acts/cmd/mesh-hand-acts/acts.go new file mode 100644 index 00000000..6e8579e8 --- /dev/null +++ b/modules/mesh-hand-acts/cmd/mesh-hand-acts/acts.go @@ -0,0 +1,375 @@ +package main + +// The hand-act log (novox/hq ADR 0565 §10, §11): every act done by hand, with who, why and its cause, so a repair a +// person keeps making is found (S15) and a person's decision is read back. mesh-hand-acts holds the seat +// mesh-hand-acts, once for the mesh, and is the one writer of its log `acts` — a stream, as ADR 0565 §10 has it: +// the mesh's module log (ADR 0297), append-only, every act kept in its order under its key, refusing a new act when +// full rather than dropping an old one, so every act is held 400 days and longer — and of its bucket `warrants` (one +// key `.` made once by create). Each act is said as `act-recorded`. +// +// **Who did it is the runtime's word, never the caller's**: `by` is the bus user the runtime proved. A warrant is the +// router's word, never the caller's: `warranted` reads the router's own record of the asker's ask and records who +// chose, through which channel, with which proofs and which answer, once per ask. The controller's bucket +// `mesh-controller_hand-acts` stays its own until it loses its lease and is read, never written: `hand-acts` answers +// over both logs, and no act is written to both (ADR 0565 §11). + +import ( + "encoding/json" + "errors" + "fmt" + "regexp" + "sort" + "strconv" + "strings" + "sync/atomic" + "time" + + "git.novox.be/novox/mesh-sdk/go/asks" + "git.novox.be/novox/mesh-sdk/go/handacts" +) + +// The seat, the log, the states and the event, as mesh-sdk's handacts names them. +const ( + Module = handacts.Module + Seat = handacts.Seat + ActsLog = handacts.ActsLog + WarrantsState = handacts.WarrantsState + ControllerActs = handacts.ControllerActs + // RouterRecords is the router's record of every ask, `.` (ADR 0259): the warrant is read there. + RouterRecords = "messenger.asks" + RecordedEvent = handacts.EventRecorded + // RecordedSubject is where the seat says it: a seat's events are on mesh.seat..event., which its + // consumers hear (catalogue.SeatSays). + RecordedSubject = "mesh.seat." + Seat + ".event." + RecordedEvent + + VerbRecord = handacts.VerbRecord + VerbDrill = handacts.VerbDrill + VerbWarrant = handacts.VerbWarrant + CauseDrill = handacts.CauseDrill + CauseOperatorAnswer = handacts.CauseOperatorAnswer +) + +// RepeatedWithin is how far back a cause counts as repeated (S15). +const RepeatedWithin = 14 * 24 * time.Hour + +// Act is one entry of the log: mesh-sdk's handacts.Act, the controller's HandAct unchanged. +type Act = handacts.Act + +// Store is the bus's buckets and this module's log as it reaches them, through its runtime. Create writes a key only +// when it has none, and says ErrExists otherwise. +type Store interface { + Get(state, key string) ([]byte, bool, error) + Create(state, key string, value any) error + Keys(state string) ([]string, error) + // Append adds an entry to the end of a log; Last is a key's newest entry; Entries every entry in order; Fill how + // many entries and bytes the log holds against its cap. + Append(log, key string, value any) error + Last(log, key string) ([]byte, bool, error) + Entries(log string) ([][]byte, error) + Fill(log string) (Fill, error) +} + +// Fill is how full a log is: its entries, its bytes and its cap in bytes. +type Fill struct { + Entries uint64 `json:"entries"` + Bytes uint64 `json:"bytes"` + MaxBytes uint64 `json:"max-bytes"` +} + +// ErrExists is a create of a key another writer made first. +var ErrExists = errors.New("the key was made by another writer first") + +// Log is mesh-hand-acts: the log and what records into it. +type Log struct { + Store Store + // Publish says a seat event on its subject, de-duplicated by id (stdio.SeatPublish). + Publish func(subject string, body any, id string) error + Now func() time.Time + Logf func(format string, args ...any) +} + +var seq atomic.Uint64 + +// actKey is an act's key: its time and a sequence, so the keys list in order, as the controller's do. +func actKey(at time.Time) string { + return "act-" + strconv.FormatInt(at.UnixNano(), 10) + "-" + strconv.FormatUint(seq.Add(1), 10) +} + +// Recorded is what a recorder is answered: the act as kept. +type Recorded struct { + Act Act `json:"act"` + Said string `json:"said"` +} + +// HandAct records one act a caller says it did, by hand or on purpose: what, why, its cause and the condition it +// addresses. by is the caller the runtime proved. verb is the act's verb as the log names it: a repair by default, a +// drill with VerbDrill, or a recorder's own (`conditions silence`, `config set`), which S15 reads as it reads the +// controller's. +func (l *Log) HandAct(by, verb, what, why, cause, condition string) (Recorded, error) { + what, why, cause, verb = strings.TrimSpace(what), strings.TrimSpace(why), strings.TrimSpace(cause), strings.TrimSpace(verb) + switch { + case !proven(by): + return Recorded{}, fmt.Errorf("who acts is the caller the runtime proves, and it proved none (%q): nothing was recorded", by) + case what == "": + return Recorded{}, errors.New("a hand act says what was done: nothing was recorded") + case why == "": + return Recorded{}, errors.New("a hand act says why: nothing was recorded") + } + if verb == "" { + verb = VerbRecord + } + if verb == VerbDrill { + cause = CauseDrill + } + switch { + case cause == "": + return Recorded{}, errors.New("a hand act says its cause, in a word or a condition's kind: the word a second act " + + "for the same reason uses, which is how a repair done twice is found. Nothing was recorded") + case strings.EqualFold(cause, CauseDrill) && verb != VerbDrill: + return Recorded{}, errors.New("a drill is not recorded as a repair: record it with verb " + VerbDrill + + ", which no healer is wanted for. Nothing was recorded") + } + at := l.Now().UTC() + act := Act{ID: actKey(at), At: at, By: by, Verb: verb, Args: []string{what}, Why: why, Cause: cause, + Condition: strings.TrimSpace(condition)} + if err := l.record(act); err != nil { + return Recorded{}, err + } + return Recorded{Act: act, Said: fmt.Sprintf("recorded as %s: %s did %q, because %q (cause: %s)", act.ID, by, what, + why, cause)}, nil +} + +// record writes one act and says it. An act not said is still kept, and the log says so. +func (l *Log) record(act Act) error { + if err := l.Store.Append(ActsLog, act.ID, act); err != nil { + return fmt.Errorf("the act could not be recorded: %w", err) + } + if l.Publish != nil { + if err := l.Publish(RecordedSubject, act, act.ID); err != nil { + l.Logf("[%s] %s is recorded and could NOT be said as %s: %v", Module, act.ID, RecordedEvent, err) + } + } + return nil +} + +// routerRecord is what warranted reads of the router's record of an ask. +type routerRecord struct { + State string `json:"state"` + Warrant *asks.Warrant `json:"warrant"` + Ask map[string]any `json:"ask"` +} + +var askerModule = regexp.MustCompile(`^[a-z0-9][a-z0-9-]{0,62}$`) + +// Warrant is one ask's warrant as recorded: the key `.` of `warrants`, naming the act it recorded. +type Warrant struct { + Act string `json:"act"` + At time.Time `json:"at"` + By string `json:"recorded-at-the-word-of"` +} + +// Warranted records an asker's act on the operator's warrant for its ask, read from the router's record, once per ask +// however often it is asked (ADR 0274, ADR 0259). The caller gives which ask and nothing else. +func (l *Log) Warranted(caller, asker, ask string) (Recorded, error) { + switch { + case !proven(caller): + return Recorded{}, fmt.Errorf("who asks is the caller the runtime proves, and it proved none (%q): nothing was recorded", caller) + case !askerModule.MatchString(asker): + return Recorded{}, fmt.Errorf("%q is not a module's name", asker) + case !asks.UsableID(ask): + return Recorded{}, fmt.Errorf("%q is not an ask's id", ask) + } + key := asker + "." + ask + claimed, found, err := l.Store.Get(WarrantsState, key) + if err != nil { + return Recorded{}, err + } + if found { + if r, ok, err := l.already(key, claimed); err != nil || ok { + return r, err + } + } + raw, found, err := l.Store.Get(RouterRecords, key) + if err != nil { + return Recorded{}, fmt.Errorf("the router's record of %s's ask %s cannot be read: %w", asker, ask, err) + } + var rec routerRecord + if found { + if err := json.Unmarshal(raw, &rec); err != nil { + return Recorded{}, fmt.Errorf("the router's record of %s's ask %s is not readable: %w", asker, ask, err) + } + } + w := rec.Warrant + switch { + case rec.State == "open" || rec.State == "held": + return Recorded{}, fmt.Errorf("%s's ask %s is still %s: nobody has answered it. Nothing was recorded", asker, ask, rec.State) + case !found || rec.State == "" || w == nil: + return Recorded{}, fmt.Errorf("the router holds no closed record of %s's ask %s. Nothing was recorded", asker, ask) + case w.Asker != asker || w.Ask != ask: + return Recorded{}, fmt.Errorf("the router's record is for %s's ask %s. Nothing was recorded", w.Asker, w.Ask) + case w.Outcome != asks.OutcomeChosen || w.By == nil: + return Recorded{}, fmt.Errorf("%s's ask %s ended %s: no person chose, so there is no warrant to record. Nothing was "+ + "recorded", asker, ask, w.Outcome) + case w.AskDigest == "": + return Recorded{}, fmt.Errorf("the router's warrant for %s's ask %s names no ask digest. Nothing was recorded", asker, ask) + } + at := l.Now().UTC() + act := Act{ID: "warrant-" + asker + "-" + ask, At: w.At.UTC(), Verb: VerbWarrant, + Args: []string{fmt.Sprintf("the operator chose %s on %s's ask %s", w.Label, asker, ask)}, + Why: fmt.Sprintf("%s (ask %s of %s)", w.Says(), ask, asker), + By: fmt.Sprintf("the %s, as %s identity %s", w.By.Who, w.By.Kind, w.By.Identity), + Cause: CauseOperatorAnswer, Via: via(*w), Ask: ask, Proofs: w.Proofs, + RequestedBy: asker + ", recorded at the word of " + caller, + Outcome: "chosen; what " + asker + " did with it is in its own record"} + if claimed == nil { + // The claim first: of any number of instances asking at once, the one whose create lands records the act. + if err := l.Store.Create(WarrantsState, key, Warrant{Act: act.ID, At: at, By: caller}); errors.Is(err, ErrExists) { + return Recorded{Said: "another instance records " + act.ID + " now"}, nil + } else if err != nil { + return Recorded{}, fmt.Errorf("the warrant could not be recorded: %w", err) + } + } + if err := l.record(act); err != nil { + return Recorded{}, err + } + return Recorded{Act: act, Said: fmt.Sprintf("recorded as %s: %s, through %s", act.ID, act.Why, act.Via)}, nil +} + +// already answers a warrant recorded before, when its act is in the log; false when the act was lost between the +// claim and the log, so it is read from the router again and written. +func (l *Log) already(key string, raw []byte) (Recorded, bool, error) { + var w Warrant + if err := json.Unmarshal(raw, &w); err != nil { + return Recorded{}, false, fmt.Errorf("the warrant %s is not readable: %w", key, err) + } + actRaw, found, err := l.Store.Last(ActsLog, w.Act) + if err != nil || !found { + return Recorded{}, false, err + } + var act Act + _ = json.Unmarshal(actRaw, &act) + return Recorded{Act: act, Said: "already recorded as " + act.ID + ": " + act.Why}, true, nil +} + +func via(w asks.Warrant) string { + if w.By == nil { + return w.Channel + } + v := w.Channel + " (" + w.By.Kind + ")" + if w.By.Verified != "" { + v += ", " + w.By.Verified + } + return v +} + +// proven is a bus user the runtime proved: never empty, never the SDK's unproven word. +func proven(by string) bool { + by = strings.TrimSpace(by) + return by != "" && !strings.EqualFold(by, "unproven") && !strings.HasPrefix(by, "(") +} + +// Listing is what hand-acts answers: every act of both logs within the days asked, newest first, narrowed by cause +// and who, and the causes repaired by hand twice or more within a fortnight. +type Listing struct { + Acts []Act `json:"acts"` + Repeated map[string]int `json:"repeated"` + Days int `json:"days"` + // Unread says which log could not be read, so what is missing is said and never read as nothing. + Unread map[string]string `json:"unread,omitempty"` + Kept map[string]int `json:"kept"` + // Fill is how full this seat's log is against its cap: full, it refuses the next act. + Fill *Fill `json:"fill,omitempty"` +} + +// HandActs reads both logs. +func (l *Log) HandActs(days int, cause, who string) Listing { + now := l.Now() + since := now.Add(-time.Duration(days) * 24 * time.Hour) + out := Listing{Acts: []Act{}, Repeated: map[string]int{}, Days: days, Kept: map[string]int{}} + var all []Act + for _, log := range []string{ActsLog, ControllerActs} { + acts, kept, err := l.read(log, now.Add(-max(time.Duration(days)*24*time.Hour, RepeatedWithin))) + if err != nil { + if out.Unread == nil { + out.Unread = map[string]string{} + } + out.Unread[log] = err.Error() + continue + } + out.Kept[log] = kept + all = append(all, acts...) + } + for _, a := range all { + if now.Sub(a.At) <= RepeatedWithin && a.Repair() { + out.Repeated[a.Cause]++ + } + if a.At.Before(since) || (cause != "" && a.Cause != cause) || (who != "" && !strings.Contains(a.By, who)) { + continue + } + out.Acts = append(out.Acts, a) + } + for c, n := range out.Repeated { + if n < 2 { + delete(out.Repeated, c) + } + } + sort.SliceStable(out.Acts, func(i, j int) bool { return out.Acts[i].At.After(out.Acts[j].At) }) + if f, err := l.Store.Fill(ActsLog); err == nil { + out.Fill = &f + } + return out +} + +// read is every act of one log since a moment, and how many it keeps: this seat's log entry by entry, the +// controller's bucket by its keys, each of which names its time, so only those within are read whole. +func (l *Log) read(log string, since time.Time) ([]Act, int, error) { + var out []Act + keep := func(raw []byte, from string) { + var a Act + if json.Unmarshal(raw, &a) != nil || a.At.Before(since) { + return + } + a.Log = from + out = append(out, a) + } + if log == ActsLog { + entries, err := l.Store.Entries(ActsLog) + if err != nil { + return nil, 0, err + } + for _, raw := range entries { + keep(raw, handacts.LogSeat) + } + return out, len(entries), nil + } + keys, err := l.Store.Keys(log) + if err != nil { + return nil, 0, err + } + for _, k := range keys { + if at, ok := keyTime(k); ok && at.Before(since) { + continue + } + raw, found, err := l.Store.Get(log, k) + if err != nil { + return nil, len(keys), err + } + if found { + keep(raw, handacts.LogController) + } + } + return out, len(keys), nil +} + +// keyTime is the time an act's key names (`act--`), when it names one. +func keyTime(key string) (time.Time, bool) { + parts := strings.Split(key, "-") + if len(parts) != 3 || parts[0] != "act" { + return time.Time{}, false + } + ns, err := strconv.ParseInt(parts[1], 10, 64) + if err != nil { + return time.Time{}, false + } + return time.Unix(0, ns), true +} diff --git a/modules/mesh-hand-acts/cmd/mesh-hand-acts/acts_test.go b/modules/mesh-hand-acts/cmd/mesh-hand-acts/acts_test.go new file mode 100644 index 00000000..73d42cef --- /dev/null +++ b/modules/mesh-hand-acts/cmd/mesh-hand-acts/acts_test.go @@ -0,0 +1,286 @@ +package main + +import ( + "encoding/json" + "errors" + "sort" + "strings" + "sync" + "testing" + "time" +) + +type entry struct { + key string + raw []byte +} + +type memStore struct { + mu sync.Mutex + kv map[string]map[string][]byte + logs map[string][]entry + keysErr map[string]error +} + +func newStore() *memStore { + return &memStore{kv: map[string]map[string][]byte{}, logs: map[string][]entry{}, keysErr: map[string]error{}} +} + +func (s *memStore) Append(log, key string, value any) error { + raw, err := json.Marshal(value) + if err != nil { + return err + } + s.mu.Lock() + defer s.mu.Unlock() + if err := s.keysErr[log]; err != nil { + return err + } + s.logs[log] = append(s.logs[log], entry{key, raw}) + return nil +} + +func (s *memStore) Last(log, key string) ([]byte, bool, error) { + s.mu.Lock() + defer s.mu.Unlock() + for i := len(s.logs[log]) - 1; i >= 0; i-- { + if s.logs[log][i].key == key { + return s.logs[log][i].raw, true, nil + } + } + return nil, false, nil +} + +func (s *memStore) Entries(log string) ([][]byte, error) { + s.mu.Lock() + defer s.mu.Unlock() + if err := s.keysErr[log]; err != nil { + return nil, err + } + var out [][]byte + for _, e := range s.logs[log] { + out = append(out, e.raw) + } + return out, nil +} + +func (s *memStore) Fill(log string) (Fill, error) { + s.mu.Lock() + defer s.mu.Unlock() + return Fill{Entries: uint64(len(s.logs[log])), MaxBytes: 1 << 30}, nil +} + +// logKeys are a log's keys in order. +func (s *memStore) logKeys(log string) []string { + s.mu.Lock() + defer s.mu.Unlock() + var out []string + for _, e := range s.logs[log] { + out = append(out, e.key) + } + return out +} + +func (s *memStore) Get(state, key string) ([]byte, bool, error) { + s.mu.Lock() + defer s.mu.Unlock() + v, ok := s.kv[state][key] + return v, ok, nil +} + +func (s *memStore) put(state, key string, value any, create bool) error { + raw, err := json.Marshal(value) + if err != nil { + return err + } + s.mu.Lock() + defer s.mu.Unlock() + if s.kv[state] == nil { + s.kv[state] = map[string][]byte{} + } + if _, ok := s.kv[state][key]; ok && create { + return ErrExists + } + s.kv[state][key] = raw + return nil +} + +func (s *memStore) Put(state, key string, value any) error { return s.put(state, key, value, false) } +func (s *memStore) Create(state, key string, value any) error { return s.put(state, key, value, true) } + +func (s *memStore) Keys(state string) ([]string, error) { + s.mu.Lock() + defer s.mu.Unlock() + if err := s.keysErr[state]; err != nil { + return nil, err + } + var out []string + for k := range s.kv[state] { + out = append(out, k) + } + sort.Strings(out) + return out, nil +} + +func newLog() (*Log, *memStore, *[]Act, *time.Time) { + store := newStore() + var said []Act + now := time.Date(2026, 10, 29, 22, 0, 0, 0, time.UTC) + l := &Log{Store: store, Logf: func(string, ...any) {}, Now: func() time.Time { return now }, + Publish: func(subject string, body any, id string) error { + if a, ok := body.(Act); ok && subject == "mesh.seat.mesh-hand-acts.event.act-recorded" && id == a.ID { + said = append(said, a) + } + return nil + }} + return l, store, &said, &now +} + +// hand-act records what the proven caller says it did, once, and says it; it refuses what cannot be read back, a +// caller the runtime did not prove, and a drill recorded as a repair (ADR 0565 §10; issue 1953 L5-C1). +func TestHandActRecordsWhatTheProvenCallerDid(t *testing.T) { + cases := []struct { + name, by, verb, what, why, cause string + wantVerb, wantCause, err string + }{ + {"a repair", "ace.mesh-config", "", "set dns", "it was wrong", "config", VerbRecord, "config", ""}, + {"a recorder's own verb", "novox.mesh-conditions", "conditions silence", "silenced x", "known", "operator-answer", + "conditions silence", "operator-answer", ""}, + {"a drill takes its cause", "novox.mesh-healers", VerbDrill, "stopped records", "does it raise", "", VerbDrill, CauseDrill, ""}, + {"no proven caller", "unproven", "", "x", "y", "z", "", "", "proves"}, + {"by said empty", "", "", "x", "y", "z", "", "", "proves"}, + {"no why", "person.jochen", "", "x", "", "z", "", "", "says why"}, + {"no what", "person.jochen", "", "", "y", "z", "", "", "says what"}, + {"no cause", "person.jochen", "", "x", "y", "", "", "", "cause"}, + {"a drill as a repair", "person.jochen", "", "x", "y", "drill", "", "", "not recorded as a repair"}, + } + for _, tc := range cases { + l, store, said, _ := newLog() + r, err := l.HandAct(tc.by, tc.verb, tc.what, tc.why, tc.cause, "bus.x.missing") + keys := store.logKeys(ActsLog) + if tc.err != "" { + if err == nil || !strings.Contains(err.Error(), tc.err) || len(keys) != 0 || len(*said) != 0 { + t.Errorf("%s: %v, kept %v", tc.name, err, keys) + } + continue + } + if err != nil || len(keys) != 1 || keys[0] != r.Act.ID || len(*said) != 1 { + t.Fatalf("%s: %v, kept %v, said %d", tc.name, err, keys, len(*said)) + } + var kept Act + raw, _, _ := store.Last(ActsLog, keys[0]) + _ = json.Unmarshal(raw, &kept) + if kept.By != tc.by || kept.Verb != tc.wantVerb || kept.Cause != tc.wantCause || kept.Condition != "bus.x.missing" || + kept.Args[0] != tc.what || kept.Log != "" { + t.Errorf("%s: kept %+v", tc.name, kept) + } + } +} + +// warranted records the router's word once per ask, never the caller's; it refuses an ask still open, one ended +// without a choice, another asker's, and one the router holds no record of (ADR 0565 §10, ADR 0259). +func TestWarrantedRecordsTheRoutersWordOnce(t *testing.T) { + chosen := `{"state":"closed","warrant":{"ask":"a1","asker":"claude-code","outcome":"chosen","label":"Approve", + "by":{"who":"operator","kind":"telegram","identity":"42","verified":"P2"},"channel":"telegram","proofs":["P1","P2"], + "at":"2026-10-29T21:00:00Z","ask-digest":"sha256:x"}}` + cases := []struct { + name, record, asker, ask, err string + // claimed is a warrant claimed by an instance that stopped before its act reached the log. + claimed bool + }{ + {"chosen", chosen, "claude-code", "a1", "", false}, + {"claimed, its act lost: written now", chosen, "claude-code", "a1", "", true}, + {"still open", `{"state":"open"}`, "claude-code", "a1", "still open", false}, + {"no record", "", "claude-code", "a1", "no closed record", false}, + {"expired", `{"state":"closed","warrant":{"ask":"a1","asker":"claude-code","outcome":"expired","at":"2026-10-29T21:00:00Z"}}`, + "claude-code", "a1", "no person chose", false}, + {"another asker's", strings.Replace(chosen, `"asker":"claude-code"`, `"asker":"mesh-config"`, 1), "claude-code", "a1", + "record is for mesh-config", false}, + {"no digest", strings.Replace(chosen, `,"ask-digest":"sha256:x"`, "", 1), "claude-code", "a1", "no ask digest", false}, + {"not a module", chosen, "Claude Code", "a1", "not a module", false}, + } + for _, tc := range cases { + l, store, said, _ := newLog() + if tc.record != "" { + _ = store.Put(RouterRecords, "claude-code.a1", json.RawMessage(tc.record)) + } + if tc.claimed { + _ = store.Create(WarrantsState, "claude-code.a1", Warrant{Act: "warrant-claude-code-a1"}) + } + r, err := l.Warranted("ace.claude-code", tc.asker, tc.ask) + if tc.err != "" { + if keys := store.logKeys(ActsLog); err == nil || !strings.Contains(err.Error(), tc.err) || len(keys) != 0 { + t.Errorf("%s: %v, kept %v", tc.name, err, keys) + } + continue + } + if err != nil || r.Act.Verb != VerbWarrant || r.Act.Cause != CauseOperatorAnswer || + !strings.Contains(r.Act.By, "operator, as telegram identity 42") || r.Act.Via != "telegram (telegram), P2" || + !strings.Contains(r.Act.RequestedBy, "ace.claude-code") || len(r.Act.Proofs) != 2 { + t.Fatalf("%s: %+v, %v", tc.name, r.Act, err) + } + // Asked again, by another instance: nothing more is recorded. + again, err := l.Warranted("g14.claude-code", "claude-code", "a1") + keys := store.logKeys(ActsLog) + if err != nil || !strings.HasPrefix(again.Said, "already recorded") || len(keys) != 1 || len(*said) != 1 { + t.Errorf("%s: a second warranted: %q, %v, kept %v, said %d", tc.name, again.Said, err, keys, len(*said)) + } + } +} + +// hand-acts answers over both logs, newest first, narrowed by days, cause and who, and counts each cause repaired by +// hand twice in a fortnight across both, never a person's decision; a log it cannot read is said, never read as empty. +func TestHandActsReadsBothLogs(t *testing.T) { + l, store, _, now := newLog() + put := func(log, verb, cause, by string, ago time.Duration) { + at := now.Add(-ago) + a := Act{At: at, Verb: verb, Cause: cause, By: by, Why: "x"} + if log == ActsLog { + _ = store.Append(log, actKey(at), a) + return + } + _ = store.Put(log, actKey(at), a) + } + put(ActsLog, VerbRecord, "relogin", "person.jochen", time.Hour) + put(ControllerActs, VerbRecord, "relogin", "shanks.claude-code", 3*24*time.Hour) + put(ControllerActs, VerbDrill, CauseDrill, "person.jochen", time.Hour) + put(ActsLog, VerbDrill, CauseDrill, "person.jochen", 2*time.Hour) + put(ActsLog, VerbWarrant, CauseOperatorAnswer, "the operator", time.Hour) + put(ActsLog, VerbWarrant, CauseOperatorAnswer, "the operator", 2*time.Hour) + put(ControllerActs, VerbRecord, "config", "ace.mesh-config", 30*24*time.Hour) + cases := []struct { + name string + days int + cause, who string + want int + }{ + {"a fortnight", 14, "", "", 6}, + {"a day", 1, "", "", 5}, + {"forty days", 40, "", "", 7}, + {"by cause", 14, "relogin", "", 2}, + {"by who", 14, "", "claude-code", 1}, + } + for _, tc := range cases { + got := l.HandActs(tc.days, tc.cause, tc.who) + if len(got.Acts) != tc.want || len(got.Repeated) != 1 || got.Repeated["relogin"] != 2 || got.Unread != nil { + t.Errorf("%s: %d acts, repeated %v, unread %v", tc.name, len(got.Acts), got.Repeated, got.Unread) + } + for i := 1; i < len(got.Acts); i++ { + if got.Acts[i].At.After(got.Acts[i-1].At) { + t.Errorf("%s: not newest first", tc.name) + } + } + } + if got := l.HandActs(14, "", ""); got.Acts[0].Log != Seat || got.Kept[ControllerActs] != 3 || got.Fill == nil || + got.Fill.Entries != 4 { + t.Errorf("the log of an act or the kept counts are not said: %+v", got.Kept) + } + store.keysErr[ControllerActs] = errors.New("the bus is away") + if got := l.HandActs(14, "", ""); got.Unread[ControllerActs] == "" || len(got.Acts) != 4 { + t.Errorf("an unreadable log: %+v", got.Unread) + } + w, err := l.Warrants() + if err != nil || w.(map[string]any)["count"] != 0 { + t.Errorf("warrants answered %v, %v", w, err) + } +} diff --git a/modules/mesh-hand-acts/cmd/mesh-hand-acts/main.go b/modules/mesh-hand-acts/cmd/mesh-hand-acts/main.go new file mode 100644 index 00000000..b103fa23 --- /dev/null +++ b/modules/mesh-hand-acts/cmd/mesh-hand-acts/main.go @@ -0,0 +1,84 @@ +// mesh-hand-acts: the holder of the seat mesh-hand-acts (novox/hq ADR 0565 §10). A Go bundle served by a runtime of +// its own, on its own bus account (`runs-as`), so that its account alone writes the hand-act log (`acts`, its module +// log: a stream that keeps every act in order) and `warrants`. It reads the router's record of each ask for a warrant and the controller's log for +// `hand-acts`. stdout is the MCP channel; what this module says, it says on stderr. +package main + +import ( + "errors" + "fmt" + "os" + "time" + + stdio "git.novox.be/novox/mesh-sdk/go" +) + +func logf(format string, a ...any) { fmt.Fprintf(os.Stderr, format+"\n", a...) } + +// runtimeStore is the bus, through the runtime that launched this bundle. +type runtimeStore struct{} + +func (runtimeStore) Get(state, key string) ([]byte, bool, error) { + e, err := stdio.State(state).Get(key) + if err != nil || e == nil { + return nil, false, err + } + return e.Value, true, nil +} + +func (runtimeStore) Create(state, key string, value any) error { + _, err := stdio.State(state).Create(key, value) + if errors.Is(err, stdio.ErrChanged) { + return ErrExists + } + return err +} + +func (runtimeStore) Keys(state string) ([]string, error) { return stdio.State(state).Keys() } + +func (runtimeStore) Append(log, key string, value any) error { + _, err := stdio.Log(log).Append(key, value) + return err +} + +func (runtimeStore) Last(log, key string) ([]byte, bool, error) { + e, err := stdio.Log(log).Last(key) + if err != nil || e == nil { + return nil, false, err + } + return e.Value, true, nil +} + +func (runtimeStore) Entries(log string) ([][]byte, error) { + var out [][]byte + for from := uint64(0); ; { + page, err := stdio.Log(log).Read(from, "", 1000) + if err != nil { + return nil, err + } + for _, e := range page.Entries { + out = append(out, e.Value) + } + if page.Next == 0 { + return out, nil + } + from = page.Next + } +} + +func (runtimeStore) Fill(log string) (Fill, error) { + i, err := stdio.Log(log).Info() + if err != nil { + return Fill{}, err + } + return Fill{Entries: i.Entries, Bytes: i.Bytes, MaxBytes: i.MaxBytes}, nil +} + +func main() { + l := &Log{Store: runtimeStore{}, Now: time.Now, Logf: logf, + Publish: stdio.SeatPublish} + if err := stdio.Serve("", Tools(l)); err != nil { + logf("%v", err) + os.Exit(1) + } +} diff --git a/modules/mesh-hand-acts/cmd/mesh-hand-acts/manifest_test.go b/modules/mesh-hand-acts/cmd/mesh-hand-acts/manifest_test.go new file mode 100644 index 00000000..cd19669a --- /dev/null +++ b/modules/mesh-hand-acts/cmd/mesh-hand-acts/manifest_test.go @@ -0,0 +1,76 @@ +package main + +import ( + "encoding/json" + "os" + "path/filepath" + "slices" + "testing" +) + +// The manifest says what the code does: it claims the seat mesh-hand-acts once for the mesh with its verbs (ADR 0565 +// §1, §10), runs on its own account, says act-recorded, reads the controller's log and the router's records, keeps its +// acts as its log (a stream, §10) and warrants as a bucket that never ages, so a warrant is recorded once. Every state +// and log it keeps and reads has a tool that reads it (ADR 0565 §15). +func TestTheManifestSaysWhatTheCodeDoes(t *testing.T) { + raw, err := os.ReadFile(filepath.Join("..", "..", "module.json")) + if err != nil { + t.Fatal(err) + } + var m struct { + Module string `json:"module"` + Runs string `json:"runs-as"` + Claims []struct { + Name string `json:"name"` + Scope string `json:"scope"` + Serves []string `json:"serves"` + } `json:"claims"` + Emits []string `json:"emits"` + Reads []string `json:"reads"` + Logs []struct { + Name string `json:"name"` + MaxAgeDays int `json:"max-age-days"` + } `json:"logs"` + State []json.RawMessage `json:"state"` + } + if err := json.Unmarshal(raw, &m); err != nil { + t.Fatal(err) + } + if m.Module != Module || m.Runs != Module { + t.Errorf("module %q runs as %q", m.Module, m.Runs) + } + if len(m.Claims) != 1 || m.Claims[0].Name != Seat || m.Claims[0].Scope != "mesh" || !slices.Equal(m.Claims[0].Serves, Verbs) { + t.Errorf("claims %+v", m.Claims) + } + // act-recorded is the seat's event, said on the seat's subject: a module lists no event of a seat it claims. + if len(m.Emits) != 0 || !slices.Equal(m.Reads, []string{ControllerActs, RouterRecords}) { + t.Errorf("emits %v, reads %v", m.Emits, m.Reads) + } + readBy := map[string]string{ActsLog: HandActsTool, WarrantsState: HandActsTool, ControllerActs: HandActsTool, + RouterRecords: WarrantedTool} + tools := map[string]bool{} + for _, tool := range Tools(&Log{}) { + tools[tool.Name] = true + } + // The acts are a log kept 400 days (ADR 0565 §10). + if len(m.Logs) != 1 || m.Logs[0].Name != ActsLog || m.Logs[0].MaxAgeDays != 400 { + t.Errorf("logs %+v, want the acts as a log kept 400 days", m.Logs) + } + for _, raw := range m.State { + var name string + if json.Unmarshal(raw, &name) != nil || name != WarrantsState { + t.Errorf("the state %s is not the warrants, or ages", raw) + } + if !tools[readBy[name]] { + t.Errorf("the state %s has no tool that reads it", name) + } + } + if !tools[readBy[ActsLog]] { + t.Errorf("the log %s has no tool that reads it", ActsLog) + } + for _, r := range m.Reads { + if !tools[readBy[r]] { + t.Errorf("the read %s has no tool that answers it", r) + } + } +} diff --git a/modules/mesh-hand-acts/cmd/mesh-hand-acts/reads-outside b/modules/mesh-hand-acts/cmd/mesh-hand-acts/reads-outside new file mode 100644 index 00000000..60d6785c --- /dev/null +++ b/modules/mesh-hand-acts/cmd/mesh-hand-acts/reads-outside @@ -0,0 +1,2 @@ +# What these tests read outside their package, as paths from the repository's root (issue 1936). +modules/mesh-hand-acts/module.json diff --git a/modules/mesh-hand-acts/cmd/mesh-hand-acts/tools.go b/modules/mesh-hand-acts/cmd/mesh-hand-acts/tools.go new file mode 100644 index 00000000..833db625 --- /dev/null +++ b/modules/mesh-hand-acts/cmd/mesh-hand-acts/tools.go @@ -0,0 +1,110 @@ +package main + +// The seat's verbs (novox/hq ADR 0565 §10, §15): hand-act and warranted record, hand-acts reads both logs and, with +// warrants, every warrant recorded. A recovery is a call: a warrant whose act was lost is written again by asking +// warranted again, from the router's record. + +import ( + "encoding/json" + "fmt" + "sort" + "strings" + + stdio "git.novox.be/novox/mesh-sdk/go" +) + +// Verbs are the seat's verbs, as the manifest claims them. +var Verbs = []string{"hand-act", "hand-acts", "warranted"} + +// Tool names. +const ( + HandActTool = Seat + ".hand-act" + HandActsTool = Seat + ".hand-acts" + WarrantedTool = Seat + ".warranted" +) + +func obj(props map[string]any, required ...string) map[string]any { + m := map[string]any{"type": "object", "properties": props} + if len(required) > 0 { + m["required"] = required + } + return m +} + +func str(d string) map[string]any { return map[string]any{"type": "string", "description": d} } + +// Tools are what the runtime serves. +func Tools(l *Log) []stdio.Tool { + return []stdio.Tool{ + {Name: HandActTool, + Description: "Record one act done by hand (novox/hq ADR 0565 §10): what was done, why, its cause in a word or a " + + "condition's kind (the word a second act for the same reason uses, which is how a repair done twice is " + + "found) and the condition it addresses. Who did it is the caller the runtime proves. verb is the act's " + + "verb as the log names it: a repair when not given, `hand-act drill` for a drill, or a recorder's own " + + "(`conditions silence`, `config set`). Said as act-recorded.", + Input: obj(map[string]any{"what": str("what was done"), "why": str("why, in a sentence"), + "cause": str("the cause, in a word or a condition's kind"), "condition": str("the condition's key, if any"), + "verb": str("the act's verb in the log; a repair when not given")}, "what", "why"), + Run: func(a map[string]any) (any, error) { + return l.HandAct(stdio.CallerOf(a), arg(a, "verb"), arg(a, "what"), arg(a, "why"), arg(a, "cause"), arg(a, "condition")) + }}, + {Name: HandActsTool, + Description: "What was done by hand lately, over both logs (this seat's and the controller's until its lease " + + "ends), newest first: each act with who, why, its cause and which log holds it, and the causes repaired by " + + "hand twice or more within a fortnight (a healer is wanted). Narrowed by cause and who; warrants=true lists " + + "every warrant recorded instead.", + Input: obj(map[string]any{"days": map[string]any{"type": "number", "description": "how many days back (default 14)"}, + "cause": str("only this cause"), "who": str("only acts whose who contains this"), + "warrants": str("\"true\": every warrant recorded, `.` with its act")}), + Run: func(a map[string]any) (any, error) { + if strings.EqualFold(arg(a, "warrants"), "true") || a["warrants"] == true { + return l.Warrants() + } + return l.HandActs(days(a), arg(a, "cause"), arg(a, "who")), nil + }}, + {Name: WarrantedTool, + Description: "Record an asker's act on the operator's warrant (ADR 0274, ADR 0259): read from the router's own " + + "record of that ask — who chose, through which channel, with which proofs and which answer — never from the " + + "caller, once per ask; refused for an ask still open, ended without a choice, or another asker's.", + Input: obj(map[string]any{"asker": str("the module that asked"), "ask": str("its ask's id")}, "asker", "ask"), + Run: func(a map[string]any) (any, error) { + return l.Warranted(stdio.CallerOf(a), arg(a, "asker"), arg(a, "ask")) + }}, + } +} + +// Warrants is every warrant recorded, by key. +func (l *Log) Warrants() (any, error) { + keys, err := l.Store.Keys(WarrantsState) + if err != nil { + return nil, fmt.Errorf("the warrants cannot be read: %w", err) + } + sort.Strings(keys) + rows := map[string]json.RawMessage{} + for _, k := range keys { + if raw, found, err := l.Store.Get(WarrantsState, k); err == nil && found { + rows[k] = raw + } + } + return map[string]any{"warrants": rows, "count": len(rows)}, nil +} + +func arg(a map[string]any, k string) string { + s, _ := a[k].(string) + return strings.TrimSpace(s) +} + +func days(a map[string]any) int { + switch v := a["days"].(type) { + case float64: + if v > 0 { + return int(v) + } + case string: + var n int + if _, err := fmt.Sscan(v, &n); err == nil && n > 0 { + return n + } + } + return 14 +} diff --git a/modules/mesh-hand-acts/cmd/mesh-hand-acts/whole_integration_test.go b/modules/mesh-hand-acts/cmd/mesh-hand-acts/whole_integration_test.go new file mode 100644 index 00000000..64ec46d2 --- /dev/null +++ b/modules/mesh-hand-acts/cmd/mesh-hand-acts/whole_integration_test.go @@ -0,0 +1,405 @@ +//go:build integration + +package main + +// mesh-hand-acts whole against a real bus (novox/hq ADR 0565 §10; issue 1953 L5-C2): the built binary, played over its +// real stdin and stdout as the node's tool runtime plays it (tools/call with the caller in _meta; mesh/state.* answered +// from a real nats-server's key-value buckets, mesh/log.* from the log's real stream, mesh/publish onto its subjects). A recorder's hand act is kept in the +// log on the bus under the proven caller and heard as act-recorded; a warrant the router recorded is recorded once. + +import ( + "bufio" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" + + "git.novox.be/novox/mesh-sdk/go/testbus" +) + +type wire struct { + ID json.RawMessage `json:"id,omitempty"` + Method string `json:"method,omitempty"` + Params json.RawMessage `json:"params,omitempty"` + Result json.RawMessage `json:"result,omitempty"` + Error *struct { + Message string `json:"message"` + } `json:"error,omitempty"` +} + +// player is the node's tool runtime, as far as the self-check needs it, over a real bus. +type player struct { + ctx context.Context + nc *nats.Conn + js jetstream.JetStream + mu sync.Mutex + in io.Writer + next int + pending map[string]chan wire +} + +func (p *player) send(m map[string]any) { + raw, _ := json.Marshal(m) + p.mu.Lock() + defer p.mu.Unlock() + _, _ = p.in.Write(append(raw, '\n')) +} + +func (p *player) answer(id json.RawMessage, result any, err error) { + if err != nil { + p.send(map[string]any{"id": id, "error": map[string]any{"code": -32000, "message": err.Error()}}) + return + } + p.send(map[string]any{"id": id, "result": result}) +} + +func (p *player) serve(out io.Reader) { + scanner := bufio.NewScanner(out) + scanner.Buffer(make([]byte, 1<<20), 1<<24) + for scanner.Scan() { + var m wire + if json.Unmarshal(scanner.Bytes(), &m) != nil { + continue + } + if m.Method == "" { + var key string + _ = json.Unmarshal(m.ID, &key) + p.mu.Lock() + ch := p.pending[key] + delete(p.pending, key) + p.mu.Unlock() + if ch != nil { + ch <- m + } + continue + } + go p.handle(m) + } +} + +// kv is the bucket a state names: this module's own by its local name, another's as `.`. +func (p *player) kv(state string) (jetstream.KeyValue, error) { + if owner, name, ok := strings.Cut(state, "."); ok { + return p.js.KeyValue(p.ctx, owner+"_"+name) + } + return p.js.KeyValue(p.ctx, Module+"_"+state) +} + +func (p *player) handle(m wire) { + var a struct { + State string `json:"state"` + Key string `json:"key"` + Value json.RawMessage `json:"value"` + Body json.RawMessage `json:"body"` + } + _ = json.Unmarshal(m.Params, &a) + switch m.Method { + case "mesh/state.get": + kv, err := p.kv(a.State) + if err != nil { + p.answer(m.ID, nil, err) + return + } + e, err := kv.Get(p.ctx, a.Key) + if errors.Is(err, jetstream.ErrKeyNotFound) { + p.answer(m.ID, nil, nil) + return + } + if err != nil { + p.answer(m.ID, nil, err) + return + } + p.answer(m.ID, map[string]any{"key": e.Key(), "value": json.RawMessage(e.Value()), "revision": e.Revision()}, nil) + case "mesh/state.put": + kv, err := p.kv(a.State) + if err == nil { + var rev uint64 + if rev, err = kv.Put(p.ctx, a.Key, a.Value); err == nil { + p.answer(m.ID, map[string]any{"revision": rev}, nil) + return + } + } + p.answer(m.ID, nil, err) + case "mesh/state.create": + kv, err := p.kv(a.State) + if err == nil { + var rev uint64 + if rev, err = kv.Create(p.ctx, a.Key, a.Value); err == nil { + p.answer(m.ID, map[string]any{"revision": rev}, nil) + return + } + if errors.Is(err, jetstream.ErrKeyExists) { + p.send(map[string]any{"id": m.ID, "error": map[string]any{"code": -32001, "message": "the key was written by another writer first"}}) + return + } + } + p.answer(m.ID, nil, err) + case "mesh/state.watch": + kv, err := p.kv(a.State) + if err != nil { + p.answer(m.ID, nil, err) + return + } + key := a.Key + if key == "" { + key = ">" + } + w, err := kv.Watch(p.ctx, key) + if err != nil { + p.answer(m.ID, nil, err) + return + } + var watch string + _ = json.Unmarshal(m.ID, &watch) + current := true + for e := range w.Updates() { + if e == nil { + current = false + p.answer(m.ID, map[string]any{}, nil) + continue + } + op := "put" + if e.Operation() != jetstream.KeyValuePut { + op = "delete" + } + change := map[string]any{"state": a.State, "key": e.Key(), "op": op, "revision": e.Revision(), "current": current} + if op == "put" { + change["value"] = json.RawMessage(e.Value()) + } + p.mu.Lock() + p.next++ + id := fmt.Sprintf("w-%d", p.next) + p.mu.Unlock() + p.send(map[string]any{"id": id, "method": "mesh/state", "params": map[string]any{"watch": watch, "change": change}}) + } + case "mesh/subscribe": + p.answer(m.ID, map[string]any{}, nil) + case "mesh/log.append", "mesh/log.read", "mesh/log.last", "mesh/log.info": + p.log(m) + case "mesh/state.keys": + kv, err := p.kv(a.State) + if err != nil { + p.answer(m.ID, nil, err) + return + } + keys, err := kv.Keys(p.ctx) + if errors.Is(err, jetstream.ErrNoKeysFound) { + keys, err = []string{}, nil + } + p.answer(m.ID, keys, err) + case "mesh/state.delete": + kv, err := p.kv(a.State) + if err == nil { + err = kv.Delete(p.ctx, a.Key) + } + p.answer(m.ID, map[string]any{}, err) + case "mesh/seat.publish": + var e struct { + Subject string `json:"subject"` + Body json.RawMessage `json:"body"` + } + _ = json.Unmarshal(m.Params, &e) + p.answer(m.ID, map[string]any{}, p.nc.Publish(e.Subject, e.Body)) + case "mesh/publish": + var e struct { + Key string `json:"key"` + Body json.RawMessage `json:"body"` + } + _ = json.Unmarshal(m.Params, &e) + p.answer(m.ID, map[string]any{}, p.nc.Publish("mesh.mod."+Module+".event."+e.Key, e.Body)) + default: + p.answer(m.ID, nil, errors.New("nats: no responders available for request")) + } +} + +// log answers the runtime's log verbs from the log's real stream, LOG__ on mesh.log.... +func (p *player) log(m wire) { + var a struct { + Log string `json:"log"` + Key string `json:"key"` + Value json.RawMessage `json:"value"` + From uint64 `json:"from"` + } + _ = json.Unmarshal(m.Params, &a) + stream := "LOG_" + Module + "_" + a.Log + subject := "mesh.log." + Module + "." + a.Log + s, err := p.js.Stream(p.ctx, stream) + if err != nil { + p.answer(m.ID, nil, err) + return + } + entry := func(r *jetstream.RawStreamMsg) map[string]any { + return map[string]any{"sequence": r.Sequence, "key": strings.TrimPrefix(r.Subject, subject+"."), + "value": json.RawMessage(r.Data), "at": r.Time} + } + switch m.Method { + case "mesh/log.append": + ack, err := p.js.Publish(p.ctx, subject+"."+a.Key, a.Value) + if err != nil { + p.answer(m.ID, nil, err) + return + } + p.answer(m.ID, map[string]any{"sequence": ack.Sequence}, nil) + case "mesh/log.last": + r, err := s.GetLastMsgForSubject(p.ctx, subject+"."+a.Key) + if errors.Is(err, jetstream.ErrMsgNotFound) { + p.answer(m.ID, nil, nil) + return + } + if err != nil { + p.answer(m.ID, nil, err) + return + } + p.answer(m.ID, entry(r), nil) + case "mesh/log.read": + info, err := s.Info(p.ctx) + if err != nil { + p.answer(m.ID, nil, err) + return + } + entries := []map[string]any{} + for seq := max(a.From, 1); seq <= info.State.LastSeq; seq++ { + if r, err := s.GetMsg(p.ctx, seq); err == nil { + entries = append(entries, entry(r)) + } + } + p.answer(m.ID, map[string]any{"entries": entries, "next": 0, "last": info.State.LastSeq}, nil) + case "mesh/log.info": + info, err := s.Info(p.ctx) + if err != nil { + p.answer(m.ID, nil, err) + return + } + p.answer(m.ID, map[string]any{"entries": info.State.Msgs, "bytes": info.State.Bytes, + "max_bytes": info.Config.MaxBytes, "first": info.State.FirstSeq, "last": info.State.LastSeq}, nil) + } +} + +func (p *player) call(name, caller string, args map[string]any) (json.RawMessage, error) { + p.mu.Lock() + p.next++ + id := fmt.Sprintf("c-%d", p.next) + ch := make(chan wire, 1) + p.pending[id] = ch + p.mu.Unlock() + p.send(map[string]any{"id": id, "method": "tools/call", "params": map[string]any{"name": name, "arguments": args, + "_meta": map[string]any{"mesh/caller": caller}}}) + select { + case m := <-ch: + if m.Error != nil { + return nil, errors.New(m.Error.Message) + } + var r struct { + Content []struct { + Text string `json:"text"` + } `json:"content"` + IsError bool `json:"isError"` + } + if err := json.Unmarshal(m.Result, &r); err != nil || len(r.Content) != 1 { + return nil, fmt.Errorf("%s answered %s", name, m.Result) + } + if r.IsError { + return nil, errors.New(r.Content[0].Text) + } + return json.RawMessage(r.Content[0].Text), nil + case <-time.After(60 * time.Second): + return nil, fmt.Errorf("%s did not answer in 60s", name) + } +} + +// 1953/L5-C2: an act recorded through the seat is kept on the bus and heard as act-recorded. +func TestTheHandActLogWholeRecordsOnTheBus(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + dir := t.TempDir() + bin := filepath.Join(dir, Module) + build := exec.Command("go", "build", "-o", bin, ".") + build.Stderr = os.Stderr + if err := build.Run(); err != nil { + t.Fatalf("mesh-hand-acts does not build: %v", err) + } + nc, err := nats.Connect(testbus.URL(t)) + if err != nil { + t.Fatal(err) + } + t.Cleanup(nc.Close) + js, err := jetstream.New(nc) + if err != nil { + t.Fatal(err) + } + for _, b := range []string{Module + "_" + WarrantsState, "mesh-controller_hand-acts", "messenger_asks"} { + if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: b}); err != nil { + t.Fatal(err) + } + } + // The log as the mesh makes it (ADR 0297): a stream kept by limits, refusing when full, never deleting. + logStream, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: "LOG_" + Module + "_" + ActsLog, + Subjects: []string{"mesh.log." + Module + "." + ActsLog + ".>"}, MaxBytes: 64 << 20, Discard: jetstream.DiscardNew, + DenyDelete: true, DenyPurge: true, AllowDirect: true}) + if err != nil { + t.Fatal(err) + } + recorded, err := nc.SubscribeSync("mesh.seat." + Seat + ".event." + RecordedEvent) + if err != nil { + t.Fatal(err) + } + cmd := exec.CommandContext(ctx, bin) + cmd.Env = append(os.Environ(), "MESH_NODE=novox", "MESH_MODULE="+Module) + stdin, _ := cmd.StdinPipe() + stdout, _ := cmd.StdoutPipe() + cmd.Stderr = os.Stderr + if err := cmd.Start(); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = stdin.Close(); cancel(); _ = cmd.Wait() }) + p := &player{ctx: ctx, nc: nc, js: js, in: stdin, pending: map[string]chan wire{}} + go p.serve(stdout) + + raw, err := p.call(HandActTool, "ace.mesh-config", map[string]any{"what": "set dns on g14", "why": "it was wrong", + "cause": "config", "by": "someone else"}) + if err != nil { + t.Fatal(err) + } + var r Recorded + if err := json.Unmarshal(raw, &r); err != nil || r.Act.By != "ace.mesh-config" { + t.Fatalf("hand-act answered %s", raw) + } + e, err := logStream.GetLastMsgForSubject(ctx, "mesh.log."+Module+"."+ActsLog+"."+r.Act.ID) + if err != nil || !strings.Contains(string(e.Data), `"by":"ace.mesh-config"`) { + t.Fatalf("the act is not kept in the log on the bus: %v", err) + } + msg, err := recorded.NextMsg(10 * time.Second) + if err != nil || !strings.Contains(string(msg.Data), r.Act.ID) { + t.Fatalf("act-recorded was not said: %v", err) + } + // A warrant the router recorded, asked twice: one act. + asks, _ := js.KeyValue(ctx, "messenger_asks") + if _, err := asks.Put(ctx, "claude-code.a1", []byte(`{"state":"closed","warrant":{"ask":"a1","asker":"claude-code", + "outcome":"chosen","label":"Approve","by":{"who":"operator","kind":"telegram","identity":"42"},"channel":"telegram", + "at":"2026-10-29T21:00:00Z","ask-digest":"sha256:x"}}`)); err != nil { + t.Fatal(err) + } + for _, caller := range []string{"ace.claude-code", "g14.claude-code"} { + if _, err := p.call(WarrantedTool, caller, map[string]any{"asker": "claude-code", "ask": "a1"}); err != nil { + t.Fatal(err) + } + } + if info, err := logStream.Info(ctx); err != nil || info.State.Msgs != 2 { + t.Errorf("the log keeps %v, want the act and one warrant (%v)", info, err) + } + listed, err := p.call(HandActsTool, "person.jochen", map[string]any{}) + if err != nil || !strings.Contains(string(listed), "warrant-claude-code-a1") || !strings.Contains(string(listed), r.Act.ID) { + t.Errorf("hand-acts answered %s, %v", listed, err) + } +} diff --git a/modules/mesh-hand-acts/go.mod b/modules/mesh-hand-acts/go.mod new file mode 100644 index 00000000..412f5438 --- /dev/null +++ b/modules/mesh-hand-acts/go.mod @@ -0,0 +1,22 @@ +module mesh-hand-acts + +go 1.26.0 + +require ( + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 + github.com/nats-io/nats.go v1.54.0 +) + +require ( + github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op // indirect + github.com/google/go-tpm v0.9.8 // indirect + github.com/klauspost/compress v1.20.0 // indirect + github.com/minio/highwayhash v1.0.4 // indirect + github.com/nats-io/jwt/v2 v2.8.1 // indirect + github.com/nats-io/nats-server/v2 v2.11.17 // indirect + github.com/nats-io/nkeys v0.4.16 // indirect + github.com/nats-io/nuid v1.0.1 // indirect + golang.org/x/crypto v0.57.0 // indirect + golang.org/x/sys v0.48.0 // indirect + golang.org/x/time v0.15.0 // indirect +) diff --git a/modules/mesh-hand-acts/go.sum b/modules/mesh-hand-acts/go.sum new file mode 100644 index 00000000..a2b8e018 --- /dev/null +++ b/modules/mesh-hand-acts/go.sum @@ -0,0 +1,27 @@ +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 h1:lKU3HjNXGiVbLW2hUgFjU/TMTUYMlNnnmZxG5T7biQA= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= +github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= +github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= +github.com/google/go-tpm v0.9.8/go.mod h1:h9jEsEECg7gtLis0upRBQU+GhYVH6jMjrFxI8u6bVUY= +github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA= +github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI= +github.com/minio/highwayhash v1.0.4 h1:asJizugGgchQod2ja9NJlGOWq4s7KsAWr5XUc9Clgl4= +github.com/minio/highwayhash v1.0.4/go.mod h1:GGYsuwP/fPD6Y9hMiXuapVvlIUEhFhMTh0rxU3ik1LQ= +github.com/nats-io/jwt/v2 v2.8.1 h1:V0xpGuD/N8Mi+fQNDynXohVvp7ZztevW5io8CUWlPmU= +github.com/nats-io/jwt/v2 v2.8.1/go.mod h1:nWnOEEiVMiKHQpnAy4eXlizVEtSfzacZ1Q43LIRavZg= +github.com/nats-io/nats-server/v2 v2.11.17 h1:GKEghcFK6A+aFx11Yf1LjgLC3txAwvyhnYzhBIQZA8I= +github.com/nats-io/nats-server/v2 v2.11.17/go.mod h1:B1sFVz4StNosQ903ak4N1G01Fl/9f8e06mXpFIE2K24= +github.com/nats-io/nats.go v1.54.0 h1:vsXoOxjHp/GmPUN+EcI7uOf/uB+iAP+kEsAFNQN0yzA= +github.com/nats-io/nats.go v1.54.0/go.mod h1:y+DZoD1oBOYfZTU681eTUiUjI0vbqYGixNVFHcjHJ0k= +github.com/nats-io/nkeys v0.4.16 h1:rd5oAuLOb8mnAycB0xleuEBNS1pVVnN0fv/FF34Eypg= +github.com/nats-io/nkeys v0.4.16/go.mod h1:llLgWoI0o4z/Q57q2R1kHfmocyhGV6VG/U18Glg1Afs= +github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= +github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= +golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M= +golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA= +golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo= +golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= +golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= +golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= diff --git a/modules/mesh-hand-acts/module.json b/modules/mesh-hand-acts/module.json new file mode 100644 index 00000000..96aff8b1 --- /dev/null +++ b/modules/mesh-hand-acts/module.json @@ -0,0 +1,65 @@ +{ + "module": "mesh-hand-acts", + "version": "1", + "runs-as": "mesh-hand-acts", + "claims": [ + { + "name": "mesh-hand-acts", + "scope": "mesh", + "serves": [ + "hand-act", + "hand-acts", + "warranted" + ] + } + ], + "reads": [ + "mesh-controller.hand-acts", + "messenger.asks" + ], + "logs": [ + { + "name": "acts", + "max-age-days": 400 + } + ], + "state": [ + "warrants" + ], + "own-secrets": { + "broker": "${dir:state}/broker" + }, + "secrets-owner": "mesh-hand-acts", + "resources": [ + { + "id": "account", + "type": "user", + "name": "mesh-hand-acts", + "shell": "/usr/bin/nologin", + "home": "/var/lib/mesh-hand-acts" + }, + { + "id": "state", + "type": "directory", + "mode": "0700", + "owner": "mesh-hand-acts", + "place": ".", + "unknown": "program" + } + ], + "build": { + "artifacts": [ + { + "name": "tools", + "kind": "bundle", + "language": "go", + "system": "arch", + "from": "cmd/mesh-hand-acts", + "binary": "mesh-hand-acts", + "loads": [ + "mesh-hand-acts" + ] + } + ] + } +} diff --git a/modules/mesh-healers/.gitignore b/modules/mesh-healers/.gitignore new file mode 100644 index 00000000..0bafa572 --- /dev/null +++ b/modules/mesh-healers/.gitignore @@ -0,0 +1,2 @@ +/mesh-healers +/cmd/mesh-healers/mesh-healers diff --git a/modules/mesh-healers/README.md b/modules/mesh-healers/README.md new file mode 100644 index 00000000..d00cdb1b --- /dev/null +++ b/modules/mesh-healers/README.md @@ -0,0 +1,46 @@ +# mesh-healers + +Holds the seat `mesh-healers` (novox/hq ADR 0564 §7, §8): the mesh's healers, moved out of the controller. Go, one +binary (`cmd/mesh-healers`), served by a runtime of its own on its own bus account (`runs-as`). + +## What it does + +It hears the condition keeper's `condition-raised` and `condition-changed` (seat `mesh-conditions`) on a durable +consumer. A condition whose kind a healer answers is acted on within the healer's budget, its settle and the mesh-wide +brake (twelve acts an hour). A repair is a call of the seat or tool that owns the act: H3 asks the grants part's +`grants_assert` for the module whose consumer is gone. H4 (the controller's own consumer) and H5 (the provider's own +repair) are registered and run elsewhere. + +Each act is begun by a create of its key in `attempts` before it is made and finished after, so a holder that dies +mid-act has spent it and a second instance cannot begin it again. The condition keeper writes the condition's +`tried`, resolver and escalation from `attempts`; a healer writes no condition. + +**Only once the controller's lease names no controller.** Until the write that hands the lease away, each act it would +take is written to `would`, never to `attempts`: it counts for no budget, settle or brake and reaches no condition. + +Each key of `attempts` and `would` lives a week from its last write, its own time to live (the buckets are declared +`key-ttl`): nothing else deletes `would`, and once the lease is handed away nothing is written there. The healers' waits +are keys of `wakes` whose time to live is the wait: `settle.` ends with a settle and judges the condition +again, `fortnight` ends when S15's oldest counted act leaves the fortnight and counts again. The watch of `wakes` hears +each expiry; no timer runs in the process. A wait still running is on the bus across a restart; one that expired while +no instance watched is judged at the start, once the lease is first read. + +S15: it hears the seat mesh-hand-acts' `act-recorded` and watches the controller's hand-act log and writes `healer-wanted` for a cause repaired by hand twice within a +fortnight; a drill and a person's decision by design never count. + +## What it keeps + +- `mesh-healers_attempts`: every act and escalation, each key kept a week. +- `mesh-healers_would`: the newest act it would take per healer and condition while the controller holds the lease, + each key kept a week. +- `mesh-healers_findings`: the brake, its own blindness, and S15. +- `mesh-healers_wakes`: the waits, each key living as long as it waits. + +## Its verbs and tools + +- `mesh-healers.healers`: the registry, the acts of the last days, the would-acts, the brake, whether it acts and why; + `recount=true` counts the hand-act log again. +- `mesh-healers.heal`: one act now on an open condition, by key and why. +- `mesh-healers.drill`: records a drill in the hand-act log of the seat `mesh-hand-acts` as `hand-act drill` (the + controller's `drill` while that seat has no holder and the controller holds its lease). S15 never counts a drill. +- `healers_findings`, `healers_would`, `healers_wakes`: the reads of its buckets. diff --git a/modules/mesh-healers/cmd/mesh-healers/healers.go b/modules/mesh-healers/cmd/mesh-healers/healers.go new file mode 100644 index 00000000..425eba47 --- /dev/null +++ b/modules/mesh-healers/cmd/mesh-healers/healers.go @@ -0,0 +1,663 @@ +package main + +// The healers (novox/hq ADR 0564 §7, §8; ADR 0231): a known failure heals itself, under a brake, and every repair is +// said. The registry is compiled in; mesh-healers hears `condition-raised` and `condition-changed` of the seat +// mesh-conditions on a durable consumer, instead of looking every half minute. A repair is a call of the seat or tool +// that owns the act (H3: the grants part's grants_assert). Each act is begun by a create of its key in the bucket +// `attempts` before it is made and finished after, so a holder that dies mid-act has spent it and a second instance +// cannot begin the same act; budgets, settles and the mesh-wide brake count from there. Success is never the +// healer's to say: the condition clears when its observer no longer sees it, and the condition keeper writes `tried`, +// the resolver and the escalation into the condition from `attempts`. +// +// **Only once the controller's lease names no controller** (ADR 0564 §8): until then each act it would take is +// written to the bucket `would`, never to `attempts`, so a would-act counts for no budget, settle or brake, reaches +// no `tried` and escalates nothing. `would` keeps each key a week (its bucket's age): nothing else deletes it, and once +// the lease is handed away nothing is written there again. + +import ( + "encoding/json" + "errors" + "fmt" + "slices" + "sort" + "strconv" + "strings" + "sync" + "time" + + stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/buscompose" + "git.novox.be/novox/mesh-sdk/go/handacts" + "git.novox.be/novox/mesh-sdk/go/health" +) + +// Module and Seat are this module and the seat it holds once for the mesh. +const ( + Module = "mesh-healers" + Seat = "mesh-healers" +) + +// The states it keeps and reads, and the event it says. +const ( + AttemptsState = health.AttemptsState + WouldState = health.WouldState + FindingsState = health.FindingsState + LeaseRead = buscompose.ControllerLease + LeaseKey = buscompose.ControllerLeaseKey + ConditionsRead = "mesh-conditions.conditions" + HandActsRead = "mesh-controller.hand-acts" + ActedEvent = health.EventHealerActed + // ActedSubject is where the seat says it, which its consumers hear (catalogue.SeatSays). + ActedSubject = "mesh.seat." + Seat + ".event." + ActedEvent + // RecordedEvent is the hand-act log's seat event, each act recorded there (ADR 0565 §11). + RecordedEvent = handacts.Seat + "." + handacts.EventRecorded + // The events it hears: the keeper's, on the seat mesh-conditions. + RaisedEvent = "mesh-conditions.condition-raised" + ChangedEvent = "mesh-conditions.condition-changed" + // AssertTool is the grants part's tool that asserts a module's buckets, logs and streams again. + AssertTool = "mesh-grants.grants_assert" +) + +// The mesh-wide brake (to-be 45 §7): how many acts all healers together may take in an hour. +const ( + BrakeLimit = 12 + BrakeWindow = time.Hour +) + +// The outcomes an attempt has. +const ( + OutcomeActing = health.OutcomeActing + OutcomeActed = health.OutcomeActed + OutcomeFailed = health.OutcomeFailed + OutcomeEscalated = health.OutcomeEscalated +) + +// Store is the bus's buckets as this module reaches them, through its runtime. CreateTTL writes a key only when it has +// none, and says ErrExists otherwise; it and PutTTL give the key its own time to live, at whose end the bus removes it +// and a watch of the bucket hears it expire (a bucket declared key-ttl). +type Store interface { + Get(state, key string) ([]byte, bool, error) + Put(state, key string, value any) error + CreateTTL(state, key string, value any, ttl time.Duration) error + PutTTL(state, key string, value any, ttl time.Duration) error + Delete(state, key string) error + Keys(state string) ([]string, error) +} + +// KeptFor is how long an attempt and a would-act are kept: each key a week from its last write (ADR 0564 Limits). +const KeptFor = 7 * 24 * time.Hour + +// WakesState holds the healers' waits as keys whose time to live is the wait: `settle.` ends with a +// settle, `fortnight` when S15's oldest counted act leaves the fortnight. Its expiry, heard on the bucket's watch, is +// the wake: no timer runs in the process, and a restart loses nothing a key on the bus holds. +const WakesState = "wakes" + +// The wakes' keys. +const ( + settlePrefix = "settle." + fortnightKey = "fortnight" +) + +// ErrExists is a create of a key another writer made first. +var ErrExists = errors.New("the key was made by another writer first") + +// Condition is what a healer reads of a condition the keeper said: the controller's condition body, its looks aside. +type Condition struct { + Key string `json:"key"` + Kind string `json:"kind"` + Subject Subject `json:"subject"` + Severity string `json:"severity"` + Resolver string `json:"resolver"` + Tried []json.RawMessage `json:"tried,omitempty"` +} + +// Escalated says a healer's budget was spent: the operator resolves it now, and no healer touches it again until +// observation clears it. +func (c Condition) Escalated() bool { return c.Resolver == "operator" && len(c.Tried) > 0 } + +// Attempt, Would and Acted are mesh-sdk's health layouts: an act begun in `attempts` (its key +// `..` is the healers' own), an act written to `would` while the lease names a +// controller (one key per healer and condition, `.`, its newest), and the body of healer-acted, +// epoch left out (ADR 0564 Limits 10). +type ( + Attempt = health.Attempt + Would = health.Would + Acted = health.HealerActed +) + +// Healer is one row of the registry. +type Healer struct { + ID string + Kinds []string + Condition string + Repair string + Then string + From string + Budget int + Window time.Duration + Settle time.Duration + // ActsIn is where the repair runs: here, or a module that repairs its own (H5), or the controller (H4). + ActsIn string + // applies says whether this condition is one it may act on, and what its budget counts against; why when not. + applies func(c Condition) (budget string, ok bool, why string) + // repair is the act: what it did, what came of it, or why it could not. + repair func(h *Healers, c Condition) (act, said string, err error) +} + +// actsHere marks a healer whose repair mesh-healers makes. +const actsHere = "mesh-healers" + +// Registry is to-be 45 §7's table, compiled in: every healer the mesh has, the ones mesh-healers runs with their act. +func Registry() []Healer { + return []Healer{ + {ID: "H3", Kinds: []string{"holder-silent", "consumer-lost"}, + Condition: "a durable consumer of a module the bus does not hold (D6), or a seat's holder that does not answer (D3)", + Repair: "assert the module's buckets, logs and streams again through the grants part (grants_assert)", + Then: "resolver operator, urgent", From: "a controller restarted to make a missing object again (208, 248)", + Budget: 1, Window: time.Hour, Settle: 6 * time.Minute, ActsIn: actsHere, + applies: appliesToAModulesObject, repair: assertAgain}, + {ID: "H4", Kinds: []string{"consumer-behind"}, + Condition: "a durable consumer far behind its stream's head (D6) that the stream table marks resettable", + Repair: "re-make the consumer to deliver from now; the one marked resettable is the controller's own", + Then: "resolver operator, urgent", From: "a consumer re-made from now by hand (248)", + Budget: 1, Window: 24 * time.Hour, Settle: 6 * time.Minute, + ActsIn: "the controller, until its lease: the one consumer marked resettable is its own (ADR 0564 §7)"}, + {ID: "H5", Kinds: []string{"provider-failing"}, + Condition: "the identity provider's administrator refusing the mesh's secret (provider-failing)", + Repair: "the provider repairs it itself through the server's own bootstrap command (ADR 0224 §5)", + Then: "announced failing by the provider, braked from ten minutes doubling to six hours", + From: "the identity provider's admin reset through its bootstrap command (179)", + Budget: 1, Window: 10 * time.Minute, Settle: 10 * time.Minute, + ActsIn: "the provider module (keycloak), ADR 0224 §5"}, + } +} + +// Healers is mesh-healers: the registry, what its acts reach, and what it holds. +type Healers struct { + Store Store + Ask func(key string, body any) (json.RawMessage, error) + // Publish says a seat event on its subject, de-duplicated by id (stdio.SeatPublish). + Publish func(subject string, body any, id string) error + Now func() time.Time + Logf func(format string, args ...any) + Registry []Healer + + mu sync.Mutex + // lease is the controller's lease as last read: acting is on only while it is handed away. + acting bool + leaseWhy string + // declined is why each condition was last passed over, said once. + declined map[string]string + // acts are the hand-act log as watched, for S15. + acts map[string]HandAct +} + +// LeaseHeard records what the lease says: acting is on only while it names a holder that is no controller (ADR 0564 +// §8, Limits 8, as ADR 0561 §3's reader reads it). +func (h *Healers) LeaseHeard(raw []byte, found bool, readErr error) { + on, why := buscompose.LeaseHandedAway(raw, found, readErr) + h.mu.Lock() + flipped := on != h.acting || h.leaseWhy == "" + h.acting, h.leaseWhy = on, leaseWords(on, why) + h.mu.Unlock() + if flipped { + h.Logf("[%s] the healers %s: %s", Module, map[bool]string{true: "ACT", false: "act in shadow"}[on], h.leaseWhy) + } +} + +// leaseWords says the lease's state as the healers read it; the SDK's words speak of minting. +func leaseWords(on bool, why string) string { + if on { + return "the controller's lease is handed away: the healers act" + } + if strings.Contains(why, "cannot be read") || strings.Contains(why, "not readable") { + return "the controller's lease cannot be read, so it is taken to name a controller: every act is written as would" + } + return "the controller's lease names a controller (or none is written, so one may take it): every act is written as would" +} + +// Acting says whether the healers act, and why. +func (h *Healers) Acting() (bool, string) { + h.mu.Lock() + defer h.mu.Unlock() + if h.leaseWhy == "" { + return false, "the controller's lease has not been read yet: every act is written as would" + } + return h.acting, h.leaseWhy +} + +// healerFor is the healer that acts here on a kind. +func (h *Healers) healerFor(kind string) (Healer, bool) { + for _, r := range h.Registry { + if r.repair != nil && slices.Contains(r.Kinds, kind) { + return r, true + } + } + return Healer{}, false +} + +// Heard is a condition the keeper raised or changed: judged, and acted on, written as would, or passed over. +func (h *Healers) Heard(body []byte) error { + var c Condition + if err := json.Unmarshal(body, &c); err != nil { + h.Logf("[%s] a condition event is not readable, and is passed over: %v", Module, err) + return nil + } + _, err := h.Judge(c, "") + return err +} + +// Judge is one condition judged by the healer registered for its kind. by names a person's `heal`, empty for an +// event. It answers what was done, in words. +func (h *Healers) Judge(c Condition, by string) (string, error) { + h.mu.Lock() + defer h.mu.Unlock() + row, ok := h.healerFor(c.Kind) + if !ok { + return "no healer here answers the kind " + c.Kind, nil + } + if c.Escalated() { + return "it is handed to the operator already: no healer touches it until observation clears it", nil + } + budget, applies, why := row.applies(c) + if !applies { + h.decline(c.Key, row.ID, why) + return fmt.Sprintf("healer %s leaves it alone: %s", row.ID, why), nil + } + delete(h.declined, c.Key) + now := h.Now() + if !h.acting { + w := Would{Healer: row.ID, Condition: c.Key, Kind: c.Kind, Act: row.Repair, At: now.UTC(), Why: h.whyNot()} + if err := h.Store.PutTTL(WouldState, row.ID+"."+c.Key, w, KeptFor); err != nil { + return "", fmt.Errorf("the would-act of %s on %s is not written: %w", row.ID, c.Key, err) + } + return fmt.Sprintf("healer %s would act on %s: %s", row.ID, c.Key, w.Why), nil + } + all, err := h.attempts() + if err != nil { + h.blind(err) + return "", fmt.Errorf("the healers cannot read what they did, so they count no budget and act on nothing: %w", err) + } + h.see("blind", nil) + acts := actsWithin(all, now.Add(-BrakeWindow)) + if braked, said := h.brakeHolds(acts, now); braked { + h.see("brake", []Observation{brakeFinding(acts, said)}) + return "every healer has stopped: " + said, nil + } + h.see("brake", nil) + spent := spentOn(all, row, budget, now) + if n := len(spent); n > 0 && now.Sub(spent[n-1].Begun) < row.Settle { + h.wake(budget, spent[n-1].Begun.Add(row.Settle), c.Key) + return fmt.Sprintf("healer %s waits: its last act on it is still the observation's to judge until %s", row.ID, + spent[n-1].Begun.Add(row.Settle).UTC().Format(time.RFC3339)), nil + } + if len(spent) >= row.Budget { + return h.escalate(row, c, budget, spent) + } + if len(acts) >= BrakeLimit { + return "the brake holds from the next act on", nil + } + return h.act(row, c, budget, len(spent), by) +} + +func (h *Healers) whyNot() string { + if h.leaseWhy == "" { + return "the controller's lease has not been read yet" + } + return h.leaseWhy +} + +// act is one healer's act: begun by a create in attempts, made, finished, and said. +func (h *Healers) act(row Healer, c Condition, budget string, spent int, by string) (string, error) { + begun := h.Now().UTC() + key := attemptKey(row.ID, c.Key, begun) + if by == "" { + by = "healer " + row.ID + } + window := fmt.Sprintf("act %d of %d within %s", spent+1, row.Budget, row.Window) + a := Attempt{Healer: row.ID, Condition: c.Key, Kind: c.Kind, BudgetKey: budget, Act: row.Repair, Outcome: OutcomeActing, + Begun: begun} + if by != "healer "+row.ID { + a.Said = "asked by " + by + } + if err := h.Store.CreateTTL(AttemptsState, key, a, KeptFor); err != nil { + if errors.Is(err, ErrExists) { + return "another instance began this act first", nil + } + return "", fmt.Errorf("healer %s did not act on %s: its attempt could not be begun: %w", row.ID, c.Key, err) + } + act, said, err := row.repair(h, c) + a.Outcome = OutcomeActed + if err != nil { + a.Outcome, said = OutcomeFailed, err.Error() + } + if act != "" { + a.Act = act + } + finished := h.Now().UTC() + if a.Said != "" { + said = a.Said + "; " + said + } + a.Said, a.Finished = said+" ("+window+")", &finished + if err := h.Store.PutTTL(AttemptsState, key, a, KeptFor); err != nil { + h.Logf("[%s] healer %s %s %s and could not record what came of it: %v", Module, row.ID, a.Outcome, c.Key, err) + } + h.tell(Acted{Healer: row.ID, Condition: c.Key, Kind: c.Kind, Act: a.Act, Outcome: a.Outcome, Said: said, Budget: window}) + h.wake(budget, begun.Add(row.Settle), c.Key) + return fmt.Sprintf("healer %s %s %s: %s — %s (%s)", row.ID, a.Outcome, c.Key, a.Act, said, window), nil +} + +// escalate is a spent budget: the condition is the operator's now. Written once as an attempt the keeper turns into +// the condition's resolver and urgency (ADR 0564 §7); an escalated condition is passed over until it clears. +func (h *Healers) escalate(row Healer, c Condition, budget string, spent []Attempt) (string, error) { + var tried []string + for _, s := range spent { + tried = append(tried, fmt.Sprintf("%s at %s (%s)", s.Outcome, s.Begun.UTC().Format("15:04 MST"), firstLine(s.Said))) + } + said := fmt.Sprintf("its budget of %d act(s) within %s is spent and %s is still open: %s", row.Budget, row.Window, + c.Key, strings.Join(tried, "; ")) + now := h.Now().UTC() + window := fmt.Sprintf("%d of %d within %s", len(spent), row.Budget, row.Window) + a := Attempt{Healer: row.ID, Condition: c.Key, Kind: c.Kind, BudgetKey: budget, Act: "handed the condition to the operator", + Outcome: OutcomeEscalated, Said: said, Begun: now, Finished: &now} + if err := h.Store.CreateTTL(AttemptsState, attemptKey(row.ID, c.Key, now), a, KeptFor); err != nil && !errors.Is(err, ErrExists) { + return "", fmt.Errorf("healer %s could not record handing %s to the operator, and does not: %w", row.ID, c.Key, err) + } + h.tell(Acted{Healer: row.ID, Condition: c.Key, Kind: c.Kind, Act: "handed to the operator", Outcome: OutcomeEscalated, + Said: said, Budget: window}) + return fmt.Sprintf("healer %s handed %s to the operator: %s", row.ID, c.Key, said), nil +} + +// tell says a heal as healer-acted. Not said is said in the log: the attempt holds it. +func (h *Healers) tell(e Acted) { + e.Event, e.At, e.By = ActedEvent, h.Now().UTC(), "healer "+e.Healer + e.Show = "mesh-conditions.conditions key=" + e.Condition + if h.Publish == nil { + return + } + if err := h.Publish(ActedSubject, e, e.Healer+"."+e.Condition+"."+e.Outcome+"."+strconv.FormatInt(e.At.UnixNano(), 10)); err != nil { + h.Logf("[%s] healer %s %s %s, and that could NOT be said on the bus: %v", Module, e.Healer, e.Outcome, e.Condition, err) + } +} + +// Rearm is the healers' start, once the lease is first read: a settle whose key expired while no instance watched has +// no expiry left to hear, so each newest act whose settle ended within its window is judged at once. A settle still +// running keeps its key on the bus and needs nothing. +func (h *Healers) Rearm() { + all, err := h.attempts() + if err != nil { + h.Logf("[%s] what the healers did cannot be read, so no ended settle is judged: %v", Module, err) + return + } + now := h.Now() + newest := map[string]Attempt{} // the newest act per healer and budget; all is oldest first + for _, a := range all { + if a.Outcome != OutcomeEscalated { + newest[a.Healer+" "+a.BudgetKey] = a + } + } + var due []string + for _, a := range newest { + for _, r := range h.Registry { + if r.ID == a.Healer && r.repair != nil && !now.Before(a.Begun.Add(r.Settle)) && now.Sub(a.Begun) <= r.Window { + due = append(due, a.Condition) + } + } + } + sort.Strings(due) + for _, key := range due { + h.again(key) + } +} + +// wake writes the wait for a settle's end as a key whose time to live is the wait; its expiry judges the condition +// again (WakeHeard). Written again with the same end, it is the same wait. +func (h *Healers) wake(_ string, at time.Time, key string) { + ttl := at.Sub(h.Now()).Round(time.Second) + time.Second + if err := h.Store.PutTTL(WakesState, settlePrefix+key, map[string]any{"condition": key, "at": at.UTC()}, ttl); err != nil { + h.Logf("[%s] the settle of %s is not written as a wait, so its end is judged at the next event: %v", Module, key, err) + } +} + +// WakeHeard is a change of the wakes bucket: an expiry is a wait ended — a settle's, whose condition is judged again, +// or S15's fortnight, counted again. +func (h *Healers) WakeHeard(key, op string) { + if op != OpExpire { + return + } + switch { + case key == fortnightKey: + h.Wanted() + case strings.HasPrefix(key, settlePrefix): + h.again(strings.TrimPrefix(key, settlePrefix)) + } +} + +// OpExpire is the watch's word for a key whose own time to live ended. +const OpExpire = stdio.OpExpire + +// again judges a condition again from the keeper's bucket, if it is still open. +func (h *Healers) again(key string) { + raw, found, err := h.Store.Get(ConditionsRead, key) + if err != nil || !found { + return + } + if _, err := h.Judge(conditionOf(raw), ""); err != nil { + h.Logf("[%s] %s judged again at its settle's end: %v", Module, key, err) + } +} + +func conditionOf(raw []byte) Condition { + var c Condition + _ = json.Unmarshal(raw, &c) + return c +} + +// decline remembers why a healer passed a condition over, said once. +func (h *Healers) decline(key, healer, why string) { + if h.declined == nil { + h.declined = map[string]string{} + } + if h.declined[key] != why { + h.Logf("[%s] healer %s leaves %s alone: %s", Module, healer, key, why) + } + h.declined[key] = why +} + +func attemptKey(healer, condition string, begun time.Time) string { + return healer + "." + condition + "." + strconv.FormatInt(begun.UnixNano(), 10) +} + +// attempts are every attempt kept, oldest first. +func (h *Healers) attempts() ([]Attempt, error) { + keys, err := h.Store.Keys(AttemptsState) + if err != nil { + return nil, err + } + var out []Attempt + for _, k := range keys { + raw, found, err := h.Store.Get(AttemptsState, k) + if err != nil { + return nil, err + } + if !found { + continue + } + var a Attempt + if err := json.Unmarshal(raw, &a); err != nil { + return nil, fmt.Errorf("the attempt %s is not readable: %w", k, err) + } + out = append(out, a) + } + sort.SliceStable(out, func(i, j int) bool { return out[i].Begun.Before(out[j].Begun) }) + return out, nil +} + +// actsWithin are the attempts since a moment that were acts: an escalation is a saying, and the brake does not count it. +func actsWithin(all []Attempt, since time.Time) []Attempt { + var out []Attempt + for _, a := range all { + if !a.Begun.Before(since) && a.Outcome != OutcomeEscalated { + out = append(out, a) + } + } + return out +} + +// spentOn is one healer's acts against one budget key within its window, oldest first. +func spentOn(all []Attempt, row Healer, budget string, now time.Time) []Attempt { + var out []Attempt + for _, a := range actsWithin(all, now.Add(-row.Window)) { + if a.Healer == row.ID && a.BudgetKey == budget { + out = append(out, a) + } + } + return out +} + +// brakeHolds says whether the mesh-wide brake holds: the limit reached within the hour, or the brake already written +// and an act within the hour still — it lets go an hour after the last act, not the first. +func (h *Healers) brakeHolds(acts []Attempt, now time.Time) (bool, string) { + said := fmt.Sprintf("%d act(s) by the healers in the last %s, the limit %d", len(acts), BrakeWindow, BrakeLimit) + if len(acts) >= BrakeLimit { + return true, said + } + _, held, _ := h.Store.Get(FindingsState, brakeKey) + if held && len(acts) > 0 { + last := acts[len(acts)-1].Begun + return true, fmt.Sprintf("%s; held until an hour after the last, at %s", said, + last.Add(BrakeWindow).UTC().Format("15:04 MST")) + } + return false, said +} + +var brakeKey = health.Key(ScopeMesh, "healers", "braked") + +// brakeFinding is the brake, as the urgent finding that says it. +func brakeFinding(acts []Attempt, said string) Observation { + counts := map[string]int{} + for _, a := range acts { + counts[a.Healer+" on "+a.Condition]++ + } + var most []string + for what, n := range counts { + most = append(most, fmt.Sprintf("%s ×%d", what, n)) + } + sort.Strings(most) + return Observation{Kind: "healers-braked", Token: "braked", Severity: Urgent, Subject: Subject{Scope: ScopeMesh, ID: "healers"}, + Summary: "every healer has stopped: the healers acted more often in an hour than the mesh allows, which is a " + + "healer looping, not repairing — " + said, + Said: strings.Join(most, ", "), + Headline: "All automatic repairs stopped", + Needs: "read the details and decide whether the repairs may run again.", + Explanation: "The healers (the mesh's automatic repairs) acted more often in an hour than allowed, which looks " + + "like a loop, so all of them stopped.", + Resolved: "Automatic repairs run again"} +} + +// blind writes the healers' own blindness: they cannot read what they did. +func (h *Healers) blind(err error) { + h.see("blind", []Observation{{Kind: "probe-failed", Token: "failed", Severity: Warning, + Subject: Subject{Scope: ScopeProbe, ID: "healers"}, + Summary: "the healers cannot read what they did, so they count no budget and act on nothing", + Said: firstLine(err.Error()), Headline: "A check of the mesh could not run", + Explanation: "The automatic repairs cannot read what they did, so they act on nothing until they can.", + Resolved: "The automatic repairs read what they did again"}}) +} + +// see writes the healers' own findings of one source, each only when it changes, and deletes those of the source it +// no longer sees (ADR 0564 §2). +func (h *Healers) see(source string, found []Observation) { + now := h.Now() + want := map[string]health.Finding{} + for _, o := range found { + o.Source = source + want[o.Key()] = o.Finding("", now) + } + keys, err := h.Store.Keys(FindingsState) + if err != nil { + h.Logf("[%s] its own findings cannot be read, so none is written or cleared: %v", Module, err) + return + } + for _, k := range keys { + raw, found, err := h.Store.Get(FindingsState, k) + if err != nil || !found { + continue + } + was, err := health.DecodeFinding(raw) + if err != nil || was.Source != source { + continue + } + if f, still := want[k]; still { + if says(was) == says(f) { + delete(want, k) + } + continue + } + if err := h.Store.Delete(FindingsState, k); err != nil { + h.Logf("[%s] the finding %s no longer seen is not deleted: %v", Module, k, err) + } + } + for k, f := range want { + raw, err := health.EncodeFinding(f) + if err == nil { + err = h.Store.Put(FindingsState, k, json.RawMessage(raw)) + } + if err != nil { + h.Logf("[%s] the finding %s is not written: %v", Module, k, err) + } + } +} + +// --- H3 -------------------------------------------------------------------------------------------------------- + +// moduleOf is the module whose objects a condition names: a consumer of a module's own (`.`). The +// controller's own consumers are not a module's: the controller asserts them itself until its lease. +func moduleOf(c Condition) (string, string) { + stream, consumer, ok := strings.Cut(c.Subject.ID, ".") + if c.Kind != "consumer-lost" || !ok || stream == "" || consumer == "" { + return "", "it names no consumer of a module whose objects the grants part asserts" + } + if consumer == buscompose.ControllerName || strings.HasPrefix(consumer, buscompose.ControllerName+"-") { + return "", "the consumer is the controller's own, which the controller asserts itself while it holds its lease" + } + return consumer, "" +} + +// appliesToAModulesObject is H3's: a module's consumer the bus does not hold, its budget per condition. +func appliesToAModulesObject(c Condition) (string, bool, string) { + if _, why := moduleOf(c); why != "" { + return "", false, why + } + return c.Key, true, "" +} + +// assertAgain is H3: the grants part asserts the module's buckets, logs and streams again. +func assertAgain(h *Healers, c Condition) (string, string, error) { + module, why := moduleOf(c) + if module == "" { + return "", "", errors.New(why) + } + raw, err := h.Ask(AssertTool, map[string]any{"module": module}) + if err != nil { + return "", "", fmt.Errorf("the grants part did not assert %s: %w", module, err) + } + var a struct { + Says string `json:"says"` + } + _ = json.Unmarshal(raw, &a) + return "asserted " + module + "'s buckets, logs and streams again through the grants part", + orSaid(a.Says, "the grants part answered") + "; the probe that raised it says on its next run whether it is there", nil +} + +func orSaid(s, otherwise string) string { + if strings.TrimSpace(s) == "" { + return otherwise + } + return s +} + +func firstLine(s string) string { + line, _, _ := strings.Cut(s, "\n") + return line +} diff --git a/modules/mesh-healers/cmd/mesh-healers/healers_test.go b/modules/mesh-healers/cmd/mesh-healers/healers_test.go new file mode 100644 index 00000000..9b6f9dce --- /dev/null +++ b/modules/mesh-healers/cmd/mesh-healers/healers_test.go @@ -0,0 +1,496 @@ +package main + +import ( + "encoding/json" + "errors" + "slices" + "sort" + "strings" + "sync" + "testing" + "time" + + "git.novox.be/novox/mesh-sdk/go/handacts" + "git.novox.be/novox/mesh-sdk/go/health" +) + +type memStore struct { + mu sync.Mutex + kv map[string]map[string][]byte + keysErr map[string]error + ttl map[string]time.Duration +} + +func newStore() *memStore { + return &memStore{kv: map[string]map[string][]byte{}, keysErr: map[string]error{}} +} + +func (s *memStore) Get(state, key string) ([]byte, bool, error) { + s.mu.Lock() + defer s.mu.Unlock() + v, ok := s.kv[state][key] + return v, ok, nil +} + +func (s *memStore) put(state, key string, value any, create bool) error { + raw, err := json.Marshal(value) + if err != nil { + return err + } + s.mu.Lock() + defer s.mu.Unlock() + if s.kv[state] == nil { + s.kv[state] = map[string][]byte{} + } + if _, ok := s.kv[state][key]; ok && create { + return ErrExists + } + s.kv[state][key] = raw + return nil +} + +func (s *memStore) Put(state, key string, value any) error { return s.put(state, key, value, false) } +func (s *memStore) Create(state, key string, value any) error { return s.put(state, key, value, true) } + +// CreateTTL and PutTTL keep the key's time to live beside it, for a test to read. +func (s *memStore) CreateTTL(state, key string, value any, ttl time.Duration) error { + if err := s.put(state, key, value, true); err != nil { + return err + } + s.setTTL(state, key, ttl) + return nil +} + +func (s *memStore) PutTTL(state, key string, value any, ttl time.Duration) error { + if err := s.put(state, key, value, false); err != nil { + return err + } + s.setTTL(state, key, ttl) + return nil +} + +func (s *memStore) setTTL(state, key string, ttl time.Duration) { + s.mu.Lock() + defer s.mu.Unlock() + if s.ttl == nil { + s.ttl = map[string]time.Duration{} + } + s.ttl[state+"/"+key] = ttl +} + +// ttlOf is a key's time to live as last written; zero for none. +func (s *memStore) ttlOf(state, key string) time.Duration { + s.mu.Lock() + defer s.mu.Unlock() + return s.ttl[state+"/"+key] +} + +func (s *memStore) Delete(state, key string) error { + s.mu.Lock() + defer s.mu.Unlock() + delete(s.kv[state], key) + return nil +} + +func (s *memStore) Keys(state string) ([]string, error) { + s.mu.Lock() + defer s.mu.Unlock() + if err := s.keysErr[state]; err != nil { + return nil, err + } + var out []string + for k := range s.kv[state] { + out = append(out, k) + } + sort.Strings(out) + return out, nil +} + +func (s *memStore) keys(state string) []string { + k, _ := s.Keys(state) + return k +} + +func (s *memStore) attempt(t *testing.T, key string) Attempt { + t.Helper() + raw, _, _ := s.Get(AttemptsState, key) + var a Attempt + if err := json.Unmarshal(raw, &a); err != nil { + t.Fatalf("%s: %s", key, raw) + } + return a +} + +// world is mesh-healers over memory, with its clock, its asks, what it said and the wakes it set. +type world struct { + h *Healers + store *memStore + now time.Time + asked []string + said []Acted +} + +func newWorld() *world { + w := &world{store: newStore(), now: time.Date(2026, 10, 29, 21, 0, 0, 0, time.UTC)} + w.fresh() + return w +} + +// fresh is the healers started again over the same buckets: nothing of the last one's memory, its wakes forgotten. +func (w *world) fresh() { + w.h = &Healers{Store: w.store, Registry: Registry(), Logf: func(string, ...any) {}, + Now: func() time.Time { return w.now }, + Ask: func(key string, body any) (json.RawMessage, error) { + raw, _ := json.Marshal(body) + w.asked = append(w.asked, key+" "+string(raw)) + return json.RawMessage(`{"says":"2 the bus lacks: created at this ask"}`), nil + }, + Publish: func(subject string, body any, _ string) error { + if a, ok := body.(Acted); ok && subject == "mesh.seat.mesh-healers.event.healer-acted" { + w.said = append(w.said, a) + } + return nil + }, + } +} + +func (w *world) handAway() { + w.h.LeaseHeard([]byte(`{"handed-to":"the operator's hand-over"}`), true, nil) +} + +// lost is a module's consumer the bus does not hold, as D6 raises it. +func lost(module string) Condition { + return Condition{Key: "bus.EVENTS." + module + ".missing", Kind: "consumer-lost", Severity: "warning", Resolver: "self", + Subject: Subject{Scope: "bus", ID: "EVENTS." + module}} +} + +func body(c Condition) []byte { + raw, _ := json.Marshal(map[string]any{"key": c.Key, "kind": c.Kind, "subject": c.Subject, "severity": c.Severity, + "resolver": c.Resolver, "tried": c.Tried, "event": "condition-raised", "show": "mesh-conditions.conditions key=" + c.Key}) + return raw +} + +// No act is made while the lease names a controller: each is written to would, never to attempts, so it counts for no +// budget or brake and nothing is asked or said; one is made from the write that hands the lease away (ADR 0564 §8). +func TestTheHealersActOnlyOnceTheLeaseNamesNoController(t *testing.T) { + cases := []struct { + name string + lease func(h *Healers) + acts bool + }{ + {"never read", func(*Healers) {}, false}, + {"absent", func(h *Healers) { h.LeaseHeard(nil, false, nil) }, false}, + {"unreadable", func(h *Healers) { h.LeaseHeard(nil, false, errors.New("timeout")) }, false}, + {"a controller holds it", func(h *Healers) { + h.LeaseHeard([]byte(`{"instance":"novox/1/2026-10-27","epoch":57}`), true, nil) + }, false}, + {"handed away", func(h *Healers) { h.LeaseHeard([]byte(`{"handed-to":"the operator"}`), true, nil) }, true}, + } + for _, tc := range cases { + w := newWorld() + tc.lease(w.h) + if err := w.h.Heard(body(lost("records"))); err != nil { + t.Fatalf("%s: %v", tc.name, err) + } + attempts, would := w.store.keys(AttemptsState), w.store.keys(WouldState) + if tc.acts { + if len(attempts) != 1 || len(would) != 0 || len(w.asked) != 1 || !strings.Contains(w.asked[0], `"module":"records"`) || + len(w.said) != 1 || w.said[0].Outcome != OutcomeActed { + t.Errorf("%s: attempts %v, would %v, asked %v, said %+v", tc.name, attempts, would, w.asked, w.said) + continue + } + a := w.store.attempt(t, attempts[0]) + if a.Outcome != OutcomeActed || a.Finished == nil || a.Healer != "H3" || a.Condition != "bus.EVENTS.records.missing" { + t.Errorf("%s: the attempt is %+v", tc.name, a) + } + continue + } + if len(attempts) != 0 || len(w.asked) != 0 || len(w.said) != 0 || !slices.Equal(would, []string{"H3.bus.EVENTS.records.missing"}) { + t.Errorf("%s: attempts %v, would %v, asked %v, said %v", tc.name, attempts, would, w.asked, w.said) + } + if _, braked, _ := w.store.Get(FindingsState, brakeKey); braked { + t.Errorf("%s: a would-act moved the brake", tc.name) + } + } +} + +// An act waits out its settle, then its spent budget hands the condition to the operator once; an escalated condition +// is passed over; a second instance cannot begin an act already begun; the brake stops every healer past twelve acts +// in an hour and lets go an hour after the last (ADR 0231 §21, ADR 0564 §7). +func TestBudgetSettleEscalationAndBrake(t *testing.T) { + w := newWorld() + w.handAway() + c := lost("records") + _ = w.store.Put(ConditionsRead, c.Key, c) + settle := settlePrefix + c.Key + steps := []struct { + name string + after time.Duration + cond Condition + attempts int + outcome string // the newest said outcome, empty for none new + // how is what happens: an event, a restart over the same buckets, or the settle key's expiry heard. + how string + // wait is the settle key's time to live after the step, zero for no check. + wait time.Duration + }{ + {"acts: its settle is a key that expires", 0, c, 1, OutcomeActed, "event", 6*time.Minute + time.Second}, + {"within its settle: waits for the same end", 2 * time.Minute, c, 1, "", "event", 4*time.Minute + time.Second}, + {"a restart within its settle: the key on the bus still waits", time.Minute, c, 1, "", "restart", 4*time.Minute + time.Second}, + {"its key expires: judged again, budget spent, escalates", 4 * time.Minute, c, 2, OutcomeEscalated, "expire", 0}, + {"escalated: passed over", time.Minute, Condition{Key: c.Key, Kind: c.Kind, Subject: c.Subject, Resolver: "operator", + Tried: []json.RawMessage{json.RawMessage(`{}`)}}, 2, "", "event", 0}, + } + for _, s := range steps { + w.now = w.now.Add(s.after) + before := len(w.said) + switch s.how { + case "restart": + w.fresh() + w.handAway() + w.h.Rearm() + case "expire": + w.h.WakeHeard(settle, OpExpire) + default: + if _, err := w.h.Judge(s.cond, ""); err != nil { + t.Fatalf("%s: %v", s.name, err) + } + } + if n := len(w.store.keys(AttemptsState)); n != s.attempts { + t.Errorf("%s: %d attempts, want %d", s.name, n, s.attempts) + } + if s.wait != 0 && w.store.ttlOf(WakesState, settle) != s.wait { + t.Errorf("%s: the settle waits %s, want %s", s.name, w.store.ttlOf(WakesState, settle), s.wait) + } + switch { + case s.outcome == "" && len(w.said) != before: + t.Errorf("%s: said %+v", s.name, w.said[before:]) + case s.outcome != "" && (len(w.said) != before+1 || w.said[before].Outcome != s.outcome): + t.Errorf("%s: said %+v, want %s", s.name, w.said[before:], s.outcome) + } + } + // Each attempt is kept a week from its last write. + for _, k := range w.store.keys(AttemptsState) { + if ttl := w.store.ttlOf(AttemptsState, k); ttl != KeptFor { + t.Errorf("the attempt %s is kept %s, want a week", k, ttl) + } + } + // A settle whose key expired while no instance watched is judged at the start. + r := newWorld() + r.handAway() + _ = r.store.Put(ConditionsRead, c.Key, c) + if _, err := r.h.Judge(c, ""); err != nil { + t.Fatal(err) + } + r.now = r.now.Add(7 * time.Minute) + r.fresh() + r.handAway() + r.h.Rearm() + if len(r.said) != 2 || r.said[1].Outcome != OutcomeEscalated { + t.Errorf("a settle that ended unheard is not judged at the start: %+v", r.said) + } + // A second instance cannot begin the act an attempt's key already holds. + x := newWorld() + x.handAway() + _ = x.store.CreateTTL(AttemptsState, attemptKey("H3", lost("mail").Key, x.now.UTC()), Attempt{}, KeptFor) + if said, _ := x.h.Judge(lost("mail"), ""); !strings.Contains(said, "another instance") || len(x.asked) != 0 { + t.Errorf("a second instance acted: %q, asked %v", said, x.asked) + } + // The brake: twelve acts in the hour stop every healer, said as an urgent finding. + b := newWorld() + b.handAway() + for i := 0; i < BrakeLimit; i++ { + b.now = b.now.Add(time.Minute) + _ = b.store.CreateTTL(AttemptsState, attemptKey("H3", "x", b.now), Attempt{Healer: "H3", Condition: "x", BudgetKey: "x", + Outcome: OutcomeActed, Begun: b.now}, KeptFor) + } + if said, _ := b.h.Judge(lost("mail"), ""); !strings.Contains(said, "stopped") || len(b.asked) != 0 { + t.Errorf("past the brake: %q, asked %v", said, b.asked) + } + raw, braked, _ := b.store.Get(FindingsState, brakeKey) + f, _ := health.DecodeFinding(raw) + if !braked || f.Severity != Urgent || f.Kind != "healers-braked" || f.Observer != Module { + t.Errorf("the brake is written as %s", raw) + } + b.now = b.now.Add(55 * time.Minute) // fewer than twelve within the hour, but the brake is said: held + if said, _ := b.h.Judge(lost("mail"), ""); !strings.Contains(said, "held until") { + t.Errorf("the brake let go before an hour after the last act: %q", said) + } + b.now = b.now.Add(6 * time.Minute) + if _, err := b.h.Judge(lost("mail"), ""); err != nil || len(b.asked) != 1 { + t.Errorf("an hour after the last act the healers do not act again: %v, asked %v", err, b.asked) + } + if _, braked, _ := b.store.Get(FindingsState, brakeKey); braked { + t.Error("the brake's finding outlived the brake") + } + // Blind to what they did, they act on nothing and say so. + z := newWorld() + z.handAway() + z.store.keysErr[AttemptsState] = errors.New("the bus is away") + if _, err := z.h.Judge(lost("mail"), ""); err == nil || len(z.asked) != 0 { + t.Errorf("blind, the healers acted: %v", err) + } + if keys := z.store.keys(FindingsState); !slices.Equal(keys, []string{"probe.healers.failed"}) { + t.Errorf("blind, the findings are %v", keys) + } +} + +// H3 answers a module's consumer the bus does not hold, never the controller's own or a condition naming no consumer; +// the registry holds every healer the mesh has, H3 run here, H4 and H5 run elsewhere (ADR 0231 §2, ADR 0564 §7). +func TestTheRegistryAndWhatH3Answers(t *testing.T) { + cases := []struct { + c Condition + holds bool + }{ + {lost("records"), true}, + {lost("controller"), false}, + {lost("controller-merges"), false}, + {Condition{Key: "seat.x.silent", Kind: "holder-silent", Subject: Subject{Scope: "seat", ID: "issue-tracker"}}, false}, + {Condition{Key: "bus.EVENTS.records.behind", Kind: "consumer-behind", Subject: Subject{Scope: "bus", ID: "EVENTS.records"}}, false}, + } + for _, tc := range cases { + w := newWorld() + w.handAway() + said, _ := w.h.Judge(tc.c, "") + if acted := len(w.asked) == 1; acted != tc.holds { + t.Errorf("%s: acted %v, want %v (%s)", tc.c.Key, acted, tc.holds, said) + } + } + runs := map[string]bool{} + for _, r := range Registry() { + if len(r.Kinds) == 0 || r.Budget < 1 || r.Window <= 0 || r.Settle <= 0 || r.Repair == "" || r.ActsIn == "" { + t.Errorf("%s is not a whole row: %+v", r.ID, r) + } + runs[r.ID] = r.repair != nil + } + if !runs["H3"] || runs["H4"] || runs["H5"] || len(runs) != 3 { + t.Errorf("the healers run here are %v, want H3 alone of H3, H4, H5", runs) + } +} + +// S15: a cause repaired by hand twice within a fortnight is a finding; a decision by design never counts; it clears +// when fewer than two remain, and the healers wake when the oldest leaves the fortnight (ADR 0564 Limits 2). +func TestACauseRepairedByHandTwiceIsAHealerWanted(t *testing.T) { + w := newWorld() + act := func(key, verb, cause string, ago time.Duration) { + raw, _ := json.Marshal(HandAct{ID: key, At: w.now.Add(-ago), By: "person.jochen", Verb: verb, Why: "x", Cause: cause}) + w.h.HandActHeard(key, "put", raw) + } + act("a1", "broker consumer-reset", "consumer-behind", 10*24*time.Hour) + if keys := w.store.keys(FindingsState); len(keys) != 0 { + t.Fatalf("one act is a finding: %v", keys) + } + // The second of the cause is the seat's own act, heard as act-recorded: one in each log counts two (ADR 0565 §11). + seatAct, _ := json.Marshal(HandAct{ID: "a2", At: w.now.Add(-24 * time.Hour), By: "person.jochen", + Verb: "broker consumer-reset", Why: "x", Cause: "consumer-behind"}) + w.h.ActRecorded(seatAct) + // A drill in each log counts none. + act("a3", "hand-act drill", "drill", time.Hour) + drill, _ := json.Marshal(HandAct{ID: "a4", At: w.now.Add(-2 * time.Hour), Verb: "hand-act drill", Why: "x", Cause: "drill"}) + w.h.ActRecorded(drill) + act("a5", "conditions silence", "operator-answer", time.Hour) + act("a6", "conditions silence", "operator-answer", time.Hour) + act("a7", "hand-act record", "relogin", 20*24*time.Hour) + act("a8", "hand-act record", "relogin", time.Hour) + want := []string{"mesh.hand-acts.consumer-behind.healer-wanted"} + if keys := w.store.keys(FindingsState); !slices.Equal(keys, want) { + t.Fatalf("findings %v, want %v", keys, want) + } + raw, _, _ := w.store.Get(FindingsState, want[0]) + if !strings.Contains(string(raw), "healer H4 answers this cause") { + t.Errorf("the finding does not say its healer is not enough: %s", raw) + } + if ttl := w.store.ttlOf(WakesState, fortnightKey); ttl != 4*24*time.Hour+time.Second { + t.Errorf("the fortnight waits %s, want the oldest act's leaving it in 4 days", ttl) + } + w.now = w.now.Add(4*24*time.Hour + time.Second) + w.h.Wanted() + if keys := w.store.keys(FindingsState); len(keys) != 0 { + t.Errorf("past the fortnight the finding stays: %v", keys) + } + w.h.HandActHeard("a5", "delete", nil) + if len(w.h.acts) != 7 { + t.Errorf("a deleted entry is kept: %d", len(w.h.acts)) + } +} + +// The verbs: heal acts on an open condition by key and says why it was asked; drill waits for the log's home; the reads +// answer every bucket. +func TestTheVerbsAndReads(t *testing.T) { + w := newWorld() + w.handAway() + c := lost("records") + _ = w.store.Put(ConditionsRead, c.Key, c) + if _, err := w.h.Heal(c.Key, "", "person.jochen"); err == nil { + t.Error("heal took no why") + } + if _, err := w.h.Heal("bus.none", "x", "person.jochen"); err == nil { + t.Error("heal acted on no open condition") + } + if _, err := w.h.Heal(c.Key, "it is gone again", "person.jochen"); err != nil { + t.Fatal(err) + } + keys := w.store.keys(AttemptsState) + if len(keys) != 1 || !strings.Contains(w.store.attempt(t, keys[0]).Said, "asked by person.jochen (it is gone again)") { + t.Errorf("heal's attempt is %v", keys) + } + answer, err := w.h.Answer(7, true) + if err != nil { + t.Fatal(err) + } + a := answer.(map[string]any) + if a["acting"] != true || len(a["acts"].([]Attempt)) != 1 || len(a["healers"].([]map[string]any)) != 3 { + t.Errorf("healers answered %v", a) + } + // drill records into mesh-hand-acts as a drill; while that seat has no holder, into the controller's drill until + // the lease is handed away (ADR 0565 §11). + drills := []struct { + name string + seatGone bool + leaseAway bool + args map[string]any + log, err string + }{ + {"into the seat", false, true, map[string]any{"what": "stopped records", "why": "does it raise"}, handacts.LogSeat, ""}, + {"no holder: the controller's", true, false, map[string]any{"what": "stopped records", "why": "x"}, handacts.LogController, ""}, + {"no holder, lease away", true, true, map[string]any{"what": "stopped records", "why": "x"}, "", "handed its lease away"}, + {"says nothing", false, true, map[string]any{}, "", "says what"}, + } + for _, d := range drills { + x := newWorld() + if d.leaseAway { + x.handAway() + } else { + x.h.LeaseHeard([]byte(`{"instance":"novox/1","epoch":5}`), true, nil) + } + var asked []string + x.h.Ask = func(key string, body any) (json.RawMessage, error) { + raw, _ := json.Marshal(body) + asked = append(asked, key+" "+string(raw)) + if d.seatGone && strings.HasPrefix(key, "seat:mesh-hand-acts.") { + return nil, errors.New("nats: no responders available for request") + } + return json.RawMessage(`{}`), nil + } + var got any + var err error + for _, tool := range Tools(x.h) { + if tool.Name == DrillTool { + got, err = tool.Run(d.args) + } + } + switch { + case d.err != "": + if err == nil || !strings.Contains(err.Error(), d.err) { + t.Errorf("drill %s: %v", d.name, err) + } + case err != nil || got.(map[string]any)["log"] != d.log: + t.Errorf("drill %s: %v, %v", d.name, got, err) + case d.log == handacts.LogSeat && !strings.Contains(asked[0], `"verb":"hand-act drill"`): + t.Errorf("drill %s asked %v", d.name, asked) + } + } + if _, err := w.h.read(WouldState, ""); err != nil { + t.Error(err) + } +} diff --git a/modules/mesh-healers/cmd/mesh-healers/main.go b/modules/mesh-healers/cmd/mesh-healers/main.go new file mode 100644 index 00000000..6e48bfba --- /dev/null +++ b/modules/mesh-healers/cmd/mesh-healers/main.go @@ -0,0 +1,118 @@ +// mesh-healers: the holder of the seat mesh-healers (novox/hq ADR 0564 §7, §8). A Go bundle served by a runtime of its +// own, on its own bus account (`runs-as`), so that its account alone writes its buckets: `attempts` (every act begun, +// made and finished, and every escalation, kept a week), `would` (what it would have done while the controller holds +// its lease, kept a week) and `findings` (the brake, its blindness, and S15). It hears the condition keeper's +// condition-raised and condition-changed on a durable consumer, watches the controller's lease, and watches the +// controller's hand-act log for S15. stdout is the MCP channel; what this module says, it says on stderr. +package main + +import ( + "encoding/json" + "errors" + "fmt" + "os" + "sync" + "time" + + stdio "git.novox.be/novox/mesh-sdk/go" +) + +func logf(format string, a ...any) { fmt.Fprintf(os.Stderr, format+"\n", a...) } + +// runtimeStore is the bus, through the runtime that launched this bundle. +type runtimeStore struct{} + +func (runtimeStore) Get(state, key string) ([]byte, bool, error) { + e, err := stdio.State(state).Get(key) + if err != nil || e == nil { + return nil, false, err + } + return e.Value, true, nil +} + +func (runtimeStore) Put(state, key string, value any) error { + _, err := stdio.State(state).Put(key, value) + return err +} + +func (runtimeStore) CreateTTL(state, key string, value any, ttl time.Duration) error { + _, err := stdio.State(state).CreateTTL(key, value, ttl) + if errors.Is(err, stdio.ErrChanged) { + return ErrExists + } + return err +} + +func (runtimeStore) PutTTL(state, key string, value any, ttl time.Duration) error { + _, err := stdio.State(state).PutTTL(key, value, ttl) + return err +} + +func (runtimeStore) Delete(state, key string) error { return stdio.State(state).Delete(key) } + +func (runtimeStore) Keys(state string) ([]string, error) { return stdio.State(state).Keys() } + +// watched asks a watch of the runtime again until it is taken: the runtime answers only once Serve runs. +func watched(state, key string, handle func(stdio.StateChange) error) { + for wait := 2 * time.Second; ; wait = min(wait*2, time.Minute) { + time.Sleep(wait) + if err := stdio.State(state).Watch(key, handle); err != nil { + logf("[%s] %s is not watched yet (%v); asking again in %s", Module, state, err, wait) + continue + } + return + } +} + +func main() { + h := &Healers{Store: runtimeStore{}, Registry: Registry(), Now: time.Now, Logf: logf, + Ask: func(key string, body any) (json.RawMessage, error) { return stdio.Ask(key, body) }, + Publish: stdio.SeatPublish} + // The lease first: until it is read the healers write would-acts only. Once it is first read, a settle that ended + // while no instance watched is judged (Rearm); the waits still running are keys on the bus, heard on their expiry. + var started sync.Once + go watched(LeaseRead, LeaseKey, func(c stdio.StateChange) error { + h.LeaseHeard(c.Value, c.Op == "put", nil) + started.Do(func() { go h.Rearm() }) + return nil + }) + go watched(WakesState, "", func(c stdio.StateChange) error { + h.WakeHeard(c.Key, c.Op) + return nil + }) + go watched(HandActsRead, "", func(c stdio.StateChange) error { + h.HandActHeard(c.Key, c.Op, c.Value) + return nil + }) + // S15 over both logs: the seat's act-recorded on a durable consumer, beside the controller's bucket's watch. + go func() { + for wait := 2 * time.Second; ; wait = min(wait*2, time.Minute) { + time.Sleep(wait) + err := stdio.Subscribe(RecordedEvent, func(e stdio.Envelope) error { + h.ActRecorded(e.Body) + return nil + }) + if err == nil { + return + } + logf("[%s] does not hear %s yet (%v); asking again in %s", Module, RecordedEvent, err, wait) + } + }() + for _, event := range []string{RaisedEvent, ChangedEvent} { + go func(event string) { + for wait := 2 * time.Second; ; wait = min(wait*2, time.Minute) { + time.Sleep(wait) + err := stdio.Subscribe(event, func(e stdio.Envelope) error { return h.Heard(e.Body) }) + if err == nil { + logf("[%s] hears %s", Module, event) + return + } + logf("[%s] does not hear %s yet (%v); asking again in %s", Module, event, err, wait) + } + }(event) + } + if err := stdio.Serve("", Tools(h)); err != nil { + logf("%v", err) + os.Exit(1) + } +} diff --git a/modules/mesh-healers/cmd/mesh-healers/manifest_test.go b/modules/mesh-healers/cmd/mesh-healers/manifest_test.go new file mode 100644 index 00000000..af3745b9 --- /dev/null +++ b/modules/mesh-healers/cmd/mesh-healers/manifest_test.go @@ -0,0 +1,92 @@ +package main + +import ( + "encoding/json" + "os" + "path/filepath" + "slices" + "strings" + "testing" +) + +type manifest struct { + Module string `json:"module"` + RunsAs string `json:"runs-as"` + Claims []struct { + Name string `json:"name"` + Scope string `json:"scope"` + Serves []string `json:"serves"` + } `json:"claims"` + Emits []string `json:"emits"` + Consumes []string `json:"consumes"` + Invokes []string `json:"invokes"` + Reads []string `json:"reads"` + State []json.RawMessage `json:"state"` +} + +// The manifest says what the code does: it claims the seat mesh-healers once for the mesh with its verbs (ADR 0564 §1, +// §7), runs on its own account, says healer-acted, hears the keeper's raised and changed, asks only the grants part's +// assert, reads the lease, the keeper's conditions and the hand-act log, and keeps attempts and would a week each. +// Every state it keeps has a tool that reads it, and every act it takes by itself a tool that starts it (ADR 0564 §12). +func TestTheManifestSaysWhatTheCodeDoes(t *testing.T) { + raw, err := os.ReadFile(filepath.Join("..", "..", "module.json")) + if err != nil { + t.Fatal(err) + } + var m manifest + if err := json.Unmarshal(raw, &m); err != nil { + t.Fatal(err) + } + if m.Module != Module || m.RunsAs != Module { + t.Errorf("module %q runs as %q", m.Module, m.RunsAs) + } + if len(m.Claims) != 1 || m.Claims[0].Name != Seat || m.Claims[0].Scope != "mesh" || !slices.Equal(m.Claims[0].Serves, Verbs) { + t.Errorf("claims %+v, want %s at mesh scope serving %v", m.Claims, Seat, Verbs) + } + for _, c := range []struct { + name string + got, want []string + }{ + {"emits (healer-acted is the seat's, said on its subject)", m.Emits, nil}, + {"consumes", m.Consumes, []string{RaisedEvent, ChangedEvent, RecordedEvent}}, + {"invokes", m.Invokes, []string{AssertTool, "seat:mesh-hand-acts.hand-act", "seat:mesh-controller.drill"}}, + {"reads", m.Reads, []string{LeaseRead, ConditionsRead, HandActsRead}}, + } { + if !slices.Equal(c.got, c.want) { + t.Errorf("%s %v, want %v", c.name, c.got, c.want) + } + } + states := map[string]bool{} + for _, s := range m.State { + var d struct { + Name string `json:"name"` + KeyTTL bool `json:"key-ttl"` + } + if json.Unmarshal(s, &d.Name) != nil { + _ = json.Unmarshal(s, &d) + } + states[d.Name] = d.KeyTTL + } + // Each key of attempts and would lives a week from its last write (KeptFor, would has no other deleter), and each + // wait in wakes as long as it waits: their buckets take a key's own time to live. + if !states[AttemptsState] || !states[WouldState] || !states[WakesState] || states[FindingsState] { + t.Errorf("the states take a key's own time to live as %v, want attempts, would and wakes alone", states) + } + readBy := map[string]string{AttemptsState: HealersTool, WouldState: WouldTool, FindingsState: FindingsTool, + WakesState: WakesTool, "a settle's end": HealTool, + "an act on an event": HealTool, "S15's count": HealersTool + " recount=true", "the lease as read": HealersTool} + tools := map[string]bool{} + for _, tool := range Tools(&Healers{}) { + tools[tool.Name] = true + } + for state := range states { + if _, ok := readBy[state]; !ok { + t.Errorf("the state %s has no tool that reads it", state) + } + } + for what, tool := range readBy { + if name, _, _ := strings.Cut(tool, " "); !tools[name] { + t.Errorf("%s is read or started by %s, which is not served", what, tool) + } + } +} diff --git a/modules/mesh-healers/cmd/mesh-healers/observation.go b/modules/mesh-healers/cmd/mesh-healers/observation.go new file mode 100644 index 00000000..853550b5 --- /dev/null +++ b/modules/mesh-healers/cmd/mesh-healers/observation.go @@ -0,0 +1,77 @@ +package main + +// What the healers observe of themselves, and the finding it is written as (novox/hq ADR 0564 §2): the brake, their +// blindness and S15, each written into the bucket `findings` as mesh-sdk's health.Finding, whose key is the +// controller's key for the same condition. + +import ( + "encoding/json" + "time" + + "git.novox.be/novox/mesh-sdk/go/health" +) + +// The severities and scopes, as mesh-sdk's health names them. +const ( + Urgent = health.Urgent + Warning = health.Warning + ScopeProbe = health.ScopeProbe + ScopeMesh = health.ScopeMesh +) + +// Subject is what a finding is about. +type Subject = health.Subject + +// Observation is one thing a probe sees wrong now. +type Observation struct { + Kind string + Token string + Subject Subject + Severity health.Severity + Summary string + Said string + Headline string + Explanation string + Resolved string + Needs string + // Source is what saw it: the brake, the healers' blindness, or S15. + Source string +} + +// Key is where it is kept: `..`, the controller's key for the same condition. +func (o Observation) Key() string { + token := o.Token + if token == "" { + token = o.Kind + } + return health.Key(o.Subject.Scope, o.Subject.ID, token) +} + +// Finding is the observation as written by run at, its evidence the one line it said. +func (o Observation) Finding(run string, at time.Time) health.Finding { + f := health.Finding{Key: o.Key(), Kind: o.Kind, Subject: o.Subject, Severity: o.Severity, Summary: o.Summary, + Headline: o.Headline, Explanation: o.Explanation, Resolved: o.Resolved, Needs: o.Needs, Source: o.Source, + Observer: Module, Run: run, At: at.UTC(), Evidence: []health.Evidence{}} + if o.Token != "" && o.Token != o.Kind { + f.Token = o.Token + } + said := o.Said + if said == "" { + said = o.Summary + } + f.Evidence = append(f.Evidence, health.Evidence{At: at.UTC(), Said: said}) + return f +} + +// says is what a finding says, its times and the run that wrote it aside: two findings that say the same are one write +// (ADR 0289 §1). +func says(f health.Finding) string { + f.At, f.Run, f.V = time.Time{}, "", 0 + ev := make([]health.Evidence, len(f.Evidence)) + for i, e := range f.Evidence { + ev[i] = health.Evidence{Said: e.Said} + } + f.Evidence = ev + raw, _ := json.Marshal(f) + return string(raw) +} diff --git a/modules/mesh-healers/cmd/mesh-healers/reads-outside b/modules/mesh-healers/cmd/mesh-healers/reads-outside new file mode 100644 index 00000000..cdc54f27 --- /dev/null +++ b/modules/mesh-healers/cmd/mesh-healers/reads-outside @@ -0,0 +1,2 @@ +# What these tests read outside their package, as paths from the repository's root (issue 1936). +modules/mesh-healers/module.json diff --git a/modules/mesh-healers/cmd/mesh-healers/tools.go b/modules/mesh-healers/cmd/mesh-healers/tools.go new file mode 100644 index 00000000..d3452b95 --- /dev/null +++ b/modules/mesh-healers/cmd/mesh-healers/tools.go @@ -0,0 +1,230 @@ +package main + +// The seat's verbs and the module's own reads (novox/hq ADR 0564 §7, §12): `healers` is the registry, what the healers +// did and would do, the brake and whether they act; `heal` takes one act now on a condition, within budget and brake; +// `drill` waits for the hand-act log's home; and a read of each bucket it keeps, so a recovery is a call. + +import ( + "encoding/json" + "errors" + "fmt" + "sort" + "strings" + "time" + + stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/handacts" +) + +// Verbs are the seat's verbs, as the manifest claims them. +var Verbs = []string{"healers", "heal", "drill"} + +// Tool names. +const ( + HealersTool = Seat + ".healers" + HealTool = Seat + ".heal" + DrillTool = Seat + ".drill" + FindingsTool = "healers_findings" + WouldTool = "healers_would" + WakesTool = "healers_wakes" +) + +func obj(props map[string]any) map[string]any { + return map[string]any{"type": "object", "properties": props} +} + +// Tools are what the runtime serves. +func Tools(h *Healers) []stdio.Tool { + return []stdio.Tool{ + {Name: HealersTool, + Description: "The mesh's healers (novox/hq ADR 0564 §7, §8): the registry, every act and escalation of the last " + + "days (the bucket attempts), whether the mesh-wide brake holds, whether they act or only write what they " + + "would do while the controller holds its lease, and the would-acts. recount=true counts the hand-act log " + + "again for S15 (a cause repaired by hand twice in a fortnight).", + Input: obj(map[string]any{ + "days": map[string]any{"type": "number", "description": "how many days of acts back (default 7)"}, + "recount": map[string]any{"type": "boolean", "description": "count the hand-act log again now"}}), + Run: func(a map[string]any) (any, error) { return h.Answer(intArg(a, "days", 7), boolArg(a, "recount")) }}, + {Name: HealTool, + Description: "One healer's act now on an open condition, by its key, within its budget and the brake, as an " + + "event would have it — or, while the controller holds its lease, the act it would take written as would. " + + "Says what was done.", + Input: obj(map[string]any{ + "key": map[string]any{"type": "string", "description": "the condition's key"}, + "why": map[string]any{"type": "string", "description": "why, in a sentence"}}), + Run: func(a map[string]any) (any, error) { + return h.Heal(strArg(a, "key"), strArg(a, "why"), stdio.CallerOf(a)) + }}, + {Name: DrillTool, + Description: "Record a drill: something broken on purpose to see the mesh raise and clear it, with what it " + + "tests, in the hand-act log of the seat mesh-hand-acts (the controller's drill while that seat has no " + + "holder and the controller holds its lease). A drill is never a repair: S15 never counts it.", + Input: obj(map[string]any{"what": map[string]any{"type": "string", "description": "what was done on purpose"}, + "why": map[string]any{"type": "string", "description": "what the drill tests"}, + "condition": map[string]any{"type": "string", "description": "the condition it is meant to raise, if any"}}), + Run: func(a map[string]any) (any, error) { + return h.Drill(strArg(a, "what"), strArg(a, "why"), strArg(a, "condition")) + }}, + {Name: FindingsTool, + Description: "What the healers find of themselves (their bucket findings): the brake, their blindness, and " + + "each cause repaired by hand twice in a fortnight (S15); or one whole by key.", + Input: obj(map[string]any{"key": map[string]any{"type": "string"}}), + Run: func(a map[string]any) (any, error) { return h.read(FindingsState, strArg(a, "key")) }}, + {Name: WakesTool, + Description: "What the healers wait for (their bucket wakes): each settle still running (`settle.`) " + + "and S15's fortnight end (`fortnight`), each a key whose time to live is the wait and whose expiry wakes them; " + + "or one whole by key. heal acts on a condition now, and healers with recount=true counts S15 now.", + Input: obj(map[string]any{"key": map[string]any{"type": "string"}}), + Run: func(a map[string]any) (any, error) { return h.read(WakesState, strArg(a, "key")) }}, + {Name: WouldTool, + Description: "What the healers would have done while the controller holds its lease (their bucket would, each " + + "key kept a week): one per healer and condition, its newest; or one whole by key.", + Input: obj(map[string]any{"key": map[string]any{"type": "string"}}), + Run: func(a map[string]any) (any, error) { return h.read(WouldState, strArg(a, "key")) }}, + } +} + +// Drill records a drill (ADR 0565 §11, which ends ADR 0564 Limits 1's refusal): through the seat mesh-hand-acts as +// the act `hand-act drill`, and while that seat has no holder that answers and the controller still holds its lease, +// through the controller's own `drill`, which records it as a drill too. It answers which log took it. +func (h *Healers) Drill(what, why, condition string) (any, error) { + if what == "" || why == "" { + return nil, errors.New("a drill says what was done on purpose and what it tests: nothing was recorded") + } + if h.Ask == nil { + return nil, errors.New("the hand-act log cannot be reached from here: nothing was recorded") + } + body := map[string]any{"what": what, "why": why, "verb": handacts.VerbDrill} + controller := map[string]any{"what": what, "why": why} + if condition != "" { + body["condition"], controller["condition"] = condition, condition + } + answer, err := h.Ask("seat:"+handacts.Seat+".hand-act", body) + if err == nil { + return map[string]any{"log": handacts.LogSeat, "answer": answer}, nil + } + if !handacts.NoResponder(err) { + return nil, fmt.Errorf("%s.hand-act refused the drill: %w", handacts.Seat, err) + } + if acting, _ := h.Acting(); acting { + return nil, fmt.Errorf("%s has no holder that answers, and the controller handed its lease away: nothing was "+ + "recorded (%w)", handacts.Seat, err) + } + answer, cerr := h.Ask("seat:mesh-controller.drill", controller) + if cerr != nil { + return nil, fmt.Errorf("%s has no holder that answers (%v), and the controller's drill refused too: %w", handacts.Seat, + err, cerr) + } + return map[string]any{"log": handacts.LogController, "answer": answer}, nil +} + +// Heal is the verb heal: one condition judged now by its healer. +func (h *Healers) Heal(key, why, caller string) (any, error) { + if key == "" || why == "" { + return nil, errors.New("heal names the condition's key and says why") + } + raw, found, err := h.Store.Get(ConditionsRead, key) + if err != nil { + return nil, fmt.Errorf("the open conditions cannot be read: %w", err) + } + if !found { + return nil, fmt.Errorf("no condition is open under %s", key) + } + said, err := h.Judge(conditionOf(raw), caller+" ("+why+")") + if err != nil { + return nil, err + } + return map[string]any{"said": said}, nil +} + +// Answer is the verb healers. +func (h *Healers) Answer(days int, recount bool) (any, error) { + if recount { + h.Wanted() + } + now := h.Now() + all, err := h.attempts() + if err != nil { + return nil, fmt.Errorf("what the healers did cannot be read: %w", err) + } + var recent []Attempt + for _, a := range all { + if now.Sub(a.Begun) <= time.Duration(days)*24*time.Hour { + recent = append(recent, a) + } + } + sort.SliceStable(recent, func(i, j int) bool { return recent[i].Begun.After(recent[j].Begun) }) + braked, said := h.brakeHolds(actsWithin(all, now.Add(-BrakeWindow)), now) + var rows []map[string]any + for _, r := range h.Registry { + rows = append(rows, map[string]any{"id": r.ID, "kinds": r.Kinds, "condition": r.Condition, "repair": r.Repair, + "budget": fmt.Sprintf("%d act(s) within %s, %s apart at least", r.Budget, r.Window, r.Settle), + "then": r.Then, "acts-in": r.ActsIn, "from": r.From}) + } + acting, why := h.Acting() + would, err := h.read(WouldState, "") + if err != nil { + would = map[string]any{"unread": err.Error()} + } + h.mu.Lock() + declined := map[string]string{} + for k, v := range h.declined { + declined[k] = v + } + h.mu.Unlock() + if recent == nil { + recent = []Attempt{} + } + return map[string]any{"healers": rows, "acts": recent, "days": days, "acting": acting, "why": why, "would": would, + "brake": map[string]any{"holds": braked, "said": said, "limit": BrakeLimit, "window": BrakeWindow.String()}, + "declined": declined, + "note": "a heal is never a hand act; whether it repaired anything is the condition's clearing to say"}, nil +} + +// read is every key of a bucket with its value, or one whole. +func (h *Healers) read(state, key string) (any, error) { + if key != "" { + raw, found, err := h.Store.Get(state, key) + if err != nil { + return nil, err + } + if !found { + return nil, fmt.Errorf("nothing is kept under %s in %s", key, state) + } + return json.RawMessage(raw), nil + } + keys, err := h.Store.Keys(state) + if err != nil { + return nil, err + } + rows := map[string]json.RawMessage{} + for _, k := range keys { + if raw, found, err := h.Store.Get(state, k); err == nil && found { + rows[k] = raw + } + } + return map[string]any{state: rows, "count": len(rows)}, nil +} + +func strArg(a map[string]any, k string) string { + s, _ := a[k].(string) + return strings.TrimSpace(s) +} + +// boolArg reads a flag as a boolean or as "true". +func boolArg(a map[string]any, k string) bool { + switch v := a[k].(type) { + case bool: + return v + case string: + return strings.EqualFold(strings.TrimSpace(v), "true") + } + return false +} + +func intArg(a map[string]any, k string, otherwise int) int { + if n, ok := a[k].(float64); ok && n > 0 { + return int(n) + } + return otherwise +} diff --git a/modules/mesh-healers/cmd/mesh-healers/wanted.go b/modules/mesh-healers/cmd/mesh-healers/wanted.go new file mode 100644 index 00000000..065f2c41 --- /dev/null +++ b/modules/mesh-healers/cmd/mesh-healers/wanted.go @@ -0,0 +1,166 @@ +package main + +// S15: a cause repaired by hand twice within a fortnight is a healer wanted (ADR 0231 §27, §28; ADR 0564 Limits 2). +// The hand-act log is the controller's bucket mesh-controller_hand-acts, a key-value bucket with no events: mesh-healers +// watches it, the one of the four holders granted its read, counts each cause again on every put, and writes the +// finding `healer-wanted` per cause. **A heal is never a hand act**, and an act that is a person's decision by design +// (a drill above all) never counts. The finding clears when fewer than two acts of its cause remain within the +// fortnight: the key `fortnight` of the healers' wakes expires when the oldest counted act leaves it. From the log's home (issue 1953) they +// watch that instead. + +import ( + "encoding/json" + "fmt" + "slices" + "sort" + "strings" + "time" + + "git.novox.be/novox/mesh-sdk/go/handacts" +) + +// Within is how far back a repeated cause counts. +const Within = 14 * 24 * time.Hour + +// HandAct is what S15 reads of one entry of the controller's hand-act log (mesh-controller internal/link HandAct). +type HandAct struct { + ID string `json:"id"` + At time.Time `json:"at"` + By string `json:"by"` + Verb string `json:"verb"` + Args []string `json:"arguments,omitempty"` + Why string `json:"why"` + Cause string `json:"cause"` + Kind string `json:"kind,omitempty"` +} + +// decided are the verbs whose entries are a person's decision by design, with the causes they hold for (every cause +// when none): the controller's handActVerbs, read as S15 reads them. A verb not listed counts as a repair. +var decided = map[string][]string{ + "warrant": nil, + "seat": {"seat-passed"}, + "conditions silence": {"operator-answer"}, + "hand-act record": {"drill"}, + "secret issued": nil, + "hand-act drill": nil, + "retire approve": nil, + "retire reject": nil, + "cleanup delete": nil, + "dead-letters deliver": nil, + "dead-letters drop": nil, + "collect": nil, + "mirrors": nil, + "move-placement": nil, + "move-config": nil, + "bus upgrade": nil, + "upgrade release-backlog": nil, + "secret accept": nil, + "secret rotate": {"leaked-in-logs"}, + "rotate": {"leaked-in-logs"}, +} + +// personsDecision is whether an act is a person's decision by design, by the verb that recorded it. +func personsDecision(a HandAct) bool { + causes, ok := decided[a.Verb] + if !ok { + return false + } + return len(causes) == 0 || slices.Contains(causes, a.Cause) +} + +// ActRecorded is one act the seat mesh-hand-acts recorded, heard as act-recorded: counted beside the controller's +// log's, under a key of its own log, so an act is counted once whichever log holds it (ADR 0565 §11). +func (h *Healers) ActRecorded(body []byte) { + var a HandAct + if json.Unmarshal(body, &a) != nil || a.ID == "" { + return + } + h.HandActHeard(handacts.LogSeat+":"+a.ID, "put", body) +} + +// HandActHeard is one change of the hand-act log: kept or forgotten, and every cause counted again. +func (h *Healers) HandActHeard(key, op string, value []byte) { + h.mu.Lock() + if h.acts == nil { + h.acts = map[string]HandAct{} + } + if op != "put" { + delete(h.acts, key) + } else { + var a HandAct + if json.Unmarshal(value, &a) == nil { + h.acts[key] = a + } + } + h.mu.Unlock() + h.Wanted() +} + +// Wanted counts every cause repaired by hand within the fortnight and writes a finding for each counted twice or more, +// deleting the rest; its wait `fortnight` expires when the oldest counted act of a wanted cause leaves the fortnight. +func (h *Healers) Wanted() []Observation { + h.mu.Lock() + defer h.mu.Unlock() + now := h.Now() + byCause := map[string][]HandAct{} + for _, a := range h.acts { + if personsDecision(a) || now.Sub(a.At) > Within { + continue + } + byCause[a.Cause] = append(byCause[a.Cause], a) + } + var causes []string + for c, acts := range byCause { + if len(acts) >= 2 { + causes = append(causes, c) + } + } + sort.Strings(causes) + var found []Observation + var next time.Time + for _, cause := range causes { + acts := byCause[cause] + sort.Slice(acts, func(i, j int) bool { return acts[i].At.Before(acts[j].At) }) + var lines []string + for _, a := range acts { + lines = append(lines, fmt.Sprintf("%s %s by %s: %s", a.At.UTC().Format("2006-01-02 15:04"), + strings.TrimSpace(a.Verb+" "+strings.Join(a.Args, " ")), a.By, a.Why)) + } + newest := acts[len(acts)-1] + wanted := "a healer is wanted for it" + for _, r := range h.Registry { + if slices.Contains(r.Kinds, cause) { + wanted = fmt.Sprintf("healer %s answers this cause and a person still repaired it: its reach or its budget "+ + "is not enough", r.ID) + } + } + words := strings.NewReplacer(".", " ", "_", " ").Replace(cause) + found = append(found, Observation{Kind: "healer-wanted", Token: "healer-wanted", Severity: Warning, + Subject: Subject{Scope: ScopeMesh, ID: "hand-acts." + cause}, + Summary: fmt.Sprintf("%q was repaired by hand %d times in %d days, the last by %s: %s", cause, len(acts), + int(Within.Hours()/24), newest.By, wanted), + Said: strings.Join(lines, "; "), + Headline: capital(words) + " keeps being fixed by hand", + Explanation: fmt.Sprintf("A person repaired %s by hand %d times in %d days, so an automatic repair is wanted "+ + "for it. Nothing is broken now.", words, len(acts), int(Within.Hours()/24)), + Resolved: "Resolved: no more hand repairs of " + words}) + if leaves := acts[0].At.Add(Within); next.IsZero() || leaves.Before(next) { + next = leaves + } + } + h.see("S15", found) + if !next.IsZero() { + ttl := next.Sub(now).Round(time.Second) + time.Second + if err := h.Store.PutTTL(WakesState, fortnightKey, map[string]any{"at": next.UTC()}, ttl); err != nil { + h.Logf("[%s] the fortnight's end is not written as a wait, so it is counted at the next act: %v", Module, err) + } + } + return found +} + +func capital(s string) string { + if s == "" { + return s + } + return strings.ToUpper(s[:1]) + s[1:] +} diff --git a/modules/mesh-healers/cmd/mesh-healers/whole_integration_test.go b/modules/mesh-healers/cmd/mesh-healers/whole_integration_test.go new file mode 100644 index 00000000..db056254 --- /dev/null +++ b/modules/mesh-healers/cmd/mesh-healers/whole_integration_test.go @@ -0,0 +1,423 @@ +//go:build integration + +package main + +// mesh-healers whole against a real bus (novox/hq ADR 0564 §7, §8; issue 1952 H3): the built binary, played over its +// real stdin and stdout as the node's tool runtime plays it (mesh/state.* and watches answered from a real nats-server's +// key-value buckets, the keeper's events handed over as the runtime hands them, the grants part's assert answered). An +// induced failure, a module's consumer gone, is only written as would while the controller holds its lease; from the +// write that hands the lease away a healer acts, its attempt on the bus and healer-acted said; past twelve acts in the +// hour the brake stops every healer, said as an urgent finding on the bus. + +import ( + "bufio" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" + + "git.novox.be/novox/mesh-sdk/go/health" + "git.novox.be/novox/mesh-sdk/go/testbus" +) + +type wire struct { + ID json.RawMessage `json:"id,omitempty"` + Method string `json:"method,omitempty"` + Params json.RawMessage `json:"params,omitempty"` + Result json.RawMessage `json:"result,omitempty"` + Error *struct { + Message string `json:"message"` + } `json:"error,omitempty"` +} + +// player is the node's tool runtime, as far as the self-check needs it, over a real bus. +type player struct { + ctx context.Context + nc *nats.Conn + js jetstream.JetStream + mu sync.Mutex + in io.Writer + next int + pending map[string]chan wire + asked []string +} + +func (p *player) send(m map[string]any) { + raw, _ := json.Marshal(m) + p.mu.Lock() + defer p.mu.Unlock() + _, _ = p.in.Write(append(raw, '\n')) +} + +func (p *player) answer(id json.RawMessage, result any, err error) { + if err != nil { + p.send(map[string]any{"id": id, "error": map[string]any{"code": -32000, "message": err.Error()}}) + return + } + p.send(map[string]any{"id": id, "result": result}) +} + +func (p *player) serve(out io.Reader) { + scanner := bufio.NewScanner(out) + scanner.Buffer(make([]byte, 1<<20), 1<<24) + for scanner.Scan() { + var m wire + if json.Unmarshal(scanner.Bytes(), &m) != nil { + continue + } + if m.Method == "" { + var key string + _ = json.Unmarshal(m.ID, &key) + p.mu.Lock() + ch := p.pending[key] + delete(p.pending, key) + p.mu.Unlock() + if ch != nil { + ch <- m + } + continue + } + go p.handle(m) + } +} + +// kv is the bucket a state names: this module's own by its local name, another's as `.`. +func (p *player) kv(state string) (jetstream.KeyValue, error) { + if owner, name, ok := strings.Cut(state, "."); ok { + return p.js.KeyValue(p.ctx, owner+"_"+name) + } + return p.js.KeyValue(p.ctx, Module+"_"+state) +} + +// deliver hands the bundle one event as the runtime does from its consumer. +func (p *player) deliver(key string, body []byte) { + p.mu.Lock() + p.next++ + id := fmt.Sprintf("e-%d", p.next) + p.mu.Unlock() + p.send(map[string]any{"id": id, "method": "mesh/event", "params": map[string]any{"envelope": map[string]any{ + "key": key, "node": "ace", "body": json.RawMessage(body)}}}) +} + +func (p *player) handle(m wire) { + var a struct { + State string `json:"state"` + Key string `json:"key"` + Value json.RawMessage `json:"value"` + Body json.RawMessage `json:"body"` + } + _ = json.Unmarshal(m.Params, &a) + switch m.Method { + case "mesh/state.get": + kv, err := p.kv(a.State) + if err != nil { + p.answer(m.ID, nil, err) + return + } + e, err := kv.Get(p.ctx, a.Key) + if errors.Is(err, jetstream.ErrKeyNotFound) { + p.answer(m.ID, nil, nil) + return + } + if err != nil { + p.answer(m.ID, nil, err) + return + } + p.answer(m.ID, map[string]any{"key": e.Key(), "value": json.RawMessage(e.Value()), "revision": e.Revision()}, nil) + case "mesh/state.put": + kv, err := p.kv(a.State) + if err == nil { + var rev uint64 + if rev, err = kv.Put(p.ctx, a.Key, a.Value); err == nil { + p.answer(m.ID, map[string]any{"revision": rev}, nil) + return + } + } + p.answer(m.ID, nil, err) + case "mesh/state.create": + kv, err := p.kv(a.State) + if err == nil { + var rev uint64 + if rev, err = kv.Create(p.ctx, a.Key, a.Value); err == nil { + p.answer(m.ID, map[string]any{"revision": rev}, nil) + return + } + if errors.Is(err, jetstream.ErrKeyExists) { + p.send(map[string]any{"id": m.ID, "error": map[string]any{"code": -32001, "message": "the key was written by another writer first"}}) + return + } + } + p.answer(m.ID, nil, err) + case "mesh/state.watch": + kv, err := p.kv(a.State) + if err != nil { + p.answer(m.ID, nil, err) + return + } + key := a.Key + if key == "" { + key = ">" + } + w, err := kv.Watch(p.ctx, key) + if err != nil { + p.answer(m.ID, nil, err) + return + } + var watch string + _ = json.Unmarshal(m.ID, &watch) + current := true + for e := range w.Updates() { + if e == nil { + current = false + p.answer(m.ID, map[string]any{}, nil) + continue + } + op := "put" + if e.Operation() != jetstream.KeyValuePut { + op = "delete" + } + change := map[string]any{"state": a.State, "key": e.Key(), "op": op, "revision": e.Revision(), "current": current} + if op == "put" { + change["value"] = json.RawMessage(e.Value()) + } + p.mu.Lock() + p.next++ + id := fmt.Sprintf("w-%d", p.next) + p.mu.Unlock() + p.send(map[string]any{"id": id, "method": "mesh/state", "params": map[string]any{"watch": watch, "change": change}}) + } + case "mesh/subscribe": + p.answer(m.ID, map[string]any{}, nil) + case "mesh/state.keys": + kv, err := p.kv(a.State) + if err != nil { + p.answer(m.ID, nil, err) + return + } + keys, err := kv.Keys(p.ctx) + if errors.Is(err, jetstream.ErrNoKeysFound) { + keys, err = []string{}, nil + } + p.answer(m.ID, keys, err) + case "mesh/state.delete": + kv, err := p.kv(a.State) + if err == nil { + err = kv.Delete(p.ctx, a.Key) + } + p.answer(m.ID, map[string]any{}, err) + case "mesh/seat.publish": + var e struct { + Subject string `json:"subject"` + Body json.RawMessage `json:"body"` + } + _ = json.Unmarshal(m.Params, &e) + p.answer(m.ID, map[string]any{}, p.nc.Publish(e.Subject, e.Body)) + case "mesh/publish": + var e struct { + Key string `json:"key"` + Body json.RawMessage `json:"body"` + } + _ = json.Unmarshal(m.Params, &e) + p.answer(m.ID, map[string]any{}, p.nc.Publish("mesh.mod."+Module+".event."+e.Key, e.Body)) + case "mesh/ask": + var ask struct { + Key string `json:"key"` + Body json.RawMessage `json:"body"` + } + _ = json.Unmarshal(m.Params, &ask) + p.mu.Lock() + p.asked = append(p.asked, ask.Key+" "+string(ask.Body)) + p.mu.Unlock() + if ask.Key != AssertTool { + p.answer(m.ID, nil, errors.New("nats: no responders available for request")) + return + } + p.answer(m.ID, map[string]any{"says": "1 the bus lacks: created at this ask"}, nil) + default: + p.answer(m.ID, nil, errors.New("nats: no responders available for request")) + } +} + +func (p *player) call(name string, args map[string]any) (json.RawMessage, error) { + p.mu.Lock() + p.next++ + id := fmt.Sprintf("c-%d", p.next) + ch := make(chan wire, 1) + p.pending[id] = ch + p.mu.Unlock() + p.send(map[string]any{"id": id, "method": "tools/call", "params": map[string]any{"name": name, "arguments": args, + "_meta": map[string]any{"mesh/caller": "person.jochen"}}}) + select { + case m := <-ch: + if m.Error != nil { + return nil, errors.New(m.Error.Message) + } + var r struct { + Content []struct { + Text string `json:"text"` + } `json:"content"` + IsError bool `json:"isError"` + } + if err := json.Unmarshal(m.Result, &r); err != nil || len(r.Content) != 1 { + return nil, fmt.Errorf("%s answered %s", name, m.Result) + } + if r.IsError { + return nil, errors.New(r.Content[0].Text) + } + return json.RawMessage(r.Content[0].Text), nil + case <-time.After(60 * time.Second): + return nil, fmt.Errorf("%s did not answer in 60s", name) + } +} + +func eventually(t *testing.T, what string, ok func() bool) { + t.Helper() + for end := time.Now().Add(30 * time.Second); time.Now().Before(end); time.Sleep(50 * time.Millisecond) { + if ok() { + return + } + } + t.Fatalf("never: %s", what) +} + +func raised(key, module string) []byte { + raw, _ := json.Marshal(map[string]any{"key": key, "kind": "consumer-lost", "severity": "warning", "resolver": "self", + "subject": map[string]any{"scope": "bus", "id": "EVENTS." + module}, "event": "condition-raised", + "show": health.ShowKeeper(key)}) + return raw +} + +// 1952/H3: an induced failure; the healer acts only from the lease write, and brakes. +func TestTheHealersWholeActOnTheBusAndBrake(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + dir := t.TempDir() + bin := filepath.Join(dir, Module) + build := exec.Command("go", "build", "-o", bin, ".") + build.Stderr = os.Stderr + if err := build.Run(); err != nil { + t.Fatalf("mesh-healers does not build: %v", err) + } + nc, err := nats.Connect(testbus.URL(t)) + if err != nil { + t.Fatal(err) + } + t.Cleanup(nc.Close) + js, err := jetstream.New(nc) + if err != nil { + t.Fatal(err) + } + for _, b := range []string{Module + "_" + AttemptsState, Module + "_" + WouldState, Module + "_" + FindingsState, Module + "_" + WakesState, + "mesh-controller_lease", "mesh-conditions_conditions", "mesh-controller_hand-acts"} { + if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: b}); err != nil { + t.Fatal(err) + } + } + bucket := func(name string) jetstream.KeyValue { + kv, err := js.KeyValue(ctx, name) + if err != nil { + t.Fatal(err) + } + return kv + } + keys := func(name string) []string { + k, _ := bucket(name).Keys(ctx) + return k + } + lease := bucket("mesh-controller_lease") + if _, err := lease.Put(ctx, LeaseKey, []byte(`{"instance":"novox/1/2026-10-27","epoch":57}`)); err != nil { + t.Fatal(err) + } + acted, err := nc.SubscribeSync("mesh.seat." + Seat + ".event." + ActedEvent) + if err != nil { + t.Fatal(err) + } + cmd := exec.CommandContext(ctx, bin) + cmd.Env = append(os.Environ(), "MESH_NODE=ace", "MESH_MODULE="+Module) + stdin, _ := cmd.StdinPipe() + stdout, _ := cmd.StdoutPipe() + cmd.Stderr = os.Stderr + if err := cmd.Start(); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = stdin.Close(); cancel(); _ = cmd.Wait() }) + p := &player{ctx: ctx, nc: nc, js: js, in: stdin, pending: map[string]chan wire{}} + go p.serve(stdout) + healers := func() map[string]any { + raw, err := p.call(HealersTool, map[string]any{}) + if err != nil { + return nil + } + var a map[string]any + _ = json.Unmarshal(raw, &a) + return a + } + eventually(t, "the lease read as a controller's", func() bool { + a := healers() + return a != nil && strings.Contains(fmt.Sprint(a["why"]), "names a controller") + }) + + // The controller holds the lease: the induced failure is written as would, never acted on. + p.deliver(RaisedEvent, raised("bus.EVENTS.records.missing", "records")) + eventually(t, "the would-act written", func() bool { return len(keys(Module+"_"+WouldState)) == 1 }) + if k := keys(Module + "_" + AttemptsState); len(k) != 0 { + t.Fatalf("acted under the controller's lease: %v", k) + } + // The write that hands the lease away: the same failure is acted on. + if _, err := lease.Put(ctx, LeaseKey, []byte(`{"handed-to":"the operator's hand-over"}`)); err != nil { + t.Fatal(err) + } + eventually(t, "the healers acting", func() bool { return healers()["acting"] == true }) + p.deliver(RaisedEvent, raised("bus.EVENTS.records.missing", "records")) + eventually(t, "the act begun and finished on the bus", func() bool { + k := keys(Module + "_" + AttemptsState) + if len(k) != 1 { + return false + } + e, err := bucket(Module+"_"+AttemptsState).Get(ctx, k[0]) + return err == nil && strings.Contains(string(e.Value()), `"outcome":"acted"`) + }) + msg, err := acted.NextMsg(10 * time.Second) + if err != nil || !strings.Contains(string(msg.Data), `"healer":"H3"`) { + t.Fatalf("healer-acted was not said: %v", err) + } + p.mu.Lock() + asked := strings.Join(p.asked, "\n") + p.mu.Unlock() + if !strings.Contains(asked, AssertTool+` {"module":"records"}`) { + t.Errorf("the grants part was asked %s", asked) + } + // Eleven more acts within the hour: the twelfth was the limit, and the next condition meets the brake. + attempts := bucket(Module + "_" + AttemptsState) + for i := 0; i < BrakeLimit-1; i++ { + begun := time.Now().Add(-time.Duration(i+1) * time.Minute).UTC() + raw, _ := json.Marshal(Attempt{Healer: "H3", Condition: fmt.Sprintf("bus.EVENTS.m%d.missing", i), BudgetKey: "x", + Outcome: OutcomeActed, Begun: begun}) + if _, err := attempts.Put(ctx, attemptKey("H3", fmt.Sprintf("bus.EVENTS.m%d.missing", i), begun), raw); err != nil { + t.Fatal(err) + } + } + p.deliver(RaisedEvent, raised("bus.EVENTS.mail.missing", "mail")) + eventually(t, "the brake written as a finding on the bus", func() bool { + e, err := bucket(Module+"_"+FindingsState).Get(ctx, brakeKey) + if err != nil { + return false + } + f, err := health.DecodeFinding(e.Value()) + return err == nil && f.Severity == health.Urgent && f.Kind == "healers-braked" + }) + if n := len(keys(Module + "_" + AttemptsState)); n != BrakeLimit { + t.Errorf("past the brake the attempts are %d, want %d", n, BrakeLimit) + } +} diff --git a/modules/mesh-healers/go.mod b/modules/mesh-healers/go.mod new file mode 100644 index 00000000..89b3df38 --- /dev/null +++ b/modules/mesh-healers/go.mod @@ -0,0 +1,22 @@ +module mesh-healers + +go 1.26.0 + +require ( + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 + github.com/nats-io/nats.go v1.54.0 +) + +require ( + github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op // indirect + github.com/google/go-tpm v0.9.8 // indirect + github.com/klauspost/compress v1.20.0 // indirect + github.com/minio/highwayhash v1.0.4 // indirect + github.com/nats-io/jwt/v2 v2.8.1 // indirect + github.com/nats-io/nats-server/v2 v2.11.17 // indirect + github.com/nats-io/nkeys v0.4.16 // indirect + github.com/nats-io/nuid v1.0.1 // indirect + golang.org/x/crypto v0.57.0 // indirect + golang.org/x/sys v0.48.0 // indirect + golang.org/x/time v0.15.0 // indirect +) diff --git a/modules/mesh-healers/go.sum b/modules/mesh-healers/go.sum new file mode 100644 index 00000000..a2b8e018 --- /dev/null +++ b/modules/mesh-healers/go.sum @@ -0,0 +1,27 @@ +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 h1:lKU3HjNXGiVbLW2hUgFjU/TMTUYMlNnnmZxG5T7biQA= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= +github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= +github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= +github.com/google/go-tpm v0.9.8/go.mod h1:h9jEsEECg7gtLis0upRBQU+GhYVH6jMjrFxI8u6bVUY= +github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA= +github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI= +github.com/minio/highwayhash v1.0.4 h1:asJizugGgchQod2ja9NJlGOWq4s7KsAWr5XUc9Clgl4= +github.com/minio/highwayhash v1.0.4/go.mod h1:GGYsuwP/fPD6Y9hMiXuapVvlIUEhFhMTh0rxU3ik1LQ= +github.com/nats-io/jwt/v2 v2.8.1 h1:V0xpGuD/N8Mi+fQNDynXohVvp7ZztevW5io8CUWlPmU= +github.com/nats-io/jwt/v2 v2.8.1/go.mod h1:nWnOEEiVMiKHQpnAy4eXlizVEtSfzacZ1Q43LIRavZg= +github.com/nats-io/nats-server/v2 v2.11.17 h1:GKEghcFK6A+aFx11Yf1LjgLC3txAwvyhnYzhBIQZA8I= +github.com/nats-io/nats-server/v2 v2.11.17/go.mod h1:B1sFVz4StNosQ903ak4N1G01Fl/9f8e06mXpFIE2K24= +github.com/nats-io/nats.go v1.54.0 h1:vsXoOxjHp/GmPUN+EcI7uOf/uB+iAP+kEsAFNQN0yzA= +github.com/nats-io/nats.go v1.54.0/go.mod h1:y+DZoD1oBOYfZTU681eTUiUjI0vbqYGixNVFHcjHJ0k= +github.com/nats-io/nkeys v0.4.16 h1:rd5oAuLOb8mnAycB0xleuEBNS1pVVnN0fv/FF34Eypg= +github.com/nats-io/nkeys v0.4.16/go.mod h1:llLgWoI0o4z/Q57q2R1kHfmocyhGV6VG/U18Glg1Afs= +github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= +github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= +golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M= +golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA= +golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo= +golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= +golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= +golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= diff --git a/modules/mesh-healers/module.json b/modules/mesh-healers/module.json new file mode 100644 index 00000000..634dd388 --- /dev/null +++ b/modules/mesh-healers/module.json @@ -0,0 +1,82 @@ +{ + "module": "mesh-healers", + "version": "1", + "runs-as": "mesh-healers", + "claims": [ + { + "name": "mesh-healers", + "scope": "mesh", + "serves": [ + "healers", + "heal", + "drill" + ] + } + ], + "consumes": [ + "mesh-conditions.condition-raised", + "mesh-conditions.condition-changed", + "mesh-hand-acts.act-recorded" + ], + "invokes": [ + "mesh-grants.grants_assert", + "seat:mesh-hand-acts.hand-act", + "seat:mesh-controller.drill" + ], + "reads": [ + "mesh-controller.lease", + "mesh-conditions.conditions", + "mesh-controller.hand-acts" + ], + "state": [ + { + "name": "attempts", + "key-ttl": true + }, + { + "name": "would", + "key-ttl": true + }, + "findings", + { + "name": "wakes", + "key-ttl": true + } + ], + "own-secrets": { + "broker": "${dir:state}/broker" + }, + "secrets-owner": "mesh-healers", + "resources": [ + { + "id": "account", + "type": "user", + "name": "mesh-healers", + "shell": "/usr/bin/nologin", + "home": "/var/lib/mesh-healers" + }, + { + "id": "state", + "type": "directory", + "mode": "0700", + "owner": "mesh-healers", + "place": ".", + "unknown": "program" + } + ], + "build": { + "artifacts": [ + { + "name": "tools", + "kind": "bundle", + "language": "go", + "system": "arch", + "from": "cmd/mesh-healers", + "binary": "mesh-healers", + "loads": [ + "mesh-healers" + ] + } + ] + } +} diff --git a/modules/mesh-issues/files/settings.json b/modules/mesh-issues/files/settings.json index 0231a55d..265fe2da 100644 --- a/modules/mesh-issues/files/settings.json +++ b/modules/mesh-issues/files/settings.json @@ -62,10 +62,13 @@ "mesh-catalog modules/mesh-build-node", "mesh-catalog modules/mesh-catalog", "mesh-catalog modules/mesh-cli", + "mesh-catalog modules/mesh-conditions", "mesh-catalog modules/mesh-config", "mesh-catalog modules/mesh-delivery", "mesh-catalog modules/mesh-delivery-gate", "mesh-catalog modules/mesh-files", + "mesh-catalog modules/mesh-hand-acts", + "mesh-catalog modules/mesh-healers", "mesh-catalog modules/mesh-issues", "mesh-catalog modules/mesh-knowledge", "mesh-catalog modules/mesh-mail", @@ -75,6 +78,7 @@ "mesh-catalog modules/mesh-provisioner", "mesh-catalog modules/mesh-provisioner-client", "mesh-catalog modules/mesh-reads", + "mesh-catalog modules/mesh-self-check", "mesh-catalog modules/mesh-vault", "mesh-catalog modules/mesh-watcher", "mesh-catalog modules/messenger", diff --git a/modules/mesh-issues/go.mod b/modules/mesh-issues/go.mod index 2aea2394..ea63f374 100644 --- a/modules/mesh-issues/go.mod +++ b/modules/mesh-issues/go.mod @@ -1,22 +1,22 @@ module mesh-issues -go 1.25.0 +go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261012082113-9747f3e1b1ac + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 github.com/nats-io/nats-server/v2 v2.11.17 - github.com/nats-io/nats.go v1.51.0 + github.com/nats-io/nats.go v1.54.0 ) require ( github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op // indirect github.com/google/go-tpm v0.9.8 // indirect - github.com/klauspost/compress v1.18.5 // indirect + github.com/klauspost/compress v1.20.0 // indirect github.com/minio/highwayhash v1.0.4 // indirect github.com/nats-io/jwt/v2 v2.8.1 // indirect - github.com/nats-io/nkeys v0.4.15 // indirect + github.com/nats-io/nkeys v0.4.16 // indirect github.com/nats-io/nuid v1.0.1 // indirect - golang.org/x/crypto v0.50.0 // indirect - golang.org/x/sys v0.43.0 // indirect + golang.org/x/crypto v0.57.0 // indirect + golang.org/x/sys v0.48.0 // indirect golang.org/x/time v0.15.0 // indirect ) diff --git a/modules/mesh-issues/go.sum b/modules/mesh-issues/go.sum index b16d5979..a2b8e018 100644 --- a/modules/mesh-issues/go.sum +++ b/modules/mesh-issues/go.sum @@ -1,27 +1,27 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261012082113-9747f3e1b1ac h1:jnZ/n+fFMo7VQPNNXK1gjStHIJPw3SPbzPxsf/PpZqw= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261012082113-9747f3e1b1ac/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 h1:lKU3HjNXGiVbLW2hUgFjU/TMTUYMlNnnmZxG5T7biQA= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= github.com/google/go-tpm v0.9.8/go.mod h1:h9jEsEECg7gtLis0upRBQU+GhYVH6jMjrFxI8u6bVUY= -github.com/klauspost/compress v1.18.5 h1:/h1gH5Ce+VWNLSWqPzOVn6XBO+vJbCNGvjoaGBFW2IE= -github.com/klauspost/compress v1.18.5/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= +github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA= +github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI= github.com/minio/highwayhash v1.0.4 h1:asJizugGgchQod2ja9NJlGOWq4s7KsAWr5XUc9Clgl4= github.com/minio/highwayhash v1.0.4/go.mod h1:GGYsuwP/fPD6Y9hMiXuapVvlIUEhFhMTh0rxU3ik1LQ= github.com/nats-io/jwt/v2 v2.8.1 h1:V0xpGuD/N8Mi+fQNDynXohVvp7ZztevW5io8CUWlPmU= github.com/nats-io/jwt/v2 v2.8.1/go.mod h1:nWnOEEiVMiKHQpnAy4eXlizVEtSfzacZ1Q43LIRavZg= github.com/nats-io/nats-server/v2 v2.11.17 h1:GKEghcFK6A+aFx11Yf1LjgLC3txAwvyhnYzhBIQZA8I= github.com/nats-io/nats-server/v2 v2.11.17/go.mod h1:B1sFVz4StNosQ903ak4N1G01Fl/9f8e06mXpFIE2K24= -github.com/nats-io/nats.go v1.51.0 h1:ByW84XTz6W03GSSsygsZcA+xgKK8vPGaa/FCAAEHnAI= -github.com/nats-io/nats.go v1.51.0/go.mod h1:26HypzazeOkyO3/mqd1zZd53STJN0EjCYF9Uy2ZOBno= -github.com/nats-io/nkeys v0.4.15 h1:JACV5jRVO9V856KOapQ7x+EY8Jo3qw1vJt/9Jpwzkk4= -github.com/nats-io/nkeys v0.4.15/go.mod h1:CpMchTXC9fxA5zrMo4KpySxNjiDVvr8ANOSZdiNfUrs= +github.com/nats-io/nats.go v1.54.0 h1:vsXoOxjHp/GmPUN+EcI7uOf/uB+iAP+kEsAFNQN0yzA= +github.com/nats-io/nats.go v1.54.0/go.mod h1:y+DZoD1oBOYfZTU681eTUiUjI0vbqYGixNVFHcjHJ0k= +github.com/nats-io/nkeys v0.4.16 h1:rd5oAuLOb8mnAycB0xleuEBNS1pVVnN0fv/FF34Eypg= +github.com/nats-io/nkeys v0.4.16/go.mod h1:llLgWoI0o4z/Q57q2R1kHfmocyhGV6VG/U18Glg1Afs= github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= -golang.org/x/crypto v0.50.0 h1:zO47/JPrL6vsNkINmLoo/PH1gcxpls50DNogFvB5ZGI= -golang.org/x/crypto v0.50.0/go.mod h1:3muZ7vA7PBCE6xgPX7nkzzjiUq87kRItoJQM1Yo8S+Q= +golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M= +golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA= golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.43.0 h1:Rlag2XtaFTxp19wS8MXlJwTvoh8ArU6ezoyFsMyCTNI= -golang.org/x/sys v0.43.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo= +golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= diff --git a/modules/mesh-issues/module.json b/modules/mesh-issues/module.json index d359bda2..196c586c 100644 --- a/modules/mesh-issues/module.json +++ b/modules/mesh-issues/module.json @@ -174,10 +174,13 @@ "mesh-catalog modules/mesh-build-node", "mesh-catalog modules/mesh-catalog", "mesh-catalog modules/mesh-cli", + "mesh-catalog modules/mesh-conditions", "mesh-catalog modules/mesh-config", "mesh-catalog modules/mesh-delivery", "mesh-catalog modules/mesh-delivery-gate", "mesh-catalog modules/mesh-files", + "mesh-catalog modules/mesh-hand-acts", + "mesh-catalog modules/mesh-healers", "mesh-catalog modules/mesh-issues", "mesh-catalog modules/mesh-knowledge", "mesh-catalog modules/mesh-mail", @@ -187,6 +190,7 @@ "mesh-catalog modules/mesh-provisioner", "mesh-catalog modules/mesh-provisioner-client", "mesh-catalog modules/mesh-reads", + "mesh-catalog modules/mesh-self-check", "mesh-catalog modules/mesh-vault", "mesh-catalog modules/mesh-watcher", "mesh-catalog modules/messenger", diff --git a/modules/mesh-mail/cmd/mesh-mail/sdk_pin_test.go b/modules/mesh-mail/cmd/mesh-mail/sdk_pin_test.go index 44f0a371..75412d92 100644 --- a/modules/mesh-mail/cmd/mesh-mail/sdk_pin_test.go +++ b/modules/mesh-mail/cmd/mesh-mail/sdk_pin_test.go @@ -13,17 +13,19 @@ import ( _ "git.novox.be/novox/mesh-sdk/go/process" ) -// 1172/AC1: given mesh-mail's go.mod, when built, then it requires mesh-sdk go at -// v0.1.14-0.20261019062050-34392e74916c and go 1.26.0, and go/process is importable (the import above). +// 1172/AC1: given mesh-mail's go.mod, when built, then it requires mesh-sdk go at the release-2 commit or after it +// (v0.1.14-0.20261019062050-34392e74916c or any later pseudo-version, which the catalogue's pins move) and go 1.26.0, and +// go/process is importable (the import above). func TestGoModPinsTheSDKsProcessRelease(t *testing.T) { raw, err := os.ReadFile("../../go.mod") if err != nil { t.Fatal(err) } mod := string(raw) - const wantSDK = "git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261019062050-34392e74916c" - if !strings.Contains(mod, wantSDK) { - t.Fatalf("go.mod does not require %q:\n%s", wantSDK, mod) + const prefix, release2 = "git.novox.be/novox/mesh-sdk/go v0.1.14-0.", "20261019062050" + at := strings.Index(mod, prefix) + if at < 0 || len(mod) < at+len(prefix)+14 || mod[at+len(prefix):at+len(prefix)+14] < release2 { + t.Fatalf("go.mod does not require mesh-sdk go at release 2 (%s) or after:\n%s", release2, mod) } if !strings.Contains(mod, "go 1.26.0") { t.Fatalf("go.mod does not declare go 1.26.0:\n%s", mod) diff --git a/modules/mesh-mail/go.mod b/modules/mesh-mail/go.mod index b5826dcc..6c9fdd03 100644 --- a/modules/mesh-mail/go.mod +++ b/modules/mesh-mail/go.mod @@ -3,7 +3,7 @@ module mesh-mail go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261019062050-34392e74916c + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 github.com/emersion/go-imap/v2 v2.0.0-beta.8 github.com/emersion/go-message v0.18.2 github.com/jackc/pgx/v5 v5.11.0 diff --git a/modules/mesh-mail/go.sum b/modules/mesh-mail/go.sum index 8aa47987..d345f9ba 100644 --- a/modules/mesh-mail/go.sum +++ b/modules/mesh-mail/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261019062050-34392e74916c h1:/f6f2ugss172hDJSL3EFZqb9PdfO5DKzYVfPiX2LdAM= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261019062050-34392e74916c/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 h1:lKU3HjNXGiVbLW2hUgFjU/TMTUYMlNnnmZxG5T7biQA= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= diff --git a/modules/mesh-provisioner-client/README.md b/modules/mesh-provisioner-client/README.md index e4382024..7c62a365 100644 --- a/modules/mesh-provisioner-client/README.md +++ b/modules/mesh-provisioner-client/README.md @@ -4,19 +4,19 @@ Holds the `node-provisioner` seat on every node: the provisioner's client (novox P5). Go, one binary (`cmd/mesh-provisioner-client`), run as a service of its own under the account `mesh-provisioner-client`, outside the tool runner, on a bus account of its own, `provisioner.`, delivered as its own secret `provisioner-broker` (mesh-controller `internal/broker/provisioner.go`). The bus grants that account -its own machine's keys and, to read alone, the whole assignments mirror, where a provider's consumers on any -machine are bound to it (ADR 0373); nothing else. +its own machine's keys and, to read alone, the seat mesh-provisioner's whole `plans` and the controller's whole +assignments mirror, where a provider's consumers on any machine are bound to it (ADR 0373); nothing else. ## What it does -For each module the controller handed to the provisioner on its node — an assignment in -`mesh-controller_assignments` (`.`) whose value carries a `provisioning` plan: the templates archive +For each module placed on its node — a plan in the seat mesh-provisioner's `plans` (`mesh-provisioner_plans`, +`.`, novox/hq ADR 0570 clause 2), or, for a module the seat holds no plan of, until the controller hands +its lease away, an assignment in `mesh-controller_assignments` whose value carries a `provisioning` plan; a module +leaves only when neither places it: the templates archive by reference and digest, the files it fills (path, template, `owner-at-runtime`, how its program takes a change) and the placement values its templates name — the client: - fetches the templates archive and uses it only when it hashes to the plan's digest, keeping it by digest; -- records each placeholder as a need in `mesh-provisioner_needs`, `...` (a character a - key cannot hold written `=`); - watches each owner's keys for its node: placement in the plan, config in `mesh-config_resolved` (`..`), both named by `mesh-provisioner_registry`; - asks the vault (`mesh-vault.get`, named by the registry) for each of the module's own secrets its plan names @@ -58,7 +58,7 @@ and the placement values its templates name — the client: 4): every file of the module goes through the node-engine again, so an `owned` file changed on disk is put back; - reports `prov.applied..` and keeps the same report in `mesh-provisioner_applied` (`.`): `applied` with each file's action and digest, `waiting`, `failed` with the reason, or `removed`; -- when a module is taken off the node, removes its needs, its items and its report (the node-engine removes the files its +- when a module is taken off the node, removes its items and its report (the node-engine removes the files its declaration no longer names); and when the bus comes back, or the client starts, fills from the current values; - fills a provider's `${consumers:}` (novox/hq ADR 0373, issue 679) with one entry per consumer bound to it for that provision, in the order of the assignments' keys and, within a plan, of its `provisions` entries: a diff --git a/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/asked_again_test.go b/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/asked_again_test.go index fc5dcfae..329d4d04 100644 --- a/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/asked_again_test.go +++ b/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/asked_again_test.go @@ -123,7 +123,7 @@ func (f fakeEntry) Operation() jetstream.KeyValueOp { return f.op } // afresh, whichever way it left or returned, so it is never filled with a generation held from before; a module // that stays placed keeps using what it holds. func TestAModulePlacedAgainAsksTheVaultAfreshAndAModuleThatStaysDoesNot(t *testing.T) { - const assignment = `{"node":"g14","module":"build","provisioning":{}}` + const assignment = `{"v":1,"node":"g14","module":"build","plan":{}}` // the seat's plan (ADR 0570 clause 2) own := VaultName("g14", "build", "bus-password") pair := PairName("g14", "db", "g14", "build", "") other := VaultName("g14", "gitea", "token") @@ -133,15 +133,15 @@ func TestAModulePlacedAgainAsksTheVaultAfreshAndAModuleThatStaysDoesNot(t *testi again int // asks of the vault after the first three: the module's own secret once more, or none }{ {name: "it stays placed: the assignment is put again", again: 0, steps: func(c *Client, s *state) { - c.take(s, update{bucket: AssignmentsBucket, entry: fakeEntry{key: "g14.build", val: []byte(assignment)}}) + c.take(s, update{bucket: PlansBucket, entry: fakeEntry{key: "g14.build", val: []byte(assignment)}}) }}, {name: "its assignment is deleted and put again", again: 1, steps: func(c *Client, s *state) { - c.take(s, update{bucket: AssignmentsBucket, entry: fakeEntry{key: "g14.build", op: jetstream.KeyValueDelete}}) - c.take(s, update{bucket: AssignmentsBucket, entry: fakeEntry{key: "g14.build", val: []byte(assignment)}}) + c.take(s, update{bucket: PlansBucket, entry: fakeEntry{key: "g14.build", op: jetstream.KeyValueDelete}}) + c.take(s, update{bucket: PlansBucket, entry: fakeEntry{key: "g14.build", val: []byte(assignment)}}) }}, {name: "it left and the client was away: only the later assignment arrives", again: 1, steps: func(c *Client, s *state) { delete(s.assigned, "build") - c.take(s, update{bucket: AssignmentsBucket, entry: fakeEntry{key: "g14.build", val: []byte(assignment)}}) + c.take(s, update{bucket: PlansBucket, entry: fakeEntry{key: "g14.build", val: []byte(assignment)}}) }}, {name: "it leaves the machine", again: 1, steps: func(c *Client, s *state) { c.leave(context.Background(), nil, s, "build") @@ -152,7 +152,7 @@ func TestAModulePlacedAgainAsksTheVaultAfreshAndAModuleThatStaysDoesNot(t *testi c := &Client{Node: "g14", Secrets: fake, Now: time.Now} s := &state{secrets: map[string]heldSecret{}, plans: map[string]Plan{}, refused: map[string]string{}, assigned: map[string]bool{"build": true}, binds: map[string]binding{}, providing: map[string]bool{}, - leftOut: map[string]string{}, needs: map[string][]byte{}, applied: map[string]bool{}, + leftOut: map[string]string{}, applied: map[string]bool{}, reported: map[string][]byte{}, items: map[string]map[string]itemValue{}, recorded: map[string][]byte{}, unplaced: map[string]bool{}} n := Need{Kind: "secret", Name: "bus-password"} diff --git a/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/asks_test.go b/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/asks_test.go index 9cf10e90..844f58a6 100644 --- a/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/asks_test.go +++ b/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/asks_test.go @@ -73,10 +73,11 @@ func (r *toolRunner) hear() chan string { return heard } -// placed writes an assignment of a module the controller has not handed to the provisioner: placed, no plan. +// placed writes the plan of a module placed here with nothing to provision: an empty plan (ADR 0570: a placed +// module has a plan key, empty or not). func (b *aBus) placed(t *testing.T, module string) { t.Helper() - b.put(t, AssignmentsBucket, node+"."+module, fmt.Sprintf(`{"node":%q,"module":%q,"assigned":"2026-10-14T20:00:00Z"}`, + b.put(t, PlansBucket, node+"."+module, fmt.Sprintf(`{"node":%q,"module":%q,"provisioning":{}}`, node, module)) } @@ -119,7 +120,7 @@ func TestNatsAModulesCodeReadsItsConfigAndIsToldOfAChange(t *testing.T) { } // Taken off the machine: said, and no longer answered. - b.del(t, AssignmentsBucket, node+".claude-code") + b.del(t, PlansBucket, node+".claude-code") until(t, heard, `claude-code {"kind":"config"}`) if a := runner.askConfig("claude-code", "managed_settings"); !strings.Contains(a.Error, "not placed on g14") { t.Fatalf("a module taken off the machine was answered %+v", a) diff --git a/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/client.go b/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/client.go index beacf539..e6815a3e 100644 --- a/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/client.go +++ b/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/client.go @@ -1,18 +1,21 @@ package main import ( + "git.novox.be/novox/mesh-sdk/go/provisions" + "bytes" "context" "encoding/json" "errors" "fmt" - "git.novox.be/novox/mesh-sdk/go/delivery" "os" "slices" "sort" "strings" "time" + "git.novox.be/novox/mesh-sdk/go/delivery" + "github.com/nats-io/nats.go" "github.com/nats-io/nats.go/jetstream" @@ -39,11 +42,16 @@ import ( // The buckets the client reads and writes, and the subject it reports on. const ( - AssignmentsBucket = "mesh-controller_assignments" - ConfigBucket = "mesh-config_resolved" - RegistryBucket = "mesh-provisioner_registry" - NeedsBucket = "mesh-provisioner_needs" - AppliedBucket = "mesh-provisioner_applied" + // PlansBucket is where each module's provisioning plan on each node is read: the seat mesh-provisioner's + // `plans` (novox/hq ADR 0570 clause 2), read as an assignment's value carried it (asAssignment). + PlansBucket = provisions.PlansBucket + // ControllerAssignmentsBucket is the controller's assignments mirror, read for a module the seat holds no plan of + // until the controller hands its lease away: the holder writes plans only for modules with a want, so a module + // that runs without one (nats on novox) would otherwise lose its files' fill (review of 2026-10-30). + ControllerAssignmentsBucket = "mesh-controller_assignments" + ConfigBucket = "mesh-config_resolved" + RegistryBucket = "mesh-provisioner_registry" + AppliedBucket = "mesh-provisioner_applied" ) // AppliedSubject is where the client says what it applied for one module (ADR 0349 clauses 4 and 15). @@ -82,16 +90,6 @@ type FileApplied struct { Takes string `json:"takes,omitempty"` } -// NeedRecord is one need a module's templates name, as mesh-provisioner_needs keeps it at -// `...`: the key's name in a form the bus takes, and the name as written here. -type NeedRecord struct { - Node string `json:"node"` - Module string `json:"module"` - Kind string `json:"kind"` - Name string `json:"name"` - Files []string `json:"files"` -} - // Answer is a registry's entry, as mesh-provisioner keeps it: which domain answers a kind, and where. type Answer struct { Kind string `json:"kind"` @@ -134,12 +132,14 @@ type Client struct { type state struct { registry map[string]Answer plans map[string]Plan - refused map[string]string // a module whose plan cannot be acted on, and why - config map[string]map[string]json.RawMessage - needs map[string][]byte // key → value recorded - applied map[string]bool // module → a report is kept - reported map[string][]byte // module → the last report said, undated - secrets map[string]heldSecret // the vault's name → what this run holds of it; never written anywhere + // seatPlans and controllerAssigned are each source's value by key, as an assignment's value reads: the seat's + // plan wins, and the controller's assignment stands for a module the seat has no plan of. + seatPlans, controllerAssigned map[string][]byte + refused map[string]string // a module whose plan cannot be acted on, and why + config map[string]map[string]json.RawMessage + applied map[string]bool // module → a report is kept + reported map[string][]byte // module → the last report said, undated + secrets map[string]heldSecret // the vault's name → what this run holds of it; never written anywhere // items is every item recorded for this node, by collection and contributor: what a collector is filled from. items map[string]map[string]itemValue // recorded is each item's key and its value as kept: what this client compares before it records again. @@ -181,7 +181,7 @@ func (c *Client) Run(ctx context.Context) error { ctx, cancel := context.WithCancel(ctx) defer cancel() kvs := map[string]jetstream.KeyValue{} - for _, b := range []string{AssignmentsBucket, ConfigBucket, RegistryBucket, NeedsBucket, AppliedBucket, + for _, b := range []string{PlansBucket, ControllerAssignmentsBucket, ConfigBucket, RegistryBucket, AppliedBucket, CollectBucket} { kv, err := c.JS.KeyValue(ctx, b) if err != nil { @@ -220,12 +220,12 @@ func (c *Client) Run(ctx context.Context) error { mine := c.Node + ".>" // The registry whole, and every machine's assignment: a provider here is bound by consumers on any machine, in // their plans (ADR 0373). Every other bucket, this machine's keys alone. - for _, b := range []string{RegistryBucket, AssignmentsBucket} { + for _, b := range []string{RegistryBucket, PlansBucket, ControllerAssignmentsBucket} { if err := watch(b, ">"); err != nil { return err } } - for _, b := range []string{ConfigBucket, NeedsBucket, AppliedBucket, CollectBucket} { + for _, b := range []string{ConfigBucket, AppliedBucket, CollectBucket} { if err := watch(b, mine); err != nil { return err } @@ -307,7 +307,7 @@ func (c *Client) Run(ctx context.Context) error { defer func() { _ = asks.Unsubscribe() }() s := &state{registry: map[string]Answer{}, plans: map[string]Plan{}, refused: map[string]string{}, - config: map[string]map[string]json.RawMessage{}, needs: map[string][]byte{}, applied: map[string]bool{}, + config: map[string]map[string]json.RawMessage{}, applied: map[string]bool{}, reported: map[string][]byte{}, secrets: map[string]heldSecret{}, items: map[string]map[string]itemValue{}, recorded: map[string][]byte{}, assigned: map[string]bool{}, binds: map[string]binding{}, providing: map[string]bool{}, leftOut: map[string]string{}, bound: map[string]map[string][]byte{}, @@ -315,7 +315,7 @@ func (c *Client) Run(ctx context.Context) error { // Every current value first: nothing is filled from half of them. current := map[string]bool{} var early []*nats.Msg - for len(current) < 6 { + for len(current) < 6 { // the registry, the plans, the controller's assignments, config, applied, collect select { case <-ctx.Done(): return ctx.Err() @@ -491,7 +491,7 @@ func (c *Client) told(u update) { if module, name, ok := strings.Cut(rest, "."); ok { c.tell(module, Changed{Kind: AskConfig, Key: name}) } - case AssignmentsBucket: + case PlansBucket, ControllerAssignmentsBucket: c.tell(rest, Changed{Kind: AskConfig}) } } @@ -539,16 +539,40 @@ func (c *Client) take(s *state, u update) []string { } // A kind's answer moves what every module waits on. return sortedKeys(s.plans) - case AssignmentsBucket: + case PlansBucket, ControllerAssignmentsBucket: + if s.seatPlans == nil { + s.seatPlans, s.controllerAssigned = map[string][]byte{}, map[string][]byte{} + } + source := s.seatPlans + if u.bucket == ControllerAssignmentsBucket { + source = s.controllerAssigned + } + if gone { + delete(source, key) + } else if u.bucket == ControllerAssignmentsBucket { + source[key] = e.Value() + } else if v, err := asAssignment(e.Value()); err != nil { + logf("[provisioner-client] the plan %s cannot be read, so the controller's assignment is read: %v", key, err) + delete(source, key) + } else { + source[key] = v + } + // The seat's plan where it holds one, the controller's assignment otherwise: a module leaves only when + // neither places it, never because the seat has no plan of it yet. + value, placed := s.seatPlans[key] + if !placed { + value, placed = s.controllerAssigned[key] + } + gone = !placed // A value that is not a standing assignment (ended, or in a state: ADR 0408 clause 4) is read as a removed // key, so the module leaves the machine as it does when the key goes (issue 1098). - gone = gone || !provider.Standing(e.Value()) + gone = gone || !provider.Standing(value) // Whom it binds, whichever machine's it is: a provider here listing its consumers is filled again. var providers []string if ended, had := s.binds[key]; gone && had { c.forgetBound(s, ended) } - if takeBinds(s, key, e.Value(), gone) { + if takeBinds(s, key, value, gone) { providers = sortedKeys(s.providing) } module, mine := strings.CutPrefix(key, c.Node+".") @@ -571,7 +595,7 @@ func (c *Client) take(s *state, u update) []string { logf("[provisioner-client] %s: its assignment changed, so the vault is asked again for %s", module, strings.Join(dropped, ", ")) } - m, p, err := planOf(c.Node, key, e.Value()) + m, p, err := planOf(c.Node, key, value) switch { case errors.Is(err, errNoPlan): case err != nil: @@ -597,12 +621,6 @@ func (c *Client) take(s *state, u update) []string { s.config[module][name] = r.Value } return []string{module} - case NeedsBucket: - if gone { - delete(s.needs, key) - } else { - s.needs[key] = e.Value() - } case AppliedBucket: module := strings.TrimPrefix(key, c.Node+".") s.applied[module] = !gone @@ -652,14 +670,9 @@ func (c *Client) take(s *state, u update) []string { return nil } -// leftModules are the modules with needs or a report kept and no assignment with a plan. +// leftModules are the modules with a report or items kept and no assignment with a plan. func (c *Client) leftModules(s *state) []string { left := map[string]bool{} - for key := range s.needs { - if parts := strings.SplitN(key, ".", 3); len(parts) == 3 && !hasPlan(s, parts[1]) && s.refused[parts[1]] == "" { - left[parts[1]] = true - } - } for m, kept := range s.applied { if kept && !hasPlan(s, m) && s.refused[m] == "" { left[m] = true @@ -673,9 +686,8 @@ func (c *Client) leftModules(s *state) []string { return sortedKeys(left) } -// leave removes a module's needs and its report: it is no longer the provisioner's on this node. +// leave removes a module's report and its items: it is no longer the provisioner's on this node. func (c *Client) leave(ctx context.Context, kvs map[string]jetstream.KeyValue, s *state, module string) { - prefix := c.Node + "." + module + "." delete(s.unplaced, module) // What it held of the vault goes with it: placed again, it asks afresh (issue 1945). c.forgetHeld(s, module) @@ -685,19 +697,8 @@ func (c *Client) leave(ctx context.Context, kvs map[string]jetstream.KeyValue, s delete(s.leftOut, k) } } - removed := 0 - for _, key := range sortedKeys(s.needs) { - if strings.HasPrefix(key, prefix) { - if err := kvs[NeedsBucket].Delete(ctx, key); err != nil { - logf("[provisioner-client] the need %s could not be removed: %v", key, err) - continue - } - delete(s.needs, key) - removed++ - } - } // Its items go: an item is in its collection only while its contributor is placed here. - removed += c.forgetItems(ctx, kvs, s, module, nil) + removed := c.forgetItems(ctx, kvs, s, module, nil) if !s.applied[module] && removed == 0 && s.reported[module] == nil { return } @@ -754,7 +755,7 @@ func (c *Client) reconcile(ctx context.Context, kvs map[string]jetstream.KeyValu for _, it := range items { named(it.id(), it.text) } - c.recordNeeds(ctx, kvs, s, module, order, needs) + _ = needs // the client records no need (ADR 0570 clause 1): the seat composes them values := map[Need]string{} var failed []string @@ -930,52 +931,6 @@ func (c *Client) valueOf(ctx context.Context, s *state, module string, p Plan, n return "", "", fmt.Sprintf("%s: this client does not fill %s placeholders yet", n, n.Kind) } -// recordNeeds makes the module's needs in mesh-provisioner_needs what its templates name: each need written when it -// is new or names other files, each one no longer named removed. -func (c *Client) recordNeeds(ctx context.Context, kvs map[string]jetstream.KeyValue, s *state, module string, - order []Need, files map[Need][]string) { - want := map[string][]byte{} - for _, n := range order { - v, _ := json.Marshal(NeedRecord{Node: c.Node, Module: module, Kind: n.Kind, Name: n.Name, Files: files[n]}) - want[NeedKey(c.Node, module, n)] = v - } - for _, key := range sortedKeys(want) { - if bytes.Equal(s.needs[key], want[key]) { - continue - } - if _, err := kvs[NeedsBucket].Put(ctx, key, want[key]); err != nil { - logf("[provisioner-client] the need %s could not be recorded: %v", key, err) - continue - } - s.needs[key] = want[key] - } - prefix := c.Node + "." + module + "." - for _, key := range sortedKeys(s.needs) { - if _, still := want[key]; strings.HasPrefix(key, prefix) && !still { - if err := kvs[NeedsBucket].Delete(ctx, key); err != nil { - logf("[provisioner-client] the need %s could not be removed: %v", key, err) - continue - } - delete(s.needs, key) - } - } -} - -// NeedKey is a need's key in mesh-provisioner_needs, `...`: the name with every -// character a key cannot hold, and `=`, written `=`, so `${seat:mesh-broker:4222}` is `seat.mesh-broker=3A4222`. -func NeedKey(node, module string, n Need) string { - var b strings.Builder - for _, r := range []byte(n.Name) { - switch { - case r >= 'a' && r <= 'z', r >= 'A' && r <= 'Z', r >= '0' && r <= '9', r == '-', r == '_', r == '/', r == '.': - b.WriteByte(r) - default: - fmt.Fprintf(&b, "=%02X", r) - } - } - return node + "." + module + "." + n.Kind + "." + b.String() -} - // report says a module's report on prov.applied and keeps it in mesh-provisioner_applied, when it says something // other than the last. func (c *Client) report(ctx context.Context, kvs map[string]jetstream.KeyValue, s *state, r Report) { @@ -1151,3 +1106,21 @@ func (c *Client) forgetItems(ctx context.Context, kvs map[string]jetstream.KeyVa } return removed } + +// asAssignment is a plan of the seat mesh-provisioner as the client's rules read an assignment's value: +// `{node, module, provisioning: plan}`. +func asAssignment(raw []byte) ([]byte, error) { + if _, err := provisions.DecodePlan(raw); err != nil { + return nil, err + } + // The plan as it is kept, its bytes unchanged: the client reads it strictly (planOf). + var kept struct { + Node string `json:"node"` + Module string `json:"module"` + Plan json.RawMessage `json:"plan"` + } + if err := json.Unmarshal(raw, &kept); err != nil { + return nil, err + } + return json.Marshal(map[string]any{"node": kept.Node, "module": kept.Module, "provisioning": kept.Plan}) +} diff --git a/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/client_test.go b/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/client_test.go index 1ad81773..fb0734ee 100644 --- a/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/client_test.go +++ b/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/client_test.go @@ -81,7 +81,7 @@ func startBus(t *testing.T) *aBus { t.Fatal(err) } b := &aBus{url: s.ClientURL(), domain: js, conn: nc, kv: map[string]jetstream.KeyValue{}} - for _, name := range []string{AssignmentsBucket, ConfigBucket, RegistryBucket, NeedsBucket, AppliedBucket, + for _, name := range []string{PlansBucket, ControllerAssignmentsBucket, ConfigBucket, RegistryBucket, AppliedBucket, CollectBucket, delivery.HaveBucket} { kv, err := js.CreateKeyValue(context.Background(), jetstream.KeyValueConfig{Bucket: name}) if err != nil { @@ -98,13 +98,25 @@ func startBus(t *testing.T) *aBus { } for _, kind := range placementKinds { b.put(t, RegistryBucket, kind, `{"kind":"`+kind+`","domain":"mesh-controller","from":"bucket",`+ - `"bucket":"mesh-controller_assignments","key":"."}`) + `"bucket":"mesh-provisioner_plans","key":"."}`) } return b } func (b *aBus) put(t *testing.T, bucket, key, value string) { t.Helper() + if bucket == PlansBucket { + // The seat mesh-provisioner keeps a plan, not an assignment (ADR 0570 clause 2): a module with no plan, or an + // ended one, has no key. + raw, ok := planned(value) + if !ok { + if err := b.kv[bucket].Delete(context.Background(), key); err != nil { + t.Fatal(err) + } + return + } + value = string(raw) + } if _, err := b.kv[bucket].PutString(context.Background(), key, value); err != nil { t.Fatalf("writing %s in %s: %v", key, bucket, err) } @@ -131,7 +143,7 @@ func (b *aBus) assign(t *testing.T, module string, plan Plan) { if err != nil { t.Fatal(err) } - b.put(t, AssignmentsBucket, node+"."+module, fmt.Sprintf(`{"node":%q,"module":%q,"assigned":"2026-10-14T20:00:00Z",`+ + b.put(t, PlansBucket, node+"."+module, fmt.Sprintf(`{"node":%q,"module":%q,"assigned":"2026-10-14T20:00:00Z",`+ `"provisioning":%s}`, node, module, p)) } @@ -430,11 +442,6 @@ func TestNatsATemplateFillsFromEachOwnersKeys(t *testing.T) { if r.Templates == "" || len(r.Files) != 1 || r.Files[0].Action != "created" || r.Files[0].Takes != "reload-on" { t.Errorf("prov.applied said %+v", r) } - // The needs, one per placeholder, under the node's and the module's name. - eventually(t, "the needs are recorded", func() bool { - got := b.keys(t, NeedsBucket, node+".dunst.") - return len(got) == 2 && got[0] == "g14.dunst.config.font-size" && got[1] == "g14.dunst.machine.account-home" - }) kept, err := b.kv[AppliedBucket].Get(context.Background(), node+".dunst") if err != nil || !strings.Contains(string(kept.Value()), `"status":"applied"`) { t.Errorf("mesh-provisioner_applied keeps %v (%v)", kept, err) @@ -603,17 +610,16 @@ func TestNatsWithNothingOverTheBusItAsksForNoWriteNoSecretAndNoReport(t *testing } } -func TestNatsAModuleTakenOffTheNodeLosesItsNeedsAndItsReport(t *testing.T) { +func TestNatsAModuleTakenOffTheNodeLosesItsReport(t *testing.T) { b := startBus(t) heard := b.listen(t) e := startEngine(t, map[string]string{dunstrc: "owned"}) b.config(t, "dunst", "font-size", "13") b.assign(t, "dunst", dunst(t)) aClient(t, b, e, t.TempDir()) - eventually(t, "the needs are recorded", func() bool { return len(b.keys(t, NeedsBucket, node+".dunst.")) == 2 }) + eventually(t, "its report is kept", func() bool { return len(b.keys(t, AppliedBucket, node+".dunst")) == 1 }) - b.del(t, AssignmentsBucket, node+".dunst") - eventually(t, "its needs go", func() bool { return len(b.keys(t, NeedsBucket, node+".dunst.")) == 0 }) + b.del(t, PlansBucket, node+".dunst") eventually(t, "its report goes", func() bool { return len(b.keys(t, AppliedBucket, node+".dunst")) == 0 }) eventually(t, "prov.applied says it is removed", func() bool { r, _ := heard.last("dunst") @@ -621,6 +627,44 @@ func TestNatsAModuleTakenOffTheNodeLosesItsNeedsAndItsReport(t *testing.T) { }) } +// Review of 2026-10-30: until the controller hands its lease away, a module the seat holds no plan of is read from +// the controller's assignment (nats on novox runs without a want, so the holder writes it no plan), and a module never +// leaves because its plan is missing: only when neither source places it. +func TestNatsAModuleWithoutASeatPlanIsFilledFromTheControllersAssignmentAndLeavesOnlyWhenNeitherPlacesIt(t *testing.T) { + b := startBus(t) + heard := b.listen(t) + e := startEngine(t, map[string]string{dunstrc: "owned"}) + b.config(t, "dunst", "font-size", "13") + p, err := json.Marshal(dunst(t)) + if err != nil { + t.Fatal(err) + } + assignment := fmt.Sprintf(`{"node":%q,"module":"dunst","assigned":"2026-10-14T20:00:00Z","provisioning":%s}`, node, p) + b.put(t, ControllerAssignmentsBucket, node+".dunst", assignment) + aClient(t, b, e, t.TempDir()) + eventually(t, "dunstrc is filled from the controller's assignment", func() bool { + return strings.Contains(e.read(t, dunstrc), "Monospace 13") + }) + + // The seat's plan comes, and goes again: the module stays, filled, its report kept. + b.assign(t, "dunst", dunst(t)) + b.del(t, PlansBucket, node+".dunst") + b.config(t, "dunst", "font-size", "15") + eventually(t, "dunstrc follows a change with only the controller's assignment", func() bool { + return strings.Contains(e.read(t, dunstrc), "Monospace 15") + }) + if r, _ := heard.last("dunst"); r.Status == StatusRemoved || len(b.keys(t, AppliedBucket, node+".dunst")) != 1 { + t.Fatalf("the module left when its plan went, though the controller still places it: %+v", r) + } + + // Neither places it: it leaves. + b.del(t, ControllerAssignmentsBucket, node+".dunst") + eventually(t, "prov.applied says it is removed", func() bool { + r, _ := heard.last("dunst") + return r.Status == StatusRemoved + }) +} + func TestNatsANodeComingBackFillsFromTheCurrentValues(t *testing.T) { b := startBus(t) const zshrc = "/home/nox/.zshrc" @@ -640,14 +684,13 @@ func TestNatsANodeComingBackFillsFromTheCurrentValues(t *testing.T) { // Away: a value changes and a module leaves, and the artifact store is gone. stop() b.config(t, "dunst", "font-size", "17") - b.del(t, AssignmentsBucket, node+".zsh") + b.del(t, PlansBucket, node+".zsh") heard := b.listen(t) aClient(t, b, e, state) eventually(t, "dunstrc has the value set while it was away", func() bool { return strings.Contains(e.read(t, dunstrc), "Monospace 17") }) - eventually(t, "the module that left lost its needs", func() bool { return len(b.keys(t, NeedsBucket, node+".zsh.")) == 0 }) eventually(t, "and its report", func() bool { r, _ := heard.last("zsh") return r.Status == StatusRemoved && len(b.keys(t, AppliedBucket, node+".zsh")) == 0 @@ -662,7 +705,7 @@ func TestNatsWhatCannotBeFilledFailsWithTheReason(t *testing.T) { e := startEngine(t, map[string]string{dunstrc: "owned", "/etc/app.conf": "owned"}) aClient(t, b, e, t.TempDir()) - b.put(t, AssignmentsBucket, node+".broken", `{"node":"g14","module":"broken","provisioning":{"templates":`+ + b.put(t, PlansBucket, node+".broken", `{"node":"g14","module":"broken","provisioning":{"templates":`+ `{"source":"x","digest":"sha256:00"},"files":[],"surprise":1}}`) p := dunst(t) p.Templates.Digest = "sha256:" + strings.Repeat("0", 64) @@ -683,19 +726,6 @@ func TestNatsWhatCannotBeFilledFailsWithTheReason(t *testing.T) { } } -func TestANeedsKeyHoldsAnyName(t *testing.T) { - for n, want := range map[Need]string{ - {Kind: "config", Name: "font-size"}: "g14.dunst.config.font-size", - {Kind: "seat", Name: "mesh-broker:4222"}: "g14.dunst.seat.mesh-broker=3A4222", - {Kind: "config", Name: "a=b c"}: "g14.dunst.config.a=3Db=20c", - {Kind: "machine", Name: "account-home"}: "g14.dunst.machine.account-home", - } { - if got := NeedKey(node, "dunst", n); got != want { - t.Errorf("%v: %s, want %s", n, got, want) - } - } -} - // TestNatsResticsFilesFillFromTheBackupPlacement is the backup lines as placement (issue 794): restic's two files // are the plan's `backup:lines` and `backup:data`, each exactly the text the controller composed, trailing newline // and all, and a changed plan refills them. @@ -778,3 +808,21 @@ func TestNatsTheNodesOwnHaveMovingFillsEveryModuleAgain(t *testing.T) { t.Errorf("dunstrc was asked %d times for one change", after-before) } } + +// planned is an assignment's value as the seat mesh-provisioner keeps its plan, `{v, node, module, plan}`; false for one +// with no plan, or ended, or in a state, which has no key there. +func planned(assignment string) ([]byte, bool) { + var a struct { + Node string `json:"node"` + Module string `json:"module"` + Provisioning json.RawMessage `json:"provisioning"` + Ended json.RawMessage `json:"ended"` + State json.RawMessage `json:"state"` + } + if json.Unmarshal([]byte(assignment), &a) != nil || len(a.Provisioning) == 0 || string(a.Provisioning) == "null" || + len(a.Ended) > 0 || len(a.State) > 0 { + return nil, false + } + raw, _ := json.Marshal(map[string]any{"v": 1, "node": a.Node, "module": a.Module, "plan": a.Provisioning}) + return raw, true +} diff --git a/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/collect_test.go b/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/collect_test.go index 71c7503a..defb001a 100644 --- a/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/collect_test.go +++ b/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/collect_test.go @@ -63,18 +63,13 @@ func TestNatsZshsFileHoldsEveryPlacedContributorsItemAndLosesOneWhoseContributor r, _ := heard.last("powerlevel10k") return r.Status == StatusApplied && len(r.Items) == 1 && r.Items[0] == "shell:zsh:normal" }) - // zsh records the collections it reads as its needs, like any value. - eventually(t, "zsh's needs are recorded", func() bool { - got := b.keys(t, NeedsBucket, node+".zsh.") - return len(got) == 2 && got[0] == "g14.zsh.collected.zsh-widgets" && got[1] == "g14.zsh.shell.zsh=3Anormal" - }) // A contributor's own value changes: its item, and so zshrc, follow. b.config(t, "fzf", "binding", `"^F"`) eventually(t, "zshrc follows the item", func() bool { return strings.Contains(e.read(t, zshrc), "bindkey ^F") }) // powerlevel10k leaves the node: its item goes, and zshrc without it. - b.del(t, AssignmentsBucket, node+".powerlevel10k") + b.del(t, PlansBucket, node+".powerlevel10k") without := "# zsh\n# mesh-cli\nalias nox=mesh-cli\n# then\nbindkey ^F fzf-file-widget\n# end\n" eventually(t, "zshrc loses the item of the contributor that left", func() bool { return e.read(t, zshrc) == without }) if got := b.keys(t, CollectBucket, node+".shell=3Azsh=3Anormal.powerlevel10k"); len(got) != 0 { @@ -98,7 +93,7 @@ func TestNatsACollectorStartingWithItsNodeHoldsTheItemsAlreadyThereAndOnesRecord eventually(t, "zshrc holds mesh-cli's item", func() bool { return strings.Contains(e.read(t, zshrc), "alias nox") }) stop() - b.del(t, AssignmentsBucket, node+".mesh-cli") + b.del(t, PlansBucket, node+".mesh-cli") b.assign(t, "zsh-autosuggestions", contributor(t, "shell:zsh:normal", "source suggestions.zsh\n")) aClient(t, b, e, state) want := "# zsh\n# zsh-autosuggestions\nsource suggestions.zsh\n# then\n# end\n" diff --git a/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/consumers_test.go b/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/consumers_test.go index e48d403b..048e218b 100644 --- a/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/consumers_test.go +++ b/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/consumers_test.go @@ -35,7 +35,7 @@ func (b *aBus) bind(t *testing.T, at, module string, provisions ...PlannedProvis if err != nil { t.Fatal(err) } - b.put(t, AssignmentsBucket, at+"."+module, fmt.Sprintf(`{"node":%q,"module":%q,"assigned":"2026-10-14T20:00:00Z",`+ + b.put(t, PlansBucket, at+"."+module, fmt.Sprintf(`{"node":%q,"module":%q,"assigned":"2026-10-14T20:00:00Z",`+ `"provisioning":{"templates":{"source":"x","digest":"sha256:%s"},"files":[{"id":"c","path":"/etc/c",`+ `"template":"c","owner-at-runtime":"owned"}],"provisions":%s}}`, at, module, strings.Repeat("0", 64), ps)) } @@ -140,7 +140,7 @@ func TestNatsAProvidersConsumersAreFilledFromTheVaultAndFollowEachBindAndRemake( return sameList(consumersIn(t, e), letta, photos, gitea) }) // A consumer unbinds. - b.del(t, AssignmentsBucket, "shanks.gitea") + b.del(t, PlansBucket, "shanks.gitea") eventually(t, "a consumer that unbinds leaves the file", func() bool { return sameList(consumersIn(t, e), letta, photos) }) @@ -160,11 +160,6 @@ func TestNatsAProvidersConsumersAreFilledFromTheVaultAndFollowEachBindAndRemake( t.Errorf("a consumer's credential crossed the bus in the clear: %q", said) } } - // The need is recorded as any other, by name: never a value. - if keys := b.keys(t, NeedsBucket, node+".postgres."); len(keys) != 1 || - keys[0] != node+".postgres.consumers.postgres-database" { - t.Errorf("the provider's needs are %v", keys) - } } // A provider with no consumer bound is filled with an empty list, never left waiting. diff --git a/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/ended_test.go b/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/ended_test.go deleted file mode 100644 index 996fdab9..00000000 --- a/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/ended_test.go +++ /dev/null @@ -1,58 +0,0 @@ -package main - -import ( - "encoding/json" - "fmt" - "testing" - "time" -) - -// Issue 1098: a value of the assignments bucket with `ended` set takes the module off the node exactly as a removed -// key does, and a value with the fields the controller adds to an assignment (build, commit, rollout, unknown ones) -// is read under the same key. -func TestNatsAnEndedAssignmentTakesAModuleOffTheNodeLikeARemovedKey(t *testing.T) { - b := startBus(t) - heard := b.listen(t) - e := startEngine(t, map[string]string{dunstrc: "owned"}) - b.config(t, "dunst", "font-size", "13") - p := dunst(t) - b.assign(t, "dunst", p) - aClient(t, b, e, t.TempDir()) - eventually(t, "the needs are recorded", func() bool { return len(b.keys(t, NeedsBucket, node+".dunst.")) == 2 }) - - b.put(t, AssignmentsBucket, node+".dunst", fmt.Sprintf(`{"node":%q,"module":"dunst","assigned":"2026-10-14T20:00:00Z",`+ - `"build":"b-1","commit":"abc","rollout":"r-1","ended":"2026-10-18T01:00:00Z",`+ - `"provisioning":%s}`, node, planJSON(t, p))) - eventually(t, "its needs go", func() bool { return len(b.keys(t, NeedsBucket, node+".dunst.")) == 0 }) - eventually(t, "its report goes", func() bool { return len(b.keys(t, AppliedBucket, node+".dunst")) == 0 }) - eventually(t, "prov.applied says it is removed", func() bool { - r, _ := heard.last("dunst") - return r.Status == StatusRemoved - }) -} - -func TestNatsAnAssignmentWithTheNewFieldsIsFilledUnderTheSameKey(t *testing.T) { - b := startBus(t) - e := startEngine(t, map[string]string{dunstrc: "owned"}) - b.config(t, "dunst", "font-size", "13") - p := dunst(t) - b.assign(t, "dunst", p) - aClient(t, b, e, t.TempDir()) - eventually(t, "the needs are recorded", func() bool { return len(b.keys(t, NeedsBucket, node+".dunst.")) == 2 }) - b.put(t, AssignmentsBucket, node+".dunst", fmt.Sprintf(`{"node":%q,"module":"dunst","assigned":"2026-10-14T20:00:00Z",`+ - `"build":"b-1","artifact":{"name":"dunst","version":"1"},"commit":"abc","rollout":"r-1",`+ - `"since":"2026-10-18T00:00:00Z","ended":"","state":"","future":1,"provisioning":%s}`, node, planJSON(t, p))) - time.Sleep(300 * time.Millisecond) - if got := b.keys(t, NeedsBucket, node+".dunst."); len(got) != 2 { - t.Errorf("a value with the new fields took the module's needs away: %v", got) - } -} - -func planJSON(t *testing.T, p Plan) []byte { - t.Helper() - raw, err := json.Marshal(p) - if err != nil { - t.Fatal(err) - } - return raw -} diff --git a/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/plan.go b/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/plan.go index 8caf8968..43cc4dfd 100644 --- a/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/plan.go +++ b/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/plan.go @@ -13,16 +13,15 @@ import ( "strings" ) -// The provisioner's plan of a module on this node (novox/hq ADR 0349 clause 4, design 57 P5): placement's owner, -// the controller, carries it in the module's assignment, `.` of mesh-controller_assignments -// (mesh-controller internal/inventory Provisioning), which this client reads for its own machine alone. An -// assignment with no plan is a module the controller has not handed to the provisioner: its files are the -// node-engine's, from the declaration, and this client leaves them alone (ADR 0349 clause 17). A module only pulled -// in by another on this node, and handed, has a value of its own there, which names what requires it and no time it -// was assigned (issue 765): read and filled the same way. +// The provisioner's plan of a module on this node (novox/hq ADR 0349 clause 4, ADR 0570 clause 2): the seat +// mesh-provisioner composes it from bus state and keeps it in `plans`, `.`, which this client reads for +// its own machine (and every machine's, for a provider's consumers). asAssignment carries a plan into the Assignment +// it acts on. A module the seat holds no plan of is read from the controller's assignment until the controller hands its +// lease away; a module neither places is not placed here, and its files are the node-engine's, from the declaration. A module only pulled in by another on this +// node has a plan of its own there: read and filled the same way. -// Assignment is one value of the assignments mirror. **Lenient at the top**, so a field the controller adds to an -// assignment (design 54 §1) does not stop the client; **strict in the plan**, which is the client's to act on. +// Assignment is one plan as the client acts on it. **Lenient at the top**; **strict in the plan**, which is the +// client's to act on. type Assignment struct { Node string `json:"node"` Module string `json:"module"` diff --git a/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/pulled_in_test.go b/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/pulled_in_test.go index 23388bec..a8b32af5 100644 --- a/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/pulled_in_test.go +++ b/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/pulled_in_test.go @@ -18,7 +18,7 @@ func (b *aBus) pullIn(t *testing.T, module, requiredBy string, plan Plan) { if err != nil { t.Fatal(err) } - b.put(t, AssignmentsBucket, node+"."+module, fmt.Sprintf(`{"node":%q,"module":%q,"required-by":%q,`+ + b.put(t, PlansBucket, node+"."+module, fmt.Sprintf(`{"node":%q,"module":%q,"required-by":%q,`+ `"provisioning":%s}`, node, module, requiredBy, p)) } @@ -70,7 +70,7 @@ func TestNatsAPulledInModuleIsFilledWithItsSecretAndItsItemIsRecorded(t *testing return r.Status == StatusApplied && len(r.Files) == 2 }) // What required it lets it go: its item leaves the collection. - b.del(t, AssignmentsBucket, node+".mesh-cli") + b.del(t, PlansBucket, node+".mesh-cli") eventually(t, "zshrc loses the item of a module no longer pulled in", func() bool { return e.read(t, zshrc) == "# zsh\n# then\n# end\n" }) diff --git a/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/route_names_test.go b/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/route_names_test.go index 08ca9042..53831d21 100644 --- a/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/route_names_test.go +++ b/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/route_names_test.go @@ -2,12 +2,14 @@ package main import ( "context" + "encoding/json" "os" "strings" "sync" "testing" "git.novox.be/novox/mesh-sdk/go/provider" + "git.novox.be/novox/mesh-sdk/go/provisions" "github.com/nats-io/nats.go/jetstream" ) @@ -49,12 +51,32 @@ func (m *memBus) Delete(_ context.Context, bucket, key string) error { return nil } -// routeEntryFor is the entry route-proxy's helper writes for module's plan entry on the route provision. +// routeEntryFor is the entry route-proxy's helper writes for module's plan entry on the route provision, the plan +// as the seat mesh-provisioner keeps it (ADR 0570 clause 2). func routeEntryFor(t *testing.T, module, planEntry string) string { t.Helper() - bus := &memBus{buckets: map[string]map[string][]byte{provider.AssignmentsBucket: { - node + "." + module: []byte(`{"node":"` + node + `","module":"` + module + `","provisioning":{"provisions":[` + - planEntry + `]}}`)}}} + bus := &memBus{buckets: map[string]map[string][]byte{provisions.PlansBucket: { + node + "." + module: []byte(`{"v":1,"node":"` + node + `","module":"` + module + `","plan":{"provisions":[` + + planEntry + `]}}`)}, provisions.ProvisionsBucket: {}}} + // The key the seat composed for that entry, provided-for: asked of route-proxy (ADR 0570 clause 3). + var b struct { + Provision string `json:"provision"` + Local string `json:"local"` + } + if err := json.Unmarshal([]byte(planEntry), &b); err != nil { + t.Fatal(err) + } + pr := provisions.Provision{Provision: b.Provision, Local: b.Local, State: provisions.Asked, + Consumer: provisions.End{Node: node, Module: module}, Provider: provisions.End{Node: "novox", Module: "route-proxy"}} + key, err := pr.Key() + if err != nil { + t.Fatal(err) + } + raw, err := provisions.Encode(pr) + if err != nil { + t.Fatal(err) + } + bus.buckets[provisions.ProvisionsBucket][key] = raw p := provider.New(provider.Options{Node: "novox", Module: "route-proxy", Bus: bus, Logf: t.Logf, Offers: map[string]provider.Offer{"route": {Credential: provider.NoCredential, Serves: map[string]any{}}}}) out, err := p.Pass(context.Background()) diff --git a/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/secrets.go b/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/secrets.go index ff2a5e0b..c8c4149f 100644 --- a/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/secrets.go +++ b/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/secrets.go @@ -161,7 +161,9 @@ type changedSecret struct { } // secretOf is a module's secret, from what the run holds or from the vault: its value, or what it waits for, or -// why it cannot be filled. +// why it cannot be filled. The client fills it in this release: the node-engine of today writes a file's text as it +// is handed, and the new one fills only a provision key's reference, never a module's own secret (review of +// 2026-10-30); its fill is an addition, for provision keys, until it fills every secret. func (c *Client) secretOf(ctx context.Context, s *state, module string, p Plan, n Need) (value, wait, fail string) { name := "" if slices.ContainsFunc(p.Secrets, func(sec PlannedSecret) bool { return sec.Name == n.Name }) { diff --git a/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/secrets_test.go b/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/secrets_test.go index 1538edb5..5c48fb45 100644 --- a/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/secrets_test.go +++ b/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/secrets_test.go @@ -170,10 +170,6 @@ func TestNatsAMovedModulesFileIsFilledWithItsSecretFromTheVault(t *testing.T) { r, _ := heard.last("app") return r.Status == StatusApplied && len(r.Files) == 2 }) - eventually(t, "the needs name the secrets, never their values", func() bool { - got := b.keys(t, NeedsBucket, node+".app.") - return strings.Join(got, " ") == "g14.app.config.font-size g14.app.secret.app-token g14.app.secret.db-password" - }) if wire.heard() == 0 { t.Fatal("the sniffer heard nothing: it proves nothing") } @@ -199,7 +195,7 @@ func TestNatsASecretRotatedInTheVaultReachesItsFileWithoutANewDeclaration(t *tes b.assign(t, "app", app(t)) aClient(t, b, e, t.TempDir()) eventually(t, "app.conf is filled", func() bool { return strings.Contains(e.read(t, appConf), "token = first\n") }) - assigned, err := b.kv[AssignmentsBucket].Get(context.Background(), node+".app") + assigned, err := b.kv[PlansBucket].Get(context.Background(), node+".app") if err != nil { t.Fatal(err) } @@ -212,7 +208,7 @@ func TestNatsASecretRotatedInTheVaultReachesItsFileWithoutANewDeclaration(t *tes r, _ := heard.last("app") return r.Status == StatusApplied && r.Files[0].Action == "updated" }) - now, err := b.kv[AssignmentsBucket].Get(context.Background(), node+".app") + now, err := b.kv[PlansBucket].Get(context.Background(), node+".app") if err != nil || now.Revision() != assigned.Revision() { t.Errorf("the module's assignment was written again: revision %d, was %d", now.Revision(), assigned.Revision()) } diff --git a/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/testdata/accounts.conf b/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/testdata/accounts.conf index 23fcc2cb..783bf836 100644 --- a/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/testdata/accounts.conf +++ b/modules/mesh-provisioner-client/cmd/mesh-provisioner-client/testdata/accounts.conf @@ -11,7 +11,7 @@ accounts { users = [ { user: "domain", password: "x" } { user: "provisioner.g14", password: "x", permissions: { - publish: { allow: ["$JS.API.CONSUMER.CREATE.KV_mesh-config_resolved.*.$KV.mesh-config_resolved.g14.>", "$JS.API.CONSUMER.CREATE.KV_mesh-controller_assignments.>", "$JS.API.CONSUMER.CREATE.KV_mesh-provisioner_applied.*.$KV.mesh-provisioner_applied.g14.>", "$JS.API.CONSUMER.CREATE.KV_mesh-provisioner_collect.*.$KV.mesh-provisioner_collect.g14.>", "$JS.API.CONSUMER.CREATE.KV_mesh-provisioner_needs.*.$KV.mesh-provisioner_needs.g14.>", "$JS.API.CONSUMER.CREATE.KV_mesh-provisioner_registry.>", "$JS.API.CONSUMER.CREATE.KV_node-engine_have.*.$KV.node-engine_have.g14.node", "$JS.API.CONSUMER.DELETE.KV_node-engine_have.>", "$JS.API.DIRECT.GET.KV_node-engine_have.$KV.node-engine_have.g14.node", "$JS.API.STREAM.INFO.KV_node-engine_have", "$JS.FC.KV_node-engine_have.>", "$JS.API.CONSUMER.CREATE.KV_postgres_bound.*.$KV.postgres_bound.g14.>", "$JS.API.CONSUMER.CREATE.KV_route-adapter_bound.*.$KV.route-adapter_bound.g14.>", "$JS.API.CONSUMER.CREATE.KV_route-proxy_bound.*.$KV.route-proxy_bound.g14.>", "$JS.API.CONSUMER.CREATE.KV_node-tools_bound.*.$KV.node-tools_bound.g14.>", "$JS.API.CONSUMER.DELETE.KV_mesh-config_resolved.>", "$JS.API.CONSUMER.DELETE.KV_mesh-controller_assignments.>", "$JS.API.CONSUMER.DELETE.KV_mesh-provisioner_applied.>", "$JS.API.CONSUMER.DELETE.KV_mesh-provisioner_collect.>", "$JS.API.CONSUMER.DELETE.KV_mesh-provisioner_needs.>", "$JS.API.CONSUMER.DELETE.KV_mesh-provisioner_registry.>", "$JS.API.CONSUMER.DELETE.KV_postgres_bound.>", "$JS.API.CONSUMER.DELETE.KV_route-adapter_bound.>", "$JS.API.CONSUMER.DELETE.KV_route-proxy_bound.>", "$JS.API.CONSUMER.DELETE.KV_node-tools_bound.>", "$JS.API.DIRECT.GET.KV_mesh-config_resolved.$KV.mesh-config_resolved.g14.>", "$JS.API.DIRECT.GET.KV_mesh-controller_assignments.>", "$JS.API.DIRECT.GET.KV_mesh-provisioner_applied.$KV.mesh-provisioner_applied.g14.>", "$JS.API.DIRECT.GET.KV_mesh-provisioner_collect.$KV.mesh-provisioner_collect.g14.>", "$JS.API.DIRECT.GET.KV_mesh-provisioner_needs.$KV.mesh-provisioner_needs.g14.>", "$JS.API.DIRECT.GET.KV_mesh-provisioner_registry.>", "$JS.API.DIRECT.GET.KV_postgres_bound.$KV.postgres_bound.g14.>", "$JS.API.DIRECT.GET.KV_route-adapter_bound.$KV.route-adapter_bound.g14.>", "$JS.API.DIRECT.GET.KV_route-proxy_bound.$KV.route-proxy_bound.g14.>", "$JS.API.DIRECT.GET.KV_node-tools_bound.$KV.node-tools_bound.g14.>", "$JS.API.STREAM.INFO.KV_mesh-config_resolved", "$JS.API.STREAM.INFO.KV_mesh-controller_assignments", "$JS.API.STREAM.INFO.KV_mesh-provisioner_applied", "$JS.API.STREAM.INFO.KV_mesh-provisioner_collect", "$JS.API.STREAM.INFO.KV_mesh-provisioner_needs", "$JS.API.STREAM.INFO.KV_mesh-provisioner_registry", "$JS.API.STREAM.INFO.KV_postgres_bound", "$JS.API.STREAM.INFO.KV_route-adapter_bound", "$JS.API.STREAM.INFO.KV_route-proxy_bound", "$JS.API.STREAM.INFO.KV_node-tools_bound", "$JS.FC.KV_mesh-config_resolved.>", "$JS.FC.KV_mesh-controller_assignments.>", "$JS.FC.KV_mesh-provisioner_applied.>", "$JS.FC.KV_mesh-provisioner_collect.>", "$JS.FC.KV_mesh-provisioner_needs.>", "$JS.FC.KV_mesh-provisioner_registry.>", "$JS.FC.KV_postgres_bound.>", "$JS.FC.KV_route-adapter_bound.>", "$JS.FC.KV_route-proxy_bound.>", "$JS.FC.KV_node-tools_bound.>", "$KV.mesh-provisioner_applied.g14.>", "$KV.mesh-provisioner_collect.g14.>", "$KV.mesh-provisioner_needs.g14.>", "mesh.seat.mesh-vault.call.get.provisioner~g14", "prov.applied.g14.>", "prov.changed.g14.>"] } + publish: { allow: ["$JS.API.CONSUMER.CREATE.KV_mesh-config_resolved.*.$KV.mesh-config_resolved.g14.>", "$JS.API.CONSUMER.CREATE.KV_mesh-provisioner_plans.>", "$JS.API.CONSUMER.CREATE.KV_mesh-controller_assignments.>", "$JS.API.CONSUMER.CREATE.KV_mesh-provisioner_applied.*.$KV.mesh-provisioner_applied.g14.>", "$JS.API.CONSUMER.CREATE.KV_mesh-provisioner_collect.*.$KV.mesh-provisioner_collect.g14.>", "$JS.API.CONSUMER.CREATE.KV_mesh-provisioner_registry.>", "$JS.API.CONSUMER.CREATE.KV_node-engine_have.*.$KV.node-engine_have.g14.node", "$JS.API.CONSUMER.DELETE.KV_node-engine_have.>", "$JS.API.DIRECT.GET.KV_node-engine_have.$KV.node-engine_have.g14.node", "$JS.API.STREAM.INFO.KV_node-engine_have", "$JS.FC.KV_node-engine_have.>", "$JS.API.CONSUMER.CREATE.KV_postgres_bound.*.$KV.postgres_bound.g14.>", "$JS.API.CONSUMER.CREATE.KV_route-adapter_bound.*.$KV.route-adapter_bound.g14.>", "$JS.API.CONSUMER.CREATE.KV_route-proxy_bound.*.$KV.route-proxy_bound.g14.>", "$JS.API.CONSUMER.CREATE.KV_node-tools_bound.*.$KV.node-tools_bound.g14.>", "$JS.API.CONSUMER.DELETE.KV_mesh-config_resolved.>", "$JS.API.CONSUMER.DELETE.KV_mesh-provisioner_plans.>", "$JS.API.CONSUMER.DELETE.KV_mesh-controller_assignments.>", "$JS.API.CONSUMER.DELETE.KV_mesh-provisioner_applied.>", "$JS.API.CONSUMER.DELETE.KV_mesh-provisioner_collect.>", "$JS.API.CONSUMER.DELETE.KV_mesh-provisioner_registry.>", "$JS.API.CONSUMER.DELETE.KV_postgres_bound.>", "$JS.API.CONSUMER.DELETE.KV_route-adapter_bound.>", "$JS.API.CONSUMER.DELETE.KV_route-proxy_bound.>", "$JS.API.CONSUMER.DELETE.KV_node-tools_bound.>", "$JS.API.DIRECT.GET.KV_mesh-config_resolved.$KV.mesh-config_resolved.g14.>", "$JS.API.DIRECT.GET.KV_mesh-provisioner_plans.>", "$JS.API.DIRECT.GET.KV_mesh-controller_assignments.>", "$JS.API.DIRECT.GET.KV_mesh-provisioner_applied.$KV.mesh-provisioner_applied.g14.>", "$JS.API.DIRECT.GET.KV_mesh-provisioner_collect.$KV.mesh-provisioner_collect.g14.>", "$JS.API.DIRECT.GET.KV_mesh-provisioner_registry.>", "$JS.API.DIRECT.GET.KV_postgres_bound.$KV.postgres_bound.g14.>", "$JS.API.DIRECT.GET.KV_route-adapter_bound.$KV.route-adapter_bound.g14.>", "$JS.API.DIRECT.GET.KV_route-proxy_bound.$KV.route-proxy_bound.g14.>", "$JS.API.DIRECT.GET.KV_node-tools_bound.$KV.node-tools_bound.g14.>", "$JS.API.STREAM.INFO.KV_mesh-config_resolved", "$JS.API.STREAM.INFO.KV_mesh-provisioner_plans", "$JS.API.STREAM.INFO.KV_mesh-controller_assignments", "$JS.API.STREAM.INFO.KV_mesh-provisioner_applied", "$JS.API.STREAM.INFO.KV_mesh-provisioner_collect", "$JS.API.STREAM.INFO.KV_mesh-provisioner_registry", "$JS.API.STREAM.INFO.KV_postgres_bound", "$JS.API.STREAM.INFO.KV_route-adapter_bound", "$JS.API.STREAM.INFO.KV_route-proxy_bound", "$JS.API.STREAM.INFO.KV_node-tools_bound", "$JS.FC.KV_mesh-config_resolved.>", "$JS.FC.KV_mesh-provisioner_plans.>", "$JS.FC.KV_mesh-controller_assignments.>", "$JS.FC.KV_mesh-provisioner_applied.>", "$JS.FC.KV_mesh-provisioner_collect.>", "$JS.FC.KV_mesh-provisioner_registry.>", "$JS.FC.KV_postgres_bound.>", "$JS.FC.KV_route-adapter_bound.>", "$JS.FC.KV_route-proxy_bound.>", "$JS.FC.KV_node-tools_bound.>", "$KV.mesh-provisioner_applied.g14.>", "$KV.mesh-provisioner_collect.g14.>", "mesh.seat.mesh-vault.call.get.provisioner~g14", "prov.applied.g14.>", "prov.changed.g14.>"] } subscribe: { allow: ["_INBOX.provisioner.g14.>", "mesh.mod.mesh-vault.event.secret-changed", "prov.ask.g14"] } allow_responses: { max: 1, ttl: "1m" } } } diff --git a/modules/mesh-provisioner-client/go.mod b/modules/mesh-provisioner-client/go.mod index a0fda0b8..cff5e3a4 100644 --- a/modules/mesh-provisioner-client/go.mod +++ b/modules/mesh-provisioner-client/go.mod @@ -3,7 +3,7 @@ module mesh-provisioner-client go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024234636-10b6e202320f + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 golang.org/x/sys v0.48.0 diff --git a/modules/mesh-provisioner-client/go.sum b/modules/mesh-provisioner-client/go.sum index c03eccc2..a2b8e018 100644 --- a/modules/mesh-provisioner-client/go.sum +++ b/modules/mesh-provisioner-client/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024234636-10b6e202320f h1:lUEyNPPF3w4Q0LIgcUUD6ofkKVhCF0n7dVjMoixQmXo= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024234636-10b6e202320f/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 h1:lKU3HjNXGiVbLW2hUgFjU/TMTUYMlNnnmZxG5T7biQA= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/mesh-provisioner/.gitignore b/modules/mesh-provisioner/.gitignore new file mode 100644 index 00000000..b51e2c5c --- /dev/null +++ b/modules/mesh-provisioner/.gitignore @@ -0,0 +1,2 @@ +/mesh-provisioner +/cmd/mesh-provisioner/mesh-provisioner diff --git a/modules/mesh-provisioner/cmd/mesh-provisioner/holder.go b/modules/mesh-provisioner/cmd/mesh-provisioner/holder.go new file mode 100644 index 00000000..c8a8b357 --- /dev/null +++ b/modules/mesh-provisioner/cmd/mesh-provisioner/holder.go @@ -0,0 +1,908 @@ +package main + +// The seat's holder (novox/hq ADR 0570 clauses 1 to 5, 7; issue 1954 H1-H3): mesh-provisioner composes every placed +// module's provisioning plan and every provision from bus state, on change events and never on a timer, and keeps them +// in its states `plans` (`.`) and `provisions` (`..[.]`), which it alone +// writes. +// +// - **Composed per node.** A change of a node's placement or of one of its wants composes that node; a change of the +// indexer's module composes the nodes the module is placed on. Each node is composed twice: at the builds its wants +// name (its plans, what its client applies) and at the newest build of each placed module, so a build that adds a +// requirement has its key before its want (clause 2). A provision is needed while either needs it (clause 5). +// - **A key's state comes from its provider's outcome alone** (clause 3, 7): the provider's own entry for that +// consumer, `_bound` under the same key: present is provided (the vault's generation carried into the +// reference, clause 6), refused is failed with why, waits is waits with what. The holder never sets provided by +// itself: a key of a provider that writes no outcome stays asked. +// - **Retiring is asked** (clause 5): a key no longer needed is `retiring`, one approve ask under mesh-provisioner's +// name says what goes from which provider on which node; the warrant goes into the key and the provider acts on it; +// a decline is kept and not asked again while nothing changes; a key needed again returns to its outcome's state. + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "slices" + "sort" + "strings" + "sync" + "time" + + "git.novox.be/novox/mesh-sdk/go/asks" + "git.novox.be/novox/mesh-sdk/go/catalogue" + "git.novox.be/novox/mesh-sdk/go/compose/frombus" + "git.novox.be/novox/mesh-sdk/go/compose/fromruntime" + "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/provisions" +) + +// The states it keeps, beside Registry, Needs, Applied and Collect. +const ( + ProvisionsState = provisions.ProvisionsState + PlansState = provisions.PlansState +) + +// The reads it composes from. +const ( + PlacementRead = delivery.PlacementRead + WantsRead = delivery.AssignerModule + ".wants" + ModulesRead = delivery.IndexerModule + ".modules" + ManifestsRead = delivery.IndexerModule + ".manifests" + BuildsRead = delivery.BuilderModule + ".builds" + HaveRead = "node-engine.have" + ResolvedRead = "mesh-config.resolved" +) + +// BoundReads are the providers' entries this module's manifest reads, `.bound`, whose changes it hears. Which +// path judges a provider is its manifest's word (keepsEntries), not this list: a provider that declares `bound` and is +// not read here leaves its keys asked, saying its entries are not read. +var BoundReads = []string{"gitea", "keycloak", "postgres", "route-adapter", "mesh-vault"} + +// boundRead is a provider's entries as this module reads them. +func boundRead(module string) string { return module + ".bound" } + +// Composer is each module's provisioning plan on a node, composed from the bus: at the builds keptAt names +// (module → fingerprint), the newest build of every other placed module. +type Composer func(ctx context.Context, node string, keptAt map[string]string) (map[string]catalogue.Plan, error) + +// Asker is the operator's channel as this module asks on it: publish an ask under its own name, and read the router's +// record of it. +type Asker interface { + Publish(subject string, body any, id string) error + Record(id string) ([]byte, bool, error) +} + +// Holder is the seat's holder. +type Holder struct { + Store Store + Compose Composer + Asks Asker + Now func() time.Time + Logf func(format string, args ...any) + + mu sync.Mutex +} + +// Composed is what one node's composition did. +type Composed struct { + Node string `json:"node"` + Plans []string `json:"plans-written,omitempty"` + Gone []string `json:"plans-removed,omitempty"` + Changed []string `json:"provisions-changed,omitempty"` + Asked []string `json:"retiring-asked,omitempty"` + Problems []string `json:"problems,omitempty"` +} + +// ComposeNode composes one node: its plans and its provisions. +func (h *Holder) ComposeNode(ctx context.Context, node string) (Composed, error) { + h.mu.Lock() + defer h.mu.Unlock() + out := Composed{Node: node} + wanted, err := h.wantedOn(node) + if err != nil { + return out, err + } + // A module held on the node with no want (the bus on the control-node) is composed at the build its have says it + // runs, so it has a plan too and its client keeps filling it (review of 2026-10-30). + for module, fp := range h.runningOn(node) { + if _, ok := wanted[module]; !ok { + wanted[module] = fp + } + } + atWants, err := h.Compose(ctx, node, wanted) + if err != nil { + return out, fmt.Errorf("the plans of %s cannot be composed: %w", node, err) + } + newest, err := h.Compose(ctx, node, nil) + if err != nil { + return out, fmt.Errorf("the plans of %s at the newest builds cannot be composed: %w", node, err) + } + if err := h.writePlans(node, atWants, &out); err != nil { + out.Problems = append(out.Problems, err.Error()) + } + needed := map[string]provisions.Provision{} + elsewhere := map[string]provisions.Provision{} + vaultNode := h.vaultNode() + for _, plans := range []map[string]catalogue.Plan{atWants, newest} { + for module, plan := range plans { + for _, p := range plan.Provisions { + consumer := provisions.End{Module: module, Node: node} + provider := provisions.End{Module: p.ProviderModule, Node: p.Provider} + pv := provisions.Provision{Provision: p.Provision, Local: p.Local, Consumer: consumer, Provider: provider, + DependsOn: provisions.DependsOn(consumer, p, h.providerPlan(p, node, atWants))} + key, err := pv.Key() + if err != nil { + out.Problems = append(out.Problems, err.Error()) + continue + } + if _, seen := needed[key]; !seen { + needed[key] = pv + } + // Its credential is a provision of its own, the vault's (clause 6): on the consumer's node for a value of the + // consumer's own, on the provider's for the provider's one shared value. + ck, ok := provisions.CredentialKey(p.CredentialForm, consumer, provider, p.Provision, p.Local) + if !ok { + continue + } + cn, cm, cp, cl, err := provisions.ParseKey(ck) + if err != nil { + continue + } + if vaultNode == "" { + out.Problems = append(out.Problems, "no node runs mesh-vault, so the credential "+ck+" has no provider") + continue + } + cred := provisions.Provision{Provision: cp, Local: cl, Consumer: provisions.End{Module: cm, Node: cn}, + Provider: provisions.End{Module: provisions.VaultModule, Node: vaultNode}} + if cn == node { + if _, seen := needed[ck]; !seen { + needed[ck] = cred + } + } else { + elsewhere[ck] = cred + } + } + } + } + h.judgeElsewhere(elsewhere, &out) + h.judgeNode(node, needed, &out) + return out, nil +} + +// providerPlan is the plan of a provision's provider on its node: from this composition when it is on this node, else +// as kept in `plans`; nil when the provider is not handed. +func (h *Holder) providerPlan(p catalogue.PlanProvision, node string, here map[string]catalogue.Plan) *catalogue.Plan { + if p.Provider == node { + if plan, ok := here[p.ProviderModule]; ok { + return &plan + } + return nil + } + key, err := provisions.PlanKey(p.Provider, p.ProviderModule) + if err != nil { + return nil + } + e, err := h.Store.Get(PlansState, key) + if err != nil || e == nil { + return nil + } + plan, err := provisions.DecodePlan(e.Value) + if err != nil { + return nil + } + return &plan.Plan +} + +// vaultNode is the node mesh-vault is placed on: the provider of every credential. +func (h *Holder) vaultNode() string { + nodes := h.placedOn(provisions.VaultModule) + if len(nodes) == 0 { + return "" + } + return nodes[0] +} + +// dependedOn is every key of a node that a provision of another node depends on. +func (h *Holder) dependedOn(node string) []string { + var out []string + keys, err := h.Store.Keys(ProvisionsState) + if err != nil { + return nil + } + for _, k := range keys { + if strings.HasPrefix(k, node+".") { + continue + } + e, err := h.Store.Get(ProvisionsState, k) + if err != nil || e == nil { + continue + } + if p, err := provisions.Decode(e.Value); err == nil { + for _, d := range p.DependsOn { + if strings.HasPrefix(d, node+".") { + out = append(out, d) + } + } + } + } + return out +} + +// judgeElsewhere writes each needed key of another node (a provider's shared credential), its state from its outcome; +// that node's own composition keeps it while a provision depends on it. +func (h *Holder) judgeElsewhere(keys map[string]provisions.Provision, out *Composed) { + for _, key := range sortedKeys(keys) { + next := keys[key] + var have provisions.Provision + e, err := h.Store.Get(ProvisionsState, key) + if err == nil && e != nil { + if have, err = provisions.Decode(e.Value); err == nil { + next.Reference = have.Reference + } + } + h.outcome(key, &next) + if e != nil && same(have, next) { + continue + } + h.put(key, next) + out.Changed = append(out.Changed, key+" "+next.State) + } +} + +// wantedOn is the fingerprint each want on a node names, by module. +func (h *Holder) wantedOn(node string) (map[string]string, error) { + out := map[string]string{} + keys, err := h.Store.Keys(WantsRead) + if err != nil { + return nil, fmt.Errorf("the wants cannot be read: %w", err) + } + for _, k := range keys { + n, module, err := delivery.ParseWantKey(k) + if err != nil || n != node { + continue + } + e, err := h.Store.Get(WantsRead, k) + if err != nil || e == nil { + continue + } + if w, err := delivery.DecodeWant(e.Value); err == nil && !w.Removed && w.FP != "" { + out[module] = w.FP + } + } + return out, nil +} + +// runningOn is the fingerprint each module's have on a node says it runs, by module: what a module held without a want +// is composed at. +func (h *Holder) runningOn(node string) map[string]string { + out := map[string]string{} + keys, err := h.Store.Keys(HaveRead) + if err != nil { + return out + } + for _, k := range keys { + n, module, isNode, err := delivery.ParseHaveKey(k) + if err != nil || isNode || n != node { + continue + } + e, err := h.Store.Get(HaveRead, k) + if err != nil || e == nil { + continue + } + if have, err := delivery.DecodeHave(e.Value); err == nil && have.FP != "" { + out[module] = have.FP + } + } + return out +} + +// writePlans writes each plan that differs and removes the node's plans of modules no longer composed there. +func (h *Holder) writePlans(node string, plans map[string]catalogue.Plan, out *Composed) error { + var problems []error + for _, module := range sortedKeys(plans) { + key, err := provisions.PlanKey(node, module) + if err != nil { + problems = append(problems, err) + continue + } + raw, err := provisions.EncodePlan(provisions.Plan{Node: node, Module: module, Plan: plans[module]}) + if err != nil { + problems = append(problems, err) + continue + } + if e, err := h.Store.Get(PlansState, key); err == nil && e != nil && bytes.Equal(e.Value, raw) { + continue + } + if err := h.Store.Put(PlansState, key, json.RawMessage(raw)); err != nil { + problems = append(problems, fmt.Errorf("the plan %s: %w", key, err)) + continue + } + out.Plans = append(out.Plans, key) + } + keys, err := h.Store.Keys(PlansState) + if err != nil { + return errors.Join(append(problems, err)...) + } + for _, k := range keys { + n, module, ok := strings.Cut(k, ".") + if !ok || n != node { + continue + } + if _, still := plans[module]; still { + continue + } + if err := h.Store.Delete(PlansState, k); err != nil { + problems = append(problems, err) + continue + } + out.Gone = append(out.Gone, k) + } + return errors.Join(problems...) +} + +// judgeNode brings the node's provision keys to what is needed and what each provider said. +func (h *Holder) judgeNode(node string, needed map[string]provisions.Provision, out *Composed) { + held := h.held(node) + // A key of this node another node's provision depends on is needed (a provider's one shared credential). + for _, dep := range h.dependedOn(node) { + if have, ok := held[dep]; ok { + if _, n := needed[dep]; !n { + have.State, have.Why, have.Ask, have.Warrant = "", "", "", nil + needed[dep] = have + } + } + } + for _, key := range unionKeys(needed, held) { + want, isNeeded := needed[key] + have, isHeld := held[key] + var next provisions.Provision + switch { + case isNeeded: + next = want + if isHeld { + next.Reference = have.Reference + } + h.outcome(key, &next) + case have.State == provisions.Retiring: + next = have + if next.Ask == "" && next.Warrant == nil { + if err := h.ask(&next, key); err != nil { + out.Problems = append(out.Problems, err.Error()) + } else { + out.Asked = append(out.Asked, key) + } + } + if gone, why := h.retired(key, have); gone { + if err := h.Store.Delete(ProvisionsState, key); err != nil { + out.Problems = append(out.Problems, err.Error()) + } else { + out.Changed = append(out.Changed, key+" (retired: "+why+")") + } + continue + } + default: + next = have + next.State, next.Why = provisions.Retiring, fmt.Sprintf("no build wanted or placed on %s requires %s of %s on %s "+ + "any more", node, have.Provision, have.Provider.Module, have.Provider.Node) + next.Warrant = nil + if err := h.ask(&next, key); err != nil { + out.Problems = append(out.Problems, err.Error()) + } else { + out.Asked = append(out.Asked, key) + } + } + if isHeld && same(have, next) { + continue + } + next.At = h.Now().UTC() + raw, err := provisions.Encode(next) + if err != nil { + out.Problems = append(out.Problems, err.Error()) + continue + } + if err := h.Store.Put(ProvisionsState, key, json.RawMessage(raw)); err != nil { + out.Problems = append(out.Problems, fmt.Sprintf("%s: %v", key, err)) + continue + } + out.Changed = append(out.Changed, key+" "+next.State) + } +} + +// held is the node's provision keys as kept. +func (h *Holder) held(node string) map[string]provisions.Provision { + out := map[string]provisions.Provision{} + keys, err := h.Store.Keys(ProvisionsState) + if err != nil { + return out + } + for _, k := range keys { + if !strings.HasPrefix(k, node+".") { + continue + } + e, err := h.Store.Get(ProvisionsState, k) + if err != nil || e == nil { + continue + } + if p, err := provisions.Decode(e.Value); err == nil { + out[k] = p + } + } + return out +} + +// same says two provisions say the same, their time aside. +func same(a, b provisions.Provision) bool { + a.At, b.At, a.V, b.V = time.Time{}, time.Time{}, 0, 0 + ra, _ := json.Marshal(a) + rb, _ := json.Marshal(b) + return bytes.Equal(ra, rb) +} + +// entry is a provider's outcome for one consumer, as its entry says it: the provider helper's Entry, read for what +// the holder needs and nothing more. +type entry struct { + // State and Why are the provider's outcome (provider.Entry, ADR 0570 clause 3): provided, failed or waits. + State string `json:"state,omitempty"` + Why string `json:"why,omitempty"` + Refused string `json:"refused,omitempty"` + Waits string `json:"waits,omitempty"` + VaultName string `json:"vault-name,omitempty"` + Generation int `json:"generation,omitempty"` +} + +// outcome sets a needed provision's state from its provider's entry: provided, failed or waits; asked while it has +// none. The holder never sets provided by itself (clause 3): a provider whose entries it does not read keeps the key +// asked, said so. +func (h *Holder) outcome(key string, p *provisions.Provision) { + p.Ask, p.Warrant = "", nil + keeps, err := h.keepsEntries(p.Provider.Module) + if err != nil { + p.State, p.Why = provisions.Asked, "the manifest of "+p.Provider.Module+" cannot be read: "+err.Error() + return + } + if !keeps { + h.haveOutcome(p) + return + } + if !slices.Contains(BoundReads, p.Provider.Module) { + p.State, p.Why = provisions.Asked, p.Provider.Module+" keeps its outcome in its entries, which mesh-provisioner's "+ + "manifest does not read yet" + return + } + e, err := h.Store.Get(boundRead(p.Provider.Module), key) + if err != nil { + p.State, p.Why = provisions.Asked, "the provider's entries cannot be read: "+err.Error() + return + } + if e == nil { + p.State, p.Why = provisions.Asked, "" + return + } + var en entry + if err := json.Unmarshal(e.Value, &en); err != nil { + p.State, p.Why = provisions.Asked, "the provider's entry is not readable" + return + } + switch { + case en.State == provisions.Failed: + p.State, p.Why = provisions.Failed, orSaid(en.Why, en.Refused, "the provider said it failed") + case en.State == provisions.Waits: + p.State, p.Why = provisions.Waits, orSaid(en.Why, en.Waits, "the provider waits for a person") + case en.State == "" && en.Refused != "": + p.State, p.Why = provisions.Failed, en.Refused + case en.State == "" && en.Waits != "": + p.State, p.Why = provisions.Waits, en.Waits + case en.State != "" && en.State != provisions.Provided: + p.State, p.Why = provisions.Asked, "the provider's entry says "+en.State+", which no state of a provision is" + return + default: + p.State, p.Why = provisions.Provided, "" + if en.VaultName != "" || en.Generation > 0 { + p.Reference = &provisions.Reference{ID: en.VaultName, Generation: en.Generation} + } + } +} + +// haveOutcome is the outcome of a provider that makes no credential per consumer: its own have on its node (the lead's +// reading of ADR 0570 clause 3, recorded as an insight). Running with health ok at the build its want names is +// provided; failed is failed with the have's words; anything else is asked, waiting for the provider to run. +func (h *Holder) haveOutcome(p *provisions.Provision) { + where := p.Provider.Module + " on " + p.Provider.Node + // It makes what the consumer needs only once all it depends on is provided (its credential first). + for _, dep := range p.DependsOn { + state := "not composed yet" + if e, err := h.Store.Get(ProvisionsState, dep); err == nil && e != nil { + if d, err := provisions.Decode(e.Value); err == nil { + state = d.State + } + } + if state != provisions.Provided { + p.State, p.Why = provisions.Asked, "waits for "+dep+" ("+state+") before "+where+" can provide it" + return + } + } + key := p.Provider.Node + "." + p.Provider.Module + e, err := h.Store.Get(HaveRead, key) + if err != nil { + p.State, p.Why = provisions.Asked, "the have of "+where+" cannot be read: "+err.Error() + return + } + if e == nil { + p.State, p.Why = provisions.Asked, "waits for "+where+" to run: it says nothing yet" + return + } + have, err := delivery.DecodeHave(e.Value) + if err != nil { + p.State, p.Why = provisions.Asked, "the have of "+where+" is not readable" + return + } + wantFP := "" + if we, err := h.Store.Get(WantsRead, "want."+key); err == nil && we != nil { + if w, err := delivery.DecodeWant(we.Value); err == nil && !w.Removed { + wantFP = w.FP + } + } + switch { + case have.Phase == delivery.PhaseFailed || have.Health == delivery.HealthFailed: + p.State, p.Why = provisions.Failed, orSaid(have.Detail, where+" failed") + case have.Phase == delivery.PhaseRunning && have.Health == delivery.HealthOK && (wantFP == "" || have.FP == wantFP): + p.State, p.Why = provisions.Provided, "" + default: + p.State, p.Why = provisions.Asked, fmt.Sprintf("waits for %s to run healthy at the build its want names (it is %s, "+ + "health %s)", where, have.Phase, have.Health) + } +} + +// keepsEntries says a provider's manifest at the trunk declares the state `bound`: its outcome per consumer is its +// entries there; any other provider's is its own have. +func (h *Holder) keepsEntries(module string) (bool, error) { + mk, err := delivery.ModulesKey(module) + if err != nil { + return false, err + } + e, err := h.Store.Get(ModulesRead, mk) + if err != nil || e == nil { + return false, fmt.Errorf("the indexer holds no modules.%s (%v)", module, err) + } + m, err := delivery.DecodeModule(e.Value) + if err != nil || m.Manifest == "" { + return false, fmt.Errorf("the indexer's modules.%s names no manifest", module) + } + key, err := delivery.ManifestsKey(m.Manifest) + if err != nil { + return false, err + } + me, err := h.Store.Get(ManifestsRead, key) + if err != nil || me == nil { + return false, fmt.Errorf("the indexer holds no manifest of %s (%v)", module, err) + } + man, err := delivery.DecodeManifest(me.Value) + if err != nil { + return false, err + } + var decl struct { + State []json.RawMessage `json:"state"` + } + if err := json.Unmarshal([]byte(man.JSON), &decl); err != nil { + return false, err + } + for _, raw := range decl.State { + var name string + if json.Unmarshal(raw, &name) != nil { + var full struct { + Name string `json:"name"` + } + _ = json.Unmarshal(raw, &full) + name = full.Name + } + if name == "bound" { + return true, nil + } + } + return false, nil +} + +// retired says a retiring key the operator approved is retired: its provider's entry is gone. +func (h *Holder) retired(key string, p provisions.Provision) (bool, string) { + if p.Warrant == nil || p.Warrant.Option != OptionApprove { + return false, "" + } + if keeps, err := h.keepsEntries(p.Provider.Module); err != nil || !keeps { + return false, "" + } + e, err := h.Store.Get(boundRead(p.Provider.Module), key) + if err != nil || e != nil { + return false, "" + } + return true, "its provider removed its credential on the operator's warrant" +} + +// ---- the retiring ask ---------------------------------------------------------------------------------------------- + +// The options a retiring ask offers. The chosen one is the warrant's option in the key, and a provider lets its entry go +// only on "yes" (mesh-sdk provider.go, the pass's retiring keys), so the approving option is "yes" (review of +// 2026-10-30: the holder wrote "approve", which no provider acted on). +const ( + OptionApprove = "yes" + OptionDecline = "no" +) + +// askLasts is how long a retiring ask stays open. +const askLasts = asks.ApproveLasts - 10*time.Minute + +// retiringAsk is the ask of one retiring provision, composed from the key alone. +func retiringAsk(key string, p provisions.Provision, at time.Time, id string) asks.Ask { + what := fmt.Sprintf("%s's access to %s on %s, for %s on %s", p.Consumer.Module, p.Provision, p.Provider.Node, + p.Consumer.Module, p.Consumer.Node) + a := asks.Ask{ID: id, Headline: "Remove " + p.Consumer.Module + "'s " + p.Provision + " access?", + Explanation: fmt.Sprintf("Nothing wanted or placed on %s needs %s any more. On Approve, %s removes %s: what it owns "+ + "is handed to the resource's owner and its credential goes; the resource and its data stay.", p.Consumer.Node, + p.Provision, p.Provider.Module, what), + Who: asks.Operator, Expires: at.Add(askLasts), OnExpiry: "nothing is removed; it is asked again when it changes", + About: "provision." + key, Details: "The provision " + key + " of mesh-provisioner's state provisions."} + for _, o := range []struct{ id, label, does string }{ + {OptionApprove, "Remove", "the provider removes the credential; the resource and its data stay"}, + {OptionDecline, "Keep", "nothing is removed, and it is not asked again while nothing changes"}, + } { + binds, _ := asks.ActDigest(retireAct(key, id, o.id)) + a.Options = append(a.Options, asks.Option{ID: o.id, Label: o.label, Does: o.does, Level: asks.Approve, Binds: binds}) + } + return a +} + +func retireAct(key, id, option string) asks.Act { + return asks.Act{"verb": "mesh-provisioner.retire", "provision": key, "ask": id, "option": option} +} + +// ask raises the retiring ask of a key, under mesh-provisioner's own name; one already raised stands. +func (h *Holder) ask(p *provisions.Provision, key string) error { + if p.Ask != "" { + return nil + } + if h.Asks == nil { + return errors.New("the operator cannot be asked from here, so " + key + " is retiring and nothing is removed") + } + now := h.Now().UTC() + id := "retire-" + strings.ReplaceAll(key, ".", "-") + "-" + fmt.Sprint(now.Unix()) + a := retiringAsk(key, *p, now, id) + if err := a.Check(now); err != nil { + return err + } + if err := h.Asks.Publish(asks.AskSubject(Module), a, id); err != nil { + return fmt.Errorf("the retiring ask of %s could not be published: %w", key, err) + } + p.Ask = id + return nil +} + +// Decide reads the router's record of each retiring key's ask and writes the warrant into the key (clause 5): the +// provider acts on an Approve; a Decline is kept, and not asked again while nothing changes. +func (h *Holder) Decide() []string { + h.mu.Lock() + defer h.mu.Unlock() + if h.Asks == nil { + return nil + } + keys, err := h.Store.Keys(ProvisionsState) + if err != nil { + return []string{"the provisions cannot be read: " + err.Error()} + } + var said []string + for _, k := range keys { + e, err := h.Store.Get(ProvisionsState, k) + if err != nil || e == nil { + continue + } + p, err := provisions.Decode(e.Value) + if err != nil || p.State != provisions.Retiring || p.Ask == "" || p.Warrant != nil { + continue + } + raw, found, err := h.Asks.Record(p.Ask) + if err != nil || !found { + continue + } + var rec struct { + State string `json:"state"` + Ask *asks.Ask `json:"ask"` + Warrant *asks.Warrant `json:"warrant"` + } + if json.Unmarshal(raw, &rec) != nil || rec.State == "open" || rec.Warrant == nil || rec.Ask == nil { + continue + } + w := *rec.Warrant + if w.Asker != Module || w.Ask != p.Ask { + continue + } + option := "" + if w.Outcome == asks.OutcomeChosen { + o, err := w.For(Module, *rec.Ask) + if err == nil { + err = o.Performs(retireAct(k, p.Ask, o.ID)) + } + if err != nil { + said = append(said, k+": the answer is refused: "+err.Error()) + continue + } + option = o.ID + } else { + // Ended with no choice: asked again at the next change. + p.Ask = "" + h.put(k, p) + said = append(said, k+": the ask "+string(w.Outcome)+"; asked again when it changes") + continue + } + by := "the operator" + if w.By != nil { + by = fmt.Sprintf("the %s, as %s identity %s", w.By.Who, w.By.Kind, w.By.Identity) + } + p.Warrant = &provisions.Warrant{Ask: p.Ask, Option: option, By: by, At: w.At.UTC()} + h.put(k, p) + said = append(said, k+": "+option+" by "+by) + } + return said +} + +func (h *Holder) put(key string, p provisions.Provision) { + p.At = h.Now().UTC() + if raw, err := provisions.Encode(p); err == nil { + if err := h.Store.Put(ProvisionsState, key, json.RawMessage(raw)); err != nil { + h.Logf("[%s] %s is not written: %v", Module, key, err) + } + } +} + +// ---- what a change touches -------------------------------------------------------------------------------------- + +// NodesOfPlacement is the node a placement key names. +func NodesOfPlacement(key string) []string { + if n, err := delivery.ParsePlacementKey(key); err == nil { + return []string{n} + } + return nil +} + +// NodesOfWant is the node a want key names. +func NodesOfWant(key string) []string { + if n, _, err := delivery.ParseWantKey(key); err == nil { + return []string{n} + } + return nil +} + +// NodesOfModule is every node a module is placed on, and every node whose plans name it as a provider: what a change +// of the module's manifest touches. +func (h *Holder) NodesOfModule(module string) []string { + nodes := map[string]bool{} + for _, n := range h.placedOn(module) { + nodes[n] = true + } + if keys, err := h.Store.Keys(ProvisionsState); err == nil { + for _, k := range keys { + if e, err := h.Store.Get(ProvisionsState, k); err == nil && e != nil { + if p, err := provisions.Decode(e.Value); err == nil && p.Provider.Module == module { + nodes[p.Consumer.Node] = true + } + } + } + } + return sortedKeys(nodes) +} + +// placedOn is every node a module is placed on, sorted. +func (h *Holder) placedOn(module string) []string { + nodes := map[string]bool{} + if keys, err := h.Store.Keys(PlacementRead); err == nil { + for _, k := range keys { + n, err := delivery.ParsePlacementKey(k) + if err != nil { + continue + } + e, err := h.Store.Get(PlacementRead, k) + if err != nil || e == nil { + continue + } + if p, err := delivery.DecodePlacement(e.Value); err == nil { + for _, m := range p.Modules { + if m.Module == module { + nodes[n] = true + } + } + } + } + } + return sortedKeys(nodes) +} + +// NodeOfEntry is the consumer's node a provider's entry key names. +func NodeOfEntry(key string) []string { + if n, _, _, _, err := provisions.ParseKey(key); err == nil { + return []string{n} + } + return nil +} + +func sortedKeys[V any](m map[string]V) []string { + out := make([]string, 0, len(m)) + for k := range m { + out = append(out, k) + } + sort.Strings(out) + return out +} + +func unionKeys[A, B any](a map[string]A, b map[string]B) []string { + seen := map[string]bool{} + for k := range a { + seen[k] = true + } + for k := range b { + seen[k] = true + } + return sortedKeys(seen) +} + +// Rejudge sets one key's state again from its provider's outcome: what a change of a provider's entry touches. A +// retiring key whose provider removed its entry on the operator's warrant goes. +func (h *Holder) Rejudge(key string) (string, error) { + h.mu.Lock() + defer h.mu.Unlock() + e, err := h.Store.Get(ProvisionsState, key) + if err != nil || e == nil { + return "", err + } + have, err := provisions.Decode(e.Value) + if err != nil { + return "", err + } + if have.State == provisions.Retiring { + if gone, why := h.retired(key, have); gone { + return key + " retired: " + why, h.Store.Delete(ProvisionsState, key) + } + return "", nil + } + next := have + h.outcome(key, &next) + if same(have, next) { + return "", nil + } + h.put(key, next) + return key + " " + next.State, nil +} + +// RetireNeeds deletes every key of the retired state `needs`, which the clients no longer write (clause 1): run once at +// the start. +func (h *Holder) RetireNeeds() (int, error) { + keys, err := h.Store.Keys(Needs) + if err != nil { + return 0, err + } + n := 0 + for _, k := range keys { + if err := h.Store.Delete(Needs, k); err != nil { + return n, err + } + n++ + } + return n, nil +} + +// orSaid is the first of the words that says something. +func orSaid(words ...string) string { + for _, w := range words { + if strings.TrimSpace(w) != "" { + return w + } + } + return "" +} + +// FromBus is the holder's composer over the bus: frombus's plans of a node, at the builds keptAt names, read through +// this module's runtime with its own `reads` (frombus.StatesReads). A plan names no value, so no own secret and no +// credential is read (compose.Planning). +func FromBus() Composer { + r := fromruntime.Reader() + return func(ctx context.Context, node string, keptAt map[string]string) (map[string]catalogue.Plan, error) { + return frombus.Plans(ctx, r, frombus.Mesh{}, frombus.Scope{Node: node, KeptAt: keptAt}) + } +} diff --git a/modules/mesh-provisioner/cmd/mesh-provisioner/holder_integration_test.go b/modules/mesh-provisioner/cmd/mesh-provisioner/holder_integration_test.go new file mode 100644 index 00000000..9aa766c3 --- /dev/null +++ b/modules/mesh-provisioner/cmd/mesh-provisioner/holder_integration_test.go @@ -0,0 +1,229 @@ +//go:build integration + +package main + +// The holder against a real bus (novox/hq ADR 0570 clauses 1-3, 5; issue 1954 H5): its states and every input it reads +// are a real nats-server's key-value buckets, its plans composed by mesh-sdk's composition from bus state over the +// in-memory mesh frombustest gives (restic on n1, requiring postgres-database from postgres there). A placement and a +// want compose the key; the providers' outcomes (the vault's entry for the credential, postgres's own have) turn it +// provided; restic leaving the node makes it retiring and raises the ask. + +import ( + "context" + "encoding/json" + "errors" + "strings" + "testing" + "time" + + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" + + "git.novox.be/novox/mesh-sdk/go/asks" + "git.novox.be/novox/mesh-sdk/go/catalogue" + "git.novox.be/novox/mesh-sdk/go/compose/frombus" + "git.novox.be/novox/mesh-sdk/go/compose/frombus/frombustest" + "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/provisions" + "git.novox.be/novox/mesh-sdk/go/testbus" +) + +// kvStore is the Store over a real bus: this module's states by their local names, another's as `.`. +type kvStore struct { + ctx context.Context + js jetstream.JetStream +} + +func bucketOf(state string) string { + if owner, name, ok := strings.Cut(state, "."); ok { + return owner + "_" + name + } + return Module + "_" + state +} + +func (s kvStore) kv(state string) (jetstream.KeyValue, error) { + kv, err := s.js.KeyValue(s.ctx, bucketOf(state)) + if errors.Is(err, jetstream.ErrBucketNotFound) { + return s.js.CreateKeyValue(s.ctx, jetstream.KeyValueConfig{Bucket: bucketOf(state)}) + } + return kv, err +} + +func (s kvStore) Get(state, key string) (*Entry, error) { + kv, err := s.kv(state) + if err != nil { + return nil, err + } + e, err := kv.Get(s.ctx, key) + if errors.Is(err, jetstream.ErrKeyNotFound) { + return nil, nil + } + if err != nil { + return nil, err + } + return &Entry{Key: key, Value: e.Value(), Revision: e.Revision()}, nil +} + +func (s kvStore) Put(state, key string, value any) error { + kv, err := s.kv(state) + if err != nil { + return err + } + raw, err := json.Marshal(value) + if err != nil { + return err + } + _, err = kv.Put(s.ctx, key, raw) + return err +} + +func (s kvStore) Delete(state, key string) error { + kv, err := s.kv(state) + if err != nil { + return err + } + return kv.Delete(s.ctx, key) +} + +func (s kvStore) Keys(state string) ([]string, error) { + kv, err := s.kv(state) + if err != nil { + return nil, err + } + keys, err := kv.Keys(s.ctx) + if errors.Is(err, jetstream.ErrNoKeysFound) { + return nil, nil + } + return keys, err +} + +type asked struct{ ids []string } + +func (a *asked) Publish(_ string, body any, id string) error { + a.ids = append(a.ids, id) + _ = body.(asks.Ask) + return nil +} +func (a *asked) Record(string) ([]byte, bool, error) { return nil, false, nil } + +// 1954/H5. +func TestTheHolderComposesProvidesAndAsksOnTheBus(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + nc, err := nats.Connect(testbus.URL(t)) + if err != nil { + t.Fatal(err) + } + t.Cleanup(nc.Close) + js, err := jetstream.New(nc) + if err != nil { + t.Fatal(err) + } + s := kvStore{ctx: ctx, js: js} + mesh, err := frombustest.Provisioned() + if err != nil { + t.Fatal(err) + } + node := frombustest.Node + // The bus as the holder reads it: the indexer's modules and manifests, a placement naming where the vault runs, the + // want of restic on n1. + for module, m := range mesh.Module { + m.V = 1 + raw, _ := json.Marshal(m) + if err := s.Put(ModulesRead, module, json.RawMessage(raw)); err != nil { + t.Fatal(err) + } + } + for digest, raw := range mesh.Manifests { + var named struct { + Module string `json:"module"` + } + _ = json.Unmarshal(raw, &named) + value, _ := delivery.EncodeManifest(delivery.Manifest{Module: named.Module, JSON: string(raw)}) + key, _ := delivery.ManifestsKey(digest) + if err := s.Put(ManifestsRead, key, json.RawMessage(value)); err != nil { + t.Fatal(err) + } + } + // The vault keeps its outcomes as entries: its manifest declares `bound`. + vaultManifest := `{"module":"mesh-vault","version":"1","state":["bound"]}` + vaultDigest := delivery.ManifestDigest([]byte(vaultManifest)) + vaultValue, _ := delivery.EncodeManifest(delivery.Manifest{Module: provisions.VaultModule, JSON: vaultManifest}) + vaultKey, _ := delivery.ManifestsKey(vaultDigest) + if err := s.Put(ManifestsRead, vaultKey, json.RawMessage(vaultValue)); err != nil { + t.Fatal(err) + } + if err := s.Put(ModulesRead, provisions.VaultModule, map[string]any{"v": 1, "none2": "the test's", "manifest": vaultDigest, + "repository": "novox/mesh-catalog", "commit": "c1"}); err != nil { + t.Fatal(err) + } + vault, _ := json.Marshal(map[string]any{"v": 1, "node": "n2", "decided": map[string]any{}, + "modules": []map[string]any{{"module": provisions.VaultModule}}}) + if err := s.Put(PlacementRead, "placement.n2", json.RawMessage(vault)); err != nil { + t.Fatal(err) + } + fp := frombustest.FPs()["restic"] + if err := s.Put(WantsRead, "want."+node+".restic", map[string]any{"v": 1, "fp": fp, "self": true}); err != nil { + t.Fatal(err) + } + a := &asked{} + h := &Holder{Store: s, Asks: a, Now: time.Now, Logf: t.Logf, + Compose: func(ctx context.Context, node string, keptAt map[string]string) (map[string]catalogue.Plan, error) { + return frombus.Plans(ctx, mesh, frombus.Mesh{}, frombus.Scope{Node: node, KeptAt: keptAt}) + }} + state := func(key string) provisions.Provision { + e, _ := s.Get(ProvisionsState, key) + if e == nil { + return provisions.Provision{} + } + p, _ := provisions.Decode(e.Value) + return p + } + key := node + ".restic.postgres-database" + if _, err := h.ComposeNode(ctx, node); err != nil { + t.Fatal(err) + } + if p := state(key); p.State != provisions.Asked || p.Provider.Module != "postgres" { + t.Fatalf("the placement and the want composed %+v", p) + } + if e, _ := s.Get(PlansState, node+".restic"); e == nil { + t.Fatal("restic's plan is not on the bus") + } + // The providers' outcomes: the vault provides the credential where restic has one, postgres runs healthy. + for _, dep := range state(key).DependsOn { + if err := s.Put(boundRead(provisions.VaultModule), dep, map[string]any{"state": "provided"}); err != nil { + t.Fatal(err) + } + } + if err := s.Put(HaveRead, node+".postgres", map[string]any{"v": 1, "fp": frombustest.FPs()["postgres"], "phase": "running", + "health": "ok"}); err != nil { + t.Fatal(err) + } + for i := 0; i < 2; i++ { // the credential judged first, then what depends on it + if _, err := h.ComposeNode(ctx, node); err != nil { + t.Fatal(err) + } + } + if p := state(key); p.State != provisions.Provided { + t.Fatalf("the outcomes did not turn it provided: %+v", p) + } + // restic leaves the node and its want goes: retiring, one ask. + p := mesh.Placement[node] + var kept []delivery.PlacedModule + for _, m := range p.Modules { + if m.Module != "restic" { + kept = append(kept, m) + } + } + p.Modules = kept + mesh.Placement[node] = p + if err := s.Delete(WantsRead, "want."+node+".restic"); err != nil { + t.Fatal(err) + } + if _, err := h.ComposeNode(ctx, node); err != nil { + t.Fatal(err) + } + if r := state(key); r.State != provisions.Retiring || r.Ask == "" || len(a.ids) == 0 { + t.Fatalf("removing the want did not ask: %+v, asked %v", r, a.ids) + } +} diff --git a/modules/mesh-provisioner/cmd/mesh-provisioner/holder_test.go b/modules/mesh-provisioner/cmd/mesh-provisioner/holder_test.go new file mode 100644 index 00000000..238b589c --- /dev/null +++ b/modules/mesh-provisioner/cmd/mesh-provisioner/holder_test.go @@ -0,0 +1,362 @@ +package main + +import ( + "context" + "encoding/json" + "go/ast" + "go/parser" + "go/token" + "path/filepath" + "slices" + "strings" + "testing" + "time" + + "git.novox.be/novox/mesh-sdk/go/asks" + "git.novox.be/novox/mesh-sdk/go/catalogue" + "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/provisions" +) + +// router is the operator's channel as the holder asks on it. +type router struct { + asked map[string]asks.Ask + records map[string][]byte +} + +func (r *router) Publish(_ string, body any, id string) error { + r.asked[id] = body.(asks.Ask) + return nil +} + +func (r *router) Record(id string) ([]byte, bool, error) { + raw, ok := r.records[id] + return raw, ok, nil +} + +// answer is the operator's answer to an ask. +func (r *router) answer(t *testing.T, id, option string) { + t.Helper() + a := r.asked[id] + o, ok := a.Option(option) + if !ok { + t.Fatalf("the ask %s offers no %s", id, option) + } + w := asks.Warrant{Ask: id, Asker: Module, About: a.About, Outcome: asks.OutcomeChosen, Option: o.ID, Label: o.Label, + Level: o.Level, By: &asks.Person{Who: asks.Operator, Kind: "telegram", Identity: "42"}, Channel: "telegram", + At: time.Date(2026, 10, 29, 23, 0, 0, 0, time.UTC), AskDigest: a.Digest()} + r.records[id], _ = json.Marshal(map[string]any{"state": "closed", "ask": a, "warrant": w}) +} + +// mesh is the holder over memory, its composer answering each node from tables the test sets: the plans at the wanted +// builds and at the newest. +type mesh struct { + h *Holder + s *memStore + r *router + atWant map[string]map[string]catalogue.Plan + atNewest map[string]map[string]catalogue.Plan + composed []string +} + +func newMesh() *mesh { + m := &mesh{s: newStore(), r: &router{asked: map[string]asks.Ask{}, records: map[string][]byte{}}, + atWant: map[string]map[string]catalogue.Plan{}, atNewest: map[string]map[string]catalogue.Plan{}} + now := time.Date(2026, 10, 29, 22, 0, 0, 0, time.UTC) + m.h = &Holder{Store: m.s, Asks: m.r, Logf: func(string, ...any) {}, + Now: func() time.Time { now = now.Add(time.Second); return now }, + Compose: func(_ context.Context, node string, keptAt map[string]string) (map[string]catalogue.Plan, error) { + m.composed = append(m.composed, node) + if keptAt != nil { + return m.atWant[node], nil + } + return m.atNewest[node], nil + }} + // What each provider's manifest declares: postgres, keycloak and the vault keep entries, the others report by their + // have. + for module, states := range map[string]string{"postgres": `["bound"]`, "keycloak": `["bound"]`, "mesh-vault": `["bound"]`, + "systemd-resolved": `[]`, "minio": `[]`} { + raw := `{"module":"` + module + `","version":"1","state":` + states + `}` + digest := delivery.ManifestDigest([]byte(raw)) + value, _ := delivery.EncodeManifest(delivery.Manifest{Module: module, JSON: raw}) + _ = m.s.Put(ManifestsRead, mustKey(delivery.ManifestsKey(digest)), json.RawMessage(value)) + _ = m.s.Put(ModulesRead, mustKey(delivery.ModulesKey(module)), + json.RawMessage(`{"v":1,"none2":"a test","manifest":"`+digest+`","repository":"novox/mesh-catalog","commit":"c1"}`)) + } + return m +} + +func mustKey(k string, err error) string { + if err != nil { + panic(err) + } + return k +} + +// needs makes a module on a node require provisions in its wanted build (and the same in its newest unless set). +func planOf(provs ...catalogue.PlanProvision) catalogue.Plan { + return catalogue.Plan{Provisions: provs} +} + +var ( + pgDB = catalogue.PlanProvision{Provision: "postgres-database", Provider: "ace", ProviderModule: "postgres"} + oidc = catalogue.PlanProvision{Provision: "oidc-client", Provider: "novox", ProviderModule: "keycloak"} + split = catalogue.PlanProvision{Provision: "split-dns", Provider: "g14", ProviderModule: "systemd-resolved"} +) + +func (m *mesh) state(t *testing.T, key string) provisions.Provision { + t.Helper() + e, _ := m.s.Get(ProvisionsState, key) + if e == nil { + return provisions.Provision{} + } + p, err := provisions.Decode(e.Value) + if err != nil { + t.Fatal(err) + } + return p +} + +func (m *mesh) compose(t *testing.T, node string) Composed { + t.Helper() + c, err := m.h.ComposeNode(context.Background(), node) + if err != nil { + t.Fatal(err) + } + return c +} + +// A node composes its plans and provisions from what is wanted and placed; a key's state follows its provider's entry +// alone; the vault's generation reaches the reference; a +// quiet pass writes nothing (ADR 0570 clauses 1-3, 6, 7; issue 1954 H1, H2). +func TestAKeysStateFollowsItsProvidersOutcome(t *testing.T) { + m := newMesh() + m.atWant["g14"] = map[string]catalogue.Plan{"gitea": planOf(pgDB, split)} + m.atNewest["g14"] = m.atWant["g14"] + key := "g14.gitea.postgres-database" + steps := []struct { + name string + entry string // the provider's entry written before the step, "-" to delete + state string + why string + }{ + {"composed: asked", "", provisions.Asked, ""}, + {"the provider waits for a person", `{"waits":"the database's owner to accept"}`, provisions.Waits, "owner"}, + {"the provider refused", `{"refused":"no room on ace"}`, provisions.Failed, "no room"}, + {"the provider says failed in its state", `{"state":"failed","why":"the disk is full"}`, provisions.Failed, "disk"}, + {"the provider made it", `{"state":"provided","vault-name":"pair/gitea-postgres","generation":3}`, provisions.Provided, ""}, + } + for _, s := range steps { + if s.entry != "" { + m.s.write(t, "postgres.bound", key, s.entry) + } + m.compose(t, "g14") + p := m.state(t, key) + if p.State != s.state || !strings.Contains(p.Why, s.why) || p.Provider.Module != "postgres" || p.Provider.Node != "ace" { + t.Errorf("%s: %+v", s.name, p) + } + } + if p := m.state(t, key); p.Reference == nil || p.Reference.ID != "pair/gitea-postgres" || p.Reference.Generation != 3 { + t.Errorf("the vault's generation does not reach the reference: %+v", p.Reference) + } + // A provider making no credential per consumer reports through its own have (clause 3): never provided by the + // holder alone. + split := "g14.gitea.split-dns" + for _, row := range []struct{ have, want, state, why string }{ + {"", "", provisions.Asked, "says nothing yet"}, + {`{"v":1,"fp":"src2:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","phase":"applying","health":"unknown"}`, "", provisions.Asked, "applying"}, + {`{"v":1,"fp":"src2:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","phase":"running","health":"ok"}`, `{"v":1,"fp":"src2:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb","self":true}`, provisions.Asked, "build its want"}, + {`{"v":1,"fp":"src2:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb","phase":"failed","health":"failed","detail":"the unit did not start"}`, "", provisions.Failed, "did not start"}, + {`{"v":1,"fp":"src2:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb","phase":"running","health":"ok"}`, `{"v":1,"fp":"src2:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb","self":true}`, provisions.Provided, ""}, + } { + if row.have != "" { + m.s.write(t, HaveRead, "g14.systemd-resolved", row.have) + } + if row.want != "" { + m.s.write(t, WantsRead, "want.g14.systemd-resolved", row.want) + } + m.compose(t, "g14") + if p := m.state(t, split); p.State != row.state || !strings.Contains(p.Why, row.why) { + t.Errorf("the provider's have %s: %+v, want %s", row.have, p, row.state) + } + } + if e, _ := m.s.Get(PlansState, "g14.gitea"); e == nil { + t.Error("the plan is not written") + } + if c := m.compose(t, "g14"); len(c.Plans)+len(c.Changed)+len(c.Asked) != 0 { + t.Errorf("a quiet pass wrote %+v", c) + } + // A provision's credential is the vault's own provision, on the consumer's node for a login of its own, and the + // provision depends on it (clauses 3, 6). + c := newMesh() + c.s.write(t, PlacementRead, "placement.novox", `{"v":1,"node":"novox","decided":{},"modules":[{"module":"mesh-vault"}]}`) + pair := pgDB + pair.CredentialForm = "pair" + c.atWant["g14"] = map[string]catalogue.Plan{"gitea": planOf(pair)} + c.atNewest["g14"] = c.atWant["g14"] + c.compose(t, "g14") + cred := c.state(t, "g14.gitea.secret-postgres-database") + if cred.Provider.Module != "mesh-vault" || cred.Provider.Node != "novox" || cred.State != provisions.Asked { + t.Errorf("the credential's provision is %+v", cred) + } + if d := c.state(t, key).DependsOn; !slices.Equal(d, []string{"g14.gitea.secret-postgres-database"}) { + t.Errorf("the provision depends on %v", d) + } + // A pair provider applying the vault's value through its templates (no entries) is provided only once its credential + // key is provided and it runs healthy (the lead's refinement of clause 3). + bucket := catalogue.PlanProvision{Provision: "s3-bucket", Provider: "g14", ProviderModule: "minio", CredentialForm: "pair"} + c.atWant["g14"] = map[string]catalogue.Plan{"gitea": planOf(bucket)} + c.atNewest["g14"] = c.atWant["g14"] + c.s.write(t, HaveRead, "g14.minio", `{"v":1,"fp":"src2:`+strings.Repeat("cc", 32)+`","phase":"running","health":"ok"}`) + s3, s3cred := "g14.gitea.s3-bucket", "g14.gitea.secret-s3-bucket" + for _, row := range []struct { + name, entry, cred, state string + }{ + {"its credential still asked", "", provisions.Asked, provisions.Asked}, + {"its credential provided by the vault", `{"state":"provided","vault-name":"pair/g14/s3-bucket/g14/gitea","generation":1}`, + provisions.Provided, provisions.Provided}, + } { + if row.entry != "" { + c.s.write(t, "mesh-vault.bound", s3cred, row.entry) + } + c.compose(t, "g14") + c.compose(t, "g14") // the credential judged first, then what depends on it + if got, dep := c.state(t, s3), c.state(t, s3cred); got.State != row.state || dep.State != row.cred { + t.Errorf("%s: %s is %s (%s), its credential %s", row.name, s3, got.State, got.Why, dep.State) + } + } + // An entry's change judges its key alone, no composition. + before := len(m.composed) + m.s.write(t, "postgres.bound", key, `{"refused":"the login is locked"}`) + if said, err := m.h.Rejudge(key); err != nil || !strings.Contains(said, provisions.Failed) || len(m.composed) != before { + t.Errorf("rejudged %q, %v, composed %d more", said, err, len(m.composed)-before) + } +} + +// A provision no build wanted or placed needs is retiring with one ask naming provider, node and what goes; one composed +// ahead of its want raises none; a change removing one requirement and adding another retires only the removed one; the +// warrant goes into the key and the key goes once the provider removed its entry; a decline is kept and not asked again; +// a provision needed again returns from retiring (ADR 0570 clause 5; issue 1954 H3). +func TestRetiringIsAskedAndActsOnlyOnTheWarrant(t *testing.T) { + m := newMesh() + pg, kc := "g14.gitea.postgres-database", "g14.gitea.oidc-client" + m.atWant["g14"] = map[string]catalogue.Plan{"gitea": planOf(pgDB)} + m.atNewest["g14"] = map[string]catalogue.Plan{"gitea": planOf(pgDB, oidc)} // the newest build adds oidc ahead of its want + m.s.write(t, "postgres.bound", pg, `{}`) + steps := []struct { + name string + change func() + asked []string + states map[string]string + removed bool + }{ + {"a key composed ahead of its want is asked, not retiring", func() {}, nil, + map[string]string{pg: provisions.Provided, kc: provisions.Asked}, false}, + {"one requirement removed and another added: only the removed retires", func() { + m.atWant["g14"] = map[string]catalogue.Plan{"gitea": planOf(oidc)} + m.atNewest["g14"] = map[string]catalogue.Plan{"gitea": planOf(oidc)} + }, []string{pg}, map[string]string{pg: provisions.Retiring, kc: provisions.Asked}, false}, + {"composed again: no second ask", func() {}, nil, map[string]string{pg: provisions.Retiring}, false}, + {"needed again before the warrant: back to its outcome", func() { + m.atWant["g14"] = map[string]catalogue.Plan{"gitea": planOf(oidc, pgDB)} + }, nil, map[string]string{pg: provisions.Provided}, false}, + {"gone again: asked again", func() { m.atWant["g14"] = map[string]catalogue.Plan{"gitea": planOf(oidc)} }, + []string{pg}, map[string]string{pg: provisions.Retiring}, false}, + } + for _, s := range steps { + s.change() + c := m.compose(t, "g14") + if !slices.Equal(c.Asked, s.asked) { + t.Errorf("%s: asked %v, want %v", s.name, c.Asked, s.asked) + } + for k, st := range s.states { + if p := m.state(t, k); p.State != st { + t.Errorf("%s: %s is %s, want %s (%s)", s.name, k, p.State, st, p.Why) + } + } + } + p := m.state(t, pg) + a := m.r.asked[p.Ask] + if !strings.Contains(a.Explanation, "postgres") || !strings.Contains(a.Explanation, "ace") || + !strings.Contains(a.Explanation, "data stay") { + t.Errorf("the ask does not say what goes from which provider on which node: %+v", a) + } + // Nothing before the warrant: the provider's entry stands and the key stays. + if said := m.h.Decide(); len(said) != 0 || m.state(t, pg).Warrant != nil { + t.Errorf("decided before an answer: %v", said) + } + m.r.answer(t, p.Ask, OptionApprove) + m.h.Decide() + if w := m.state(t, pg).Warrant; w == nil || w.Option != OptionApprove || !strings.Contains(w.By, "42") { + t.Fatalf("the warrant is not in the key: %+v", w) + } + // The word the SDK's provider lets an entry go on (provider.go): any other, and no provider ever retires it. + if w := m.state(t, pg).Warrant; w.Option != "yes" { + t.Fatalf("the warrant says %q, and a provider lets go only on \"yes\"", w.Option) + } + if said, _ := m.h.Rejudge(pg); said != "" || m.state(t, pg).State != provisions.Retiring { + t.Errorf("retired while the provider's entry stands: %q", said) + } + _ = m.s.Delete("postgres.bound", pg) + if said, _ := m.h.Rejudge(pg); !strings.Contains(said, "retired") || m.state(t, pg).State != "" { + t.Errorf("not retired once the provider removed its entry: %q", said) + } + // A decline is kept, and nothing is asked again while nothing changes. + d := newMesh() + d.atWant["g14"] = map[string]catalogue.Plan{"gitea": planOf(pgDB)} + d.compose(t, "g14") + d.atWant["g14"] = map[string]catalogue.Plan{} + d.compose(t, "g14") + d.r.answer(t, d.state(t, pg).Ask, OptionDecline) + d.h.Decide() + if c := d.compose(t, "g14"); len(c.Asked) != 0 || d.state(t, pg).Warrant.Option != OptionDecline { + t.Errorf("a decline was asked again: %+v", c) + } +} + +// Review of 2026-10-30: a module held on a node with no want (the bus on the control-node) is composed at the build +// its have says it runs, so the holder writes it a plan and its client keeps filling it. +func TestAModuleHeldWithoutAWantIsComposedAtTheBuildItRuns(t *testing.T) { + m := newMesh() + fp := "src2:" + strings.Repeat("dd", 32) + m.s.write(t, HaveRead, "novox.nats", `{"v":1,"fp":"`+fp+`","phase":"running","health":"ok"}`) + m.s.write(t, HaveRead, "g14.dunst", `{"v":1,"fp":"src2:`+strings.Repeat("ee", 32)+`","phase":"running","health":"ok"}`) + var keptAt map[string]string + m.h.Compose = func(_ context.Context, node string, at map[string]string) (map[string]catalogue.Plan, error) { + if at != nil { + keptAt = at + return map[string]catalogue.Plan{"nats": {}}, nil + } + return nil, nil + } + c := m.compose(t, "novox") + if keptAt["nats"] != fp || len(keptAt) != 1 { + t.Fatalf("composed at %v, want nats at the build it runs and nothing of another node", keptAt) + } + if !slices.Contains(c.Plans, "novox.nats") { + t.Fatalf("no plan written for a module held without a want: %+v", c) + } +} + +// The holder holds no ticker: it composes on change events alone (ADR 0570 clause 2; issue 1954 H1). +func TestTheHolderHoldsNoTimer(t *testing.T) { + files, _ := filepath.Glob("*.go") + for _, f := range files { + if strings.HasSuffix(f, "_test.go") { + continue + } + file, err := parser.ParseFile(token.NewFileSet(), f, nil, 0) + if err != nil { + t.Fatal(err) + } + ast.Inspect(file, func(n ast.Node) bool { + if sel, ok := n.(*ast.SelectorExpr); ok { + if id, ok := sel.X.(*ast.Ident); ok && id.Name == "time" && + slices.Contains([]string{"NewTicker", "Tick", "AfterFunc", "NewTimer"}, sel.Sel.Name) { + t.Errorf("%s uses time.%s: the holder composes on events alone", f, sel.Sel.Name) + } + } + return true + }) + } +} diff --git a/modules/mesh-provisioner/cmd/mesh-provisioner/holdertools.go b/modules/mesh-provisioner/cmd/mesh-provisioner/holdertools.go new file mode 100644 index 00000000..30a16446 --- /dev/null +++ b/modules/mesh-provisioner/cmd/mesh-provisioner/holdertools.go @@ -0,0 +1,104 @@ +package main + +// The holder's tools (novox/hq ADR 0570 Consequences, ADR 0564 §12's rule): a read of every key of `provisions` and +// `plans`, one key whole, each state's fill, composing one node now, and reading the operator's answers now. + +import ( + "context" + "encoding/json" + "fmt" + "strings" + + stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/provisions" +) + +// The holder's tools. +const ( + ProvisionsTool = "provisioner_provisions" + PlansTool = "provisioner_plans" + ComposeTool = "provisioner_compose" + DecideTool = "provisioner_decide" +) + +func str(d string) map[string]any { return map[string]any{"type": "string", "description": d} } + +func holderTools(h *Holder) []stdio.Tool { + arg := func(a map[string]any, k string) string { s, _ := a[k].(string); return strings.TrimSpace(s) } + return []stdio.Tool{ + {Name: ProvisionsTool, + Description: "Every provision the seat keeps (novox/hq ADR 0570): its key ..[.], " + + "provider, consumer and state (asked, provided, waits, failed, retiring, with why), narrowed by node and " + + "state, with how many keys and bytes the state holds; or one key whole.", + Input: map[string]any{"type": "object", "properties": map[string]any{"key": str("one key, whole"), + "node": str("a consumer's node"), "state": str("asked, provided, waits, failed or retiring")}}, + Run: func(a map[string]any) (any, error) { + return h.read(ProvisionsState, arg(a, "key"), arg(a, "node"), arg(a, "state")) + }}, + {Name: PlansTool, + Description: "Every module's provisioning plan the seat composed, ., narrowed by node, with how many " + + "keys and bytes the state holds; or one plan whole.", + Input: map[string]any{"type": "object", "properties": map[string]any{"key": str("one plan, ."), + "node": str("a node")}}, + Run: func(a map[string]any) (any, error) { return h.read(PlansState, arg(a, "key"), arg(a, "node"), "") }}, + {Name: ComposeTool, + Description: "Compose one node's plans and provisions now from the bus, as a change event would, and say what " + + "was written, removed, changed and asked.", + Input: map[string]any{"type": "object", "properties": map[string]any{"node": str("the node")}, + "required": []string{"node"}}, + Run: func(a map[string]any) (any, error) { + if arg(a, "node") == "" { + return nil, fmt.Errorf("compose names the node") + } + return h.ComposeNode(context.Background(), arg(a, "node")) + }}, + {Name: DecideTool, + Description: "Read the operator's answers to every retiring provision's ask now, writing each warrant into its " + + "key, as the router's word on an ask would.", + Run: func(map[string]any) (any, error) { return map[string]any{"said": h.Decide()}, nil }}, + } +} + +// read is a state's keys narrowed, or one whole, with the state's fill. +func (h *Holder) read(state, key, node, want string) (any, error) { + if key != "" { + e, err := h.Store.Get(state, key) + if err != nil { + return nil, err + } + if e == nil { + return nil, fmt.Errorf("%s holds no %s", state, key) + } + return json.RawMessage(e.Value), nil + } + keys, err := h.Store.Keys(state) + if err != nil { + return nil, err + } + rows := []map[string]any{} + bytes := 0 + for _, k := range keys { + e, err := h.Store.Get(state, k) + if err != nil || e == nil { + continue + } + bytes += len(e.Value) + if node != "" && !strings.HasPrefix(k, node+".") { + continue + } + row := map[string]any{"key": k} + if state == ProvisionsState { + p, err := provisions.Decode(e.Value) + if err != nil { + row["unreadable"] = err.Error() + } else { + if want != "" && p.State != want { + continue + } + row["state"], row["provider"], row["why"] = p.State, p.Provider.Module+" on "+p.Provider.Node, p.Why + } + } + rows = append(rows, row) + } + return map[string]any{state: rows, "count": len(rows), "fill": map[string]int{"keys": len(keys), "bytes": bytes}}, nil +} diff --git a/modules/mesh-provisioner/cmd/mesh-provisioner/main.go b/modules/mesh-provisioner/cmd/mesh-provisioner/main.go index 3955516d..cca4a622 100644 --- a/modules/mesh-provisioner/cmd/mesh-provisioner/main.go +++ b/modules/mesh-provisioner/cmd/mesh-provisioner/main.go @@ -1,23 +1,28 @@ // mesh-provisioner: the provisioner's server, holder of the mesh-provisioner seat once for the mesh (novox/hq ADR -// 0349 clause 3, design 57 P4). A Go bundle served by a runtime of its own, on its own bus account (`runs-as`, ADR -// 0259 §8), keeping four buckets: mesh-provisioner_registry (which domain answers which kind of placeholder, its -// alone), and mesh-provisioner_needs, _applied and _collect, which each machine's provisioner client writes for its -// own machine (the controller grants the client those keys alone, issue 670). stdout is the MCP channel; what this -// module says, it says on stderr. +// 0349 clause 3, ADR 0570). A Go bundle served by a runtime of its own, on its own bus account (`runs-as`, ADR 0259 +// §8), keeping mesh-provisioner_provisions and _plans (every provision and every plan, composed from the bus on change +// events, its alone), _registry (which domain answers which kind of placeholder, its alone), and _applied and +// _collect, which each machine's provisioner client writes for its own machine; _needs is retired and emptied at its +// start. stdout is the MCP channel; what this module says, it says on stderr. package main import ( + "context" "fmt" "os" + "sync" "time" stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/asks" + "git.novox.be/novox/mesh-sdk/go/delivery" ) // Seat is the role this module holds, once for the mesh, and Verb its one verb. const ( - Seat = "mesh-provisioner" - Verb = "provisioning" + Seat = "mesh-provisioner" + Module = "mesh-provisioner" + Verb = "provisioning" ) func logf(format string, a ...any) { fmt.Fprintf(os.Stderr, format+"\n", a...) } @@ -42,30 +47,149 @@ func (runtimeStore) Delete(state, key string) error { return stdio.State(state). func (runtimeStore) Keys(state string) ([]string, error) { return stdio.State(state).Keys() } +// runtimeAsks is the operator's channel through the runtime: asks published under this module's own name, and the +// router's record of one read under that name. +type runtimeAsks struct{} + +func (runtimeAsks) Publish(subject string, body any, id string) error { + return stdio.SeatPublish(subject, body, id) +} + +func (runtimeAsks) Record(id string) ([]byte, bool, error) { + e, err := stdio.SeatRecord("", id) + if err != nil || e == nil { + return nil, false, err + } + return e.Value, true, nil +} + +// work is the nodes a change touched, composed one at a time by one worker: no timer, a change's event is the wake. +type work struct { + mu sync.Mutex + pending map[string]bool + wake chan struct{} +} + +func (w *work) add(nodes ...string) { + if len(nodes) == 0 { + return + } + w.mu.Lock() + for _, n := range nodes { + w.pending[n] = true + } + w.mu.Unlock() + select { + case w.wake <- struct{}{}: + default: + } +} + +func (w *work) run(h *Holder) { + for range w.wake { + w.mu.Lock() + nodes := sortedKeys(w.pending) + w.pending = map[string]bool{} + w.mu.Unlock() + for _, n := range nodes { + c, err := h.ComposeNode(context.Background(), n) + switch { + case err != nil: + logf("[%s] %s is not composed: %v", Module, n, err) + case len(c.Plans)+len(c.Gone)+len(c.Changed)+len(c.Asked)+len(c.Problems) > 0: + logf("[%s] composed %s: %+v", Module, n, c) + } + } + } +} + +// watched asks a watch of the runtime again until it is taken: the runtime answers only once Serve runs. +func watched(state string, handle func(stdio.StateChange) error) { + for wait := 2 * time.Second; ; wait = min(wait*2, time.Minute) { + time.Sleep(wait) + if err := stdio.State(state).Watch("", handle); err != nil { + logf("[%s] %s is not watched yet (%v); asking again in %s", Module, state, err, wait) + continue + } + return + } +} + func main() { - go publishing(runtimeStore{}) - if err := stdio.Serve("", Tools(runtimeStore{})); err != nil { + s := runtimeStore{} + h := &Holder{Store: s, Compose: FromBus(), Asks: runtimeAsks{}, Now: time.Now, Logf: logf} + w := &work{pending: map[string]bool{}, wake: make(chan struct{}, 1)} + go w.run(h) + go starting(h) + go watched(PlacementRead, func(c stdio.StateChange) error { w.add(NodesOfPlacement(c.Key)...); return nil }) + go watched(WantsRead, func(c stdio.StateChange) error { w.add(NodesOfWant(c.Key)...); return nil }) + go watched(ModulesRead, func(c stdio.StateChange) error { + if module, err := delivery.ParseModulesKey(c.Key); err == nil { + w.add(h.NodesOfModule(module)...) + } + return nil + }) + for _, provider := range BoundReads { + go watched(boundRead(provider), func(c stdio.StateChange) error { + if said, err := h.Rejudge(c.Key); err != nil { + logf("[%s] %s is not judged again: %v", Module, c.Key, err) + } else if said != "" { + logf("[%s] %s", Module, said) + } + return nil + }) + } + go watched(Registry, func(c stdio.StateChange) error { + if c.Op != "put" { + if _, _, err := Publish(s); err != nil { + logf("[%s] the registry is not written back: %v", Module, err) + } + } + return nil + }) + go func() { + for wait := 2 * time.Second; ; wait = min(wait*2, time.Minute) { + time.Sleep(wait) + err := stdio.Subscribe(asks.Seat+".decided."+Module, func(stdio.Envelope) error { + for _, said := range h.Decide() { + logf("[%s] %s", Module, said) + } + return nil + }) + if err == nil { + return + } + logf("[%s] does not hear the operator's answers yet (%v); asking again in %s", Module, err, wait) + } + }() + if err := stdio.Serve("", append(Tools(s), holderTools(h)...)); err != nil { logf("%v", err) os.Exit(1) } } -// publishing writes the registry once the runtime answers, and again every hour, so a registry emptied by hand or -// a bucket made again is written back without a restart. The runtime answers only once Serve runs, so the first -// pass is tried again until it lands. -func publishing(s Store) { - for wait := 2 * time.Second; ; { - time.Sleep(wait) - written, deleted, err := Publish(s) - switch { - case err != nil: - logf("[mesh-provisioner] the registry is not written yet (%v); trying again in %s", err, wait) - wait = min(wait*2, time.Minute) - continue - case len(written) > 0 || len(deleted) > 0: - logf("[mesh-provisioner] the registry: wrote %v, deleted %v", written, deleted) +// starting writes the registry, deletes the retired needs and reads the operator's answers once the runtime answers, +// each tried again until it lands. +func starting(h *Holder) { + <-stdio.Served() + for wait := 2 * time.Second; ; wait = min(wait*2, time.Minute) { + written, deleted, err := Publish(h.Store) + if err == nil { + if len(written)+len(deleted) > 0 { + logf("[%s] the registry: wrote %v, deleted %v", Module, written, deleted) + } + break } - wait = time.Hour + logf("[%s] the registry is not written yet (%v); trying again in %s", Module, err, wait) + time.Sleep(wait) + } + if n, err := h.RetireNeeds(); err != nil { + logf("[%s] the retired needs are not all deleted: %v", Module, err) + } else if n > 0 { + logf("[%s] deleted %d key(s) of the retired needs", Module, n) + } + for _, said := range h.Decide() { + logf("[%s] %s", Module, said) } } diff --git a/modules/mesh-provisioner/cmd/mesh-provisioner/provisioner_test.go b/modules/mesh-provisioner/cmd/mesh-provisioner/provisioner_test.go index 4661b226..dd0ccde3 100644 --- a/modules/mesh-provisioner/cmd/mesh-provisioner/provisioner_test.go +++ b/modules/mesh-provisioner/cmd/mesh-provisioner/provisioner_test.go @@ -8,6 +8,8 @@ import ( "strings" "sync" "testing" + + "git.novox.be/novox/mesh-sdk/go/compose/frombus" ) // memStore is the bus as mesh-provisioner reaches it, in memory: the states it keeps. @@ -106,9 +108,9 @@ func TestTheRegistryAnswersEveryKindATemplateMayName(t *testing.T) { for kind, bucket := range map[string]string{"config": "mesh-config_resolved", "collected": "mesh-provisioner_" + Collect, "shell": "mesh-provisioner_" + Collect, "environment": "mesh-provisioner_" + Collect, "contribution": "mesh-provisioner_" + Collect, - "dir": "mesh-controller_assignments", "access": "mesh-controller_assignments", - "port": "mesh-controller_assignments", "seat": "mesh-controller_assignments", - "machine": "mesh-controller_assignments", "backup": "mesh-controller_assignments"} { + "dir": "mesh-provisioner_plans", "access": "mesh-provisioner_plans", + "port": "mesh-provisioner_plans", "seat": "mesh-provisioner_plans", + "machine": "mesh-provisioner_plans", "backup": "mesh-provisioner_plans"} { i := slices.IndexFunc(TheRegistry, func(a Answer) bool { return a.Kind == kind }) if TheRegistry[i].Bucket != bucket { t.Errorf("%s is found in %s, want %s", kind, TheRegistry[i].Bucket, bucket) @@ -288,6 +290,8 @@ func TestTheManifestDeclaresWhatTheCodeKeeps(t *testing.T) { Serves []string `json:"serves"` } `json:"claims"` State []string `json:"state"` + Reads []string `json:"reads"` + Tools []string `json:"tools"` OwnSecrets map[string]string `json:"own-secrets"` } if err := json.Unmarshal(raw, &m); err != nil { @@ -300,7 +304,39 @@ func TestTheManifestDeclaresWhatTheCodeKeeps(t *testing.T) { !slices.Equal(m.Claims[0].Serves, []string{Verb}) { t.Errorf("the claim: %+v", m.Claims) } - if want := []string{Registry, Needs, Applied, Collect}; !slices.Equal(m.State, want) { + // needs stays declared this release so its first start can empty it (ADR 0570 clause 1). + if want := []string{Registry, Needs, Applied, Collect, ProvisionsState, PlansState}; !slices.Equal(m.State, want) { t.Errorf("the state is %v, want %v", m.State, want) } + // Every state and act has its tool (ADR 0570 Consequences): the verb reads the clients' states, the holder's tools + // its own and the composition it does on a change. + served := map[string]bool{} + for _, tool := range append(Tools(newStore()), holderTools(&Holder{})...) { + served[tool.Name] = true + } + for what, tool := range map[string]string{Registry: Seat + "." + Verb, Needs: Seat + "." + Verb, + Applied: Seat + "." + Verb, Collect: Seat + "." + Verb, ProvisionsState: ProvisionsTool, PlansState: PlansTool, + "a composition on a change": ComposeTool, "a decision on an answer": DecideTool} { + if !served[tool] { + t.Errorf("%s has no tool: %s is not served", what, tool) + } + } + for _, tool := range m.Tools { + if !served[tool] { + t.Errorf("the manifest lists %s, which is not served", tool) + } + } + for _, r := range append([]string{WantsRead}, frombus.StatesReads...) { // what fromruntime.Reader reads + if !slices.Contains(m.Reads, r) { + t.Errorf("the manifest does not read %s, which the composition reads", r) + } + } + for _, p := range BoundReads { + if !slices.Contains(m.Reads, boundRead(p)) { + t.Errorf("the manifest does not read %s's entries", p) + } + } + if slices.Contains(m.Reads, "mesh-controller.assignments") { + t.Error("the manifest reads the controller's assignments (ADR 0570 clause 2)") + } } diff --git a/modules/mesh-provisioner/cmd/mesh-provisioner/reads-outside b/modules/mesh-provisioner/cmd/mesh-provisioner/reads-outside new file mode 100644 index 00000000..203aee31 --- /dev/null +++ b/modules/mesh-provisioner/cmd/mesh-provisioner/reads-outside @@ -0,0 +1,2 @@ +# What these tests read outside their package, as paths from the repository's root (issue 1936). +modules/mesh-provisioner/module.json diff --git a/modules/mesh-provisioner/cmd/mesh-provisioner/registry.go b/modules/mesh-provisioner/cmd/mesh-provisioner/registry.go index 7d7d6f3e..15f0bb3f 100644 --- a/modules/mesh-provisioner/cmd/mesh-provisioner/registry.go +++ b/modules/mesh-provisioner/cmd/mesh-provisioner/registry.go @@ -48,16 +48,15 @@ type Answer struct { Decision string `json:"decision"` } -// placement is the controller's: where a module runs, its directories, accesses, ports, seats and the machine's -// facts (ADR 0349 clause 6). On the bus, in the provisioner's plan of each module the controller handed to it, which -// its assignment carries (mesh-controller internal/inventory Provisioning, issue 671), or for a module only pulled in -// by another, its value of its own under the same key (issue 765): a client never reads the node's declaration, -// which the node-engine keeps for root alone. +// placement is the seat's: where a module runs, its directories, accesses, ports, seats and the machine's facts (ADR +// 0349 clause 6), in the provisioning plan this seat composes of each module on each node from the bus, its state +// `plans` (novox/hq ADR 0570 clauses 1, 2): a client never reads the node's declaration, which the node-engine keeps for +// root alone, nor the controller's assignments. func placement(kind, what string) Answer { - return Answer{Kind: kind, Domain: "mesh-controller", From: FromBucket, Bucket: "mesh-controller_assignments", + return Answer{Kind: kind, Domain: Module, From: FromBucket, Bucket: Module + "_" + PlansState, Key: ".", - Why: what + ": placement, the controller's, in the provisioner's plan its assignment carries (provisioning." + - "placement)", Decision: "novox/hq ADR 0349 clause 6"} + Why: what + ": placement, in the provisioning plan the seat mesh-provisioner composes of the module on the node " + + "(plans, plan.placement)", Decision: "novox/hq ADR 0570 clause 1"} } // oldCollection is a namespace read as a collection of its own name (novox/hq ADR 0349 clause 13, design 57 L1). diff --git a/modules/mesh-provisioner/go.mod b/modules/mesh-provisioner/go.mod index 9f0b6d60..db34240f 100644 --- a/modules/mesh-provisioner/go.mod +++ b/modules/mesh-provisioner/go.mod @@ -2,4 +2,21 @@ module mesh-provisioner go 1.26.0 -require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021100209-cb54756c3865 +require ( + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 + github.com/nats-io/nats.go v1.54.0 +) + +require ( + github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op // indirect + github.com/google/go-tpm v0.9.8 // indirect + github.com/klauspost/compress v1.20.0 // indirect + github.com/minio/highwayhash v1.0.4 // indirect + github.com/nats-io/jwt/v2 v2.8.1 // indirect + github.com/nats-io/nats-server/v2 v2.11.17 // indirect + github.com/nats-io/nkeys v0.4.16 // indirect + github.com/nats-io/nuid v1.0.1 // indirect + golang.org/x/crypto v0.57.0 // indirect + golang.org/x/sys v0.48.0 // indirect + golang.org/x/time v0.15.0 // indirect +) diff --git a/modules/mesh-provisioner/go.sum b/modules/mesh-provisioner/go.sum index bb6beaa1..a2b8e018 100644 --- a/modules/mesh-provisioner/go.sum +++ b/modules/mesh-provisioner/go.sum @@ -1,2 +1,27 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021100209-cb54756c3865 h1:VLir9NZbyi1LdH1YwWDIVRuL+lDC+AYlm6FZPlvmlm4= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021100209-cb54756c3865/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 h1:lKU3HjNXGiVbLW2hUgFjU/TMTUYMlNnnmZxG5T7biQA= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= +github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= +github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= +github.com/google/go-tpm v0.9.8/go.mod h1:h9jEsEECg7gtLis0upRBQU+GhYVH6jMjrFxI8u6bVUY= +github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA= +github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI= +github.com/minio/highwayhash v1.0.4 h1:asJizugGgchQod2ja9NJlGOWq4s7KsAWr5XUc9Clgl4= +github.com/minio/highwayhash v1.0.4/go.mod h1:GGYsuwP/fPD6Y9hMiXuapVvlIUEhFhMTh0rxU3ik1LQ= +github.com/nats-io/jwt/v2 v2.8.1 h1:V0xpGuD/N8Mi+fQNDynXohVvp7ZztevW5io8CUWlPmU= +github.com/nats-io/jwt/v2 v2.8.1/go.mod h1:nWnOEEiVMiKHQpnAy4eXlizVEtSfzacZ1Q43LIRavZg= +github.com/nats-io/nats-server/v2 v2.11.17 h1:GKEghcFK6A+aFx11Yf1LjgLC3txAwvyhnYzhBIQZA8I= +github.com/nats-io/nats-server/v2 v2.11.17/go.mod h1:B1sFVz4StNosQ903ak4N1G01Fl/9f8e06mXpFIE2K24= +github.com/nats-io/nats.go v1.54.0 h1:vsXoOxjHp/GmPUN+EcI7uOf/uB+iAP+kEsAFNQN0yzA= +github.com/nats-io/nats.go v1.54.0/go.mod h1:y+DZoD1oBOYfZTU681eTUiUjI0vbqYGixNVFHcjHJ0k= +github.com/nats-io/nkeys v0.4.16 h1:rd5oAuLOb8mnAycB0xleuEBNS1pVVnN0fv/FF34Eypg= +github.com/nats-io/nkeys v0.4.16/go.mod h1:llLgWoI0o4z/Q57q2R1kHfmocyhGV6VG/U18Glg1Afs= +github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= +github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= +golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M= +golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA= +golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo= +golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= +golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= +golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= diff --git a/modules/mesh-provisioner/module.json b/modules/mesh-provisioner/module.json index 650425b3..1aba09af 100644 --- a/modules/mesh-provisioner/module.json +++ b/modules/mesh-provisioner/module.json @@ -11,11 +11,36 @@ ] } ], + "uses": [ + "operator-channel" + ], + "reads": [ + "mesh-delivery-assigner.placement", + "mesh-delivery-assigner.wants", + "mesh-delivery-indexer.modules", + "mesh-delivery-indexer.manifests", + "mesh-delivery-builder.builds", + "node-engine.have", + "mesh-config.resolved", + "gitea.bound", + "keycloak.bound", + "postgres.bound", + "route-adapter.bound", + "mesh-vault.bound" + ], + "tools": [ + "provisioner_provisions", + "provisioner_plans", + "provisioner_compose", + "provisioner_decide" + ], "state": [ "registry", "needs", "applied", - "collect" + "collect", + "provisions", + "plans" ], "own-secrets": { "broker": "${dir:state}/broker" diff --git a/modules/mesh-self-check/.gitignore b/modules/mesh-self-check/.gitignore new file mode 100644 index 00000000..2c7bac8b --- /dev/null +++ b/modules/mesh-self-check/.gitignore @@ -0,0 +1,2 @@ +/mesh-self-check +/cmd/mesh-self-check/mesh-self-check diff --git a/modules/mesh-self-check/README.md b/modules/mesh-self-check/README.md new file mode 100644 index 00000000..9411474c --- /dev/null +++ b/modules/mesh-self-check/README.md @@ -0,0 +1,33 @@ +# mesh-self-check + +Holds the seat `mesh-self-check` (novox/hq ADR 0564 §6): the mesh's self-check, moved out of the controller. Go, one +binary (`cmd/mesh-self-check`), served by a runtime of its own on its own bus account (`runs-as`), so that its account +alone writes its buckets. + +## What it does + +Every five minutes (the first run a minute after it starts) it runs its registry of probes, each within thirty +seconds. A probe is one live invariant of a design, with its assertion as the controller had it, reading its facts on +the bus (the bus module's tools for the bus's own state). A probe whose facts have no home on the bus yet answers +`deferred`, naming them: never a pass. A probe that cannot run is said in the verdict at once, and written as a finding +of its own (`probe..failed`) when the next run cannot run it either. + +What a probe finds is written as a finding, one key `..` per fault, only when what it says changes, and +deleted when a run no longer sees it. The condition keeper (`mesh-conditions`) raises and clears conditions from them. +A finding one look can be wrong about (a consumer behind after a burst) is written when two runs in a row see it. + +Every run ends with the event `checked`, the body of the controller's `doctor-heartbeat`, which the watchman (S10) and +mesh-watcher on another machine hear: a self-check that stops checking is said through a path that does not pass +through it. + +## What it keeps + +- `mesh-self-check_findings`: what it finds wrong now. +- `mesh-self-check_verdict`: the key `last`, the last run whole. +- `mesh-self-check_verdicts`: every run (each probe's findings at most twenty), kept a week by the bucket's age. + +## Its verbs and tools + +- `mesh-self-check.doctor`: the last verdict; `run=true` runs now; `probe=` one probe whole. +- `mesh-self-check.probes`: the registry, with what each deferred probe lacks. +- `self_check_findings`, `self_check_verdicts`: the reads of its buckets. diff --git a/modules/mesh-self-check/cmd/mesh-self-check/main.go b/modules/mesh-self-check/cmd/mesh-self-check/main.go new file mode 100644 index 00000000..6d3d53f8 --- /dev/null +++ b/modules/mesh-self-check/cmd/mesh-self-check/main.go @@ -0,0 +1,74 @@ +// mesh-self-check: the holder of the seat mesh-self-check (novox/hq ADR 0564 §6). A Go bundle served by a runtime of +// its own, on its own bus account (`runs-as`), so that its account alone writes its buckets: `findings` (what it finds +// wrong now, which the condition keeper raises and clears conditions from), `verdict` (the last run) and `verdicts` +// (every run, kept a week). It runs the probes every five minutes and says `checked` at the end of each run. stdout is +// the MCP channel; what this module says, it says on stderr. +package main + +import ( + "context" + "encoding/json" + "fmt" + "os" + "time" + + stdio "git.novox.be/novox/mesh-sdk/go" +) + +func logf(format string, a ...any) { fmt.Fprintf(os.Stderr, format+"\n", a...) } + +// runtimeStore is the bus, through the runtime that launched this bundle. +type runtimeStore struct{} + +func (runtimeStore) Get(state, key string) ([]byte, bool, error) { + e, err := stdio.State(state).Get(key) + if err != nil || e == nil { + return nil, false, err + } + return e.Value, true, nil +} + +func (runtimeStore) Put(state, key string, value any) error { + _, err := stdio.State(state).Put(key, value) + return err +} + +func (runtimeStore) Delete(state, key string) error { return stdio.State(state).Delete(key) } + +func (runtimeStore) Keys(state string) ([]string, error) { return stdio.State(state).Keys() } + +// NewChecker is the self-check over the runtime. +func NewChecker() *Checker { + return &Checker{Store: runtimeStore{}, Registry: Registry(), Now: time.Now, Logf: logf, Host: os.Getenv("MESH_NODE"), + Enabled: enabledProbes, + Ask: func(key string, body any) (json.RawMessage, error) { return stdio.Ask(key, body) }, + Publish: stdio.SeatPublish} +} + +func main() { + c := NewChecker() + go func() { + <-stdio.Served() + c.Keep(context.Background()) + }() + if err := stdio.Serve("", Tools(c)); err != nil { + logf("%v", err) + os.Exit(1) + } +} + +// enabledProbes is the setting `probes`, read at each run so a change applies at the next; with it unreadable, the +// default list. +func enabledProbes() []string { + raw, err := stdio.Config("probes") + if err != nil { + logf("the setting probes cannot be read (%v); the default list runs", err) + return DefaultProbes + } + var list []string + if json.Unmarshal(raw, &list) != nil { + logf("the setting probes is not a list of probe ids (%s); the default list runs", raw) + return DefaultProbes + } + return list +} diff --git a/modules/mesh-self-check/cmd/mesh-self-check/manifest_test.go b/modules/mesh-self-check/cmd/mesh-self-check/manifest_test.go new file mode 100644 index 00000000..e288898c --- /dev/null +++ b/modules/mesh-self-check/cmd/mesh-self-check/manifest_test.go @@ -0,0 +1,101 @@ +package main + +import ( + "encoding/json" + "os" + "path/filepath" + "slices" + "testing" +) + +type manifest struct { + Module string `json:"module"` + RunsAs string `json:"runs-as"` + Claims []struct { + Name string `json:"name"` + Scope string `json:"scope"` + Serves []string `json:"serves"` + } `json:"claims"` + Emits []string `json:"emits"` + Invokes []string `json:"invokes"` + Reads []string `json:"reads"` + State []json.RawMessage `json:"state"` +} + +func readManifest(t *testing.T) manifest { + t.Helper() + raw, err := os.ReadFile(filepath.Join("..", "..", "module.json")) + if err != nil { + t.Fatal(err) + } + var m manifest + if err := json.Unmarshal(raw, &m); err != nil { + t.Fatal(err) + } + return m +} + +// The manifest says what the code does: it claims the seat mesh-self-check once for the mesh with the verbs it serves +// (ADR 0564 §1, §6), runs on its own account, says checked, asks only what its probes ask, keeps a week of verdicts, +// and reads no bucket of the controller's. Every state it keeps has a tool that reads it, and every act it takes by +// itself a tool that starts it by hand (ADR 0564 §12). +func TestTheManifestSaysWhatTheCodeDoes(t *testing.T) { + m := readManifest(t) + if m.Module != Module || m.RunsAs != Module { + t.Errorf("module %q runs as %q", m.Module, m.RunsAs) + } + if len(m.Claims) != 1 || m.Claims[0].Name != Seat || m.Claims[0].Scope != "mesh" || !slices.Equal(m.Claims[0].Serves, Verbs) { + t.Errorf("claims %+v, want %s at mesh scope serving %v", m.Claims, Seat, Verbs) + } + // checked is the seat's event, said on the seat's subject: a module lists no event of a seat it claims. + if len(m.Emits) != 0 { + t.Errorf("emits %v", m.Emits) + } + if !slices.Equal(m.Invokes, Invokes) { + t.Errorf("invokes %v, want what the probes ask, %v", m.Invokes, Invokes) + } + if !slices.Equal(m.Reads, Reads) { + t.Errorf("reads %v, want what the probes read, %v", m.Reads, Reads) + } + states := map[string]int{} + for _, s := range m.State { + var d struct { + Name string `json:"name"` + TTL int `json:"ttl-seconds"` + } + if json.Unmarshal(s, &d.Name) != nil { + _ = json.Unmarshal(s, &d) + } + states[d.Name] = d.TTL + } + if states[VerdictsState] != 7*24*60*60 { + t.Errorf("verdicts are kept %d s, want a week", states[VerdictsState]) + } + // ADR 0564 §12: each state with the tool that reads it, each act with the tool that starts it. + readBy := map[string]string{FindingsState: FindingsTool, VerdictState: DoctorTool, VerdictsState: VerdictsTool, + "the run": DoctorTool + " run=true"} + tools := map[string]bool{} + for _, tool := range Tools(&Checker{}) { + tools[tool.Name] = true + } + for state := range states { + if _, ok := readBy[state]; !ok { + t.Errorf("the state %s has no tool that reads it", state) + } + } + for what, tool := range readBy { + name, _, _ := cutSpace(tool) + if !tools[name] { + t.Errorf("%s is read or started by %s, which is not served", what, tool) + } + } +} + +func cutSpace(s string) (string, string, bool) { + for i, r := range s { + if r == ' ' { + return s[:i], s[i+1:], true + } + } + return s, "", false +} diff --git a/modules/mesh-self-check/cmd/mesh-self-check/observation.go b/modules/mesh-self-check/cmd/mesh-self-check/observation.go new file mode 100644 index 00000000..71209ce9 --- /dev/null +++ b/modules/mesh-self-check/cmd/mesh-self-check/observation.go @@ -0,0 +1,142 @@ +package main + +// What a probe observes, and the finding it is written as (novox/hq ADR 0564 §2). A probe says what it sees as an +// Observation, the controller's observation: its token, its evidence in one line, and whether one look can be wrong +// about it. It is written into the bucket `findings` as mesh-sdk's health.Finding, whose key is the controller's key +// for the same condition, so the condition the keeper raises from it and the one the controller raised share one key. + +import ( + "encoding/json" + "time" + + "git.novox.be/novox/mesh-sdk/go/health" +) + +// The severities and scopes, as mesh-sdk's health names them. +const ( + Urgent = health.Urgent + Warning = health.Warning + ScopeBus = health.ScopeBus + ScopeCore = health.ScopeCore + ScopeProbe = health.ScopeProbe + ScopeMesh = health.ScopeMesh +) + +// Subject is what a finding is about. +type Subject = health.Subject + +// Observation is one thing a probe sees wrong now. +type Observation struct { + Kind string + Token string + Subject Subject + Severity health.Severity + Summary string + Said string + Headline string + Explanation string + Resolved string + Needs string + // Source is the probe that saw it, or the self-check itself for a probe that could not run. + Source string + // Confirm says one look can be wrong about it: written only when two runs in a row see it (issue 277). + Confirm bool +} + +// Key is where it is kept: `..`, the controller's key for the same condition. +func (o Observation) Key() string { + token := o.Token + if token == "" { + token = o.Kind + } + return health.Key(o.Subject.Scope, o.Subject.ID, token) +} + +// Finding is the observation as written by run at, its evidence the one line it said. +func (o Observation) Finding(run string, at time.Time) health.Finding { + f := health.Finding{Key: o.Key(), Kind: o.Kind, Subject: o.Subject, Severity: o.Severity, Summary: o.Summary, + Headline: o.Headline, Explanation: o.Explanation, Resolved: o.Resolved, Needs: o.Needs, Source: o.Source, + Observer: Module, Run: run, At: at.UTC(), Evidence: []health.Evidence{}} + if o.Token != "" && o.Token != o.Kind { + f.Token = o.Token + } + said := o.Said + if said == "" { + said = o.Summary + } + f.Evidence = append(f.Evidence, health.Evidence{At: at.UTC(), Said: said}) + return f +} + +// says is what a finding says, its times and the run that wrote it aside: two findings that say the same are one write +// (ADR 0289 §1). +func says(f health.Finding) string { + f.At, f.Run, f.V = time.Time{}, "", 0 + ev := make([]health.Evidence, len(f.Evidence)) + for i, e := range f.Evidence { + ev[i] = health.Evidence{Said: e.Said} + } + f.Evidence = ev + raw, _ := json.Marshal(f) + return string(raw) +} + +// worded gives a finding the operator's words registered for its kind, where it says none of its own. +func worded(f Observation) Observation { + w, ok := words[f.Kind] + if !ok { + return f + } + if f.Headline == "" { + f.Headline = w.Headline + } + if f.Explanation == "" { + f.Explanation = w.Explanation + } + if f.Resolved == "" { + f.Resolved = w.Resolved + } + if f.Needs == "" { + f.Needs = w.Needs + } + return f +} + +type plain struct{ Headline, Explanation, Resolved, Needs string } + +// words are the controller's words for each kind the self-check raises (mesh-controller plain_words.go), the +// controller's name for itself replaced by the self-check's where the words spoke of it. +var words = map[string]plain{ + "consumer-lost": {Headline: "A listener on the bus is gone", + Explanation: "One of the mesh's listeners is no longer on the bus; the next send sets it up again.", + Resolved: "The listener is back"}, + "consumer-wrong": {Headline: "A listener on the bus is set up wrong", + Explanation: "One of the mesh's listeners on the bus is missing or not as defined; the next send sets it up again.", + Resolved: "The listener is set up right again"}, + "consumer-behind": {Headline: "Messages pile up for a listener", + Explanation: "One of the mesh's listeners on the bus is far behind, so what it handles happens late. A healer " + + "restarts it if it stays.", + Resolved: "The listener caught up"}, + "stream-wrong": {Headline: "Part of the bus's storage is wrong", + Needs: "check the machine the bus runs on; the details say what is missing.", + Explanation: "A store the mesh keeps on the bus is missing or not as defined, so what it keeps there may be lost.", + Resolved: "The bus's storage is right again"}, + "bus-reconnects": {Headline: "A client keeps losing the bus", + Explanation: "One of the mesh's clients lost its connection to the bus again and again in the last hour. While " + + "it reconnects, what it says and what it is asked waits.", + Resolved: "Resolved: the client stays connected"}, + "core-unhealthy": {Headline: "The bus is not healthy", + Needs: "check the machine the bus runs on; the details say what fails.", + Explanation: "The bus, which carries every message in the mesh, is not healthy. Instructions, builds and tool " + + "calls may fail until it recovers.", + Resolved: "The bus is healthy again"}, + "watchdogs-silent": {Headline: "The mesh's watchdogs stopped", + Needs: "restart the watchman if this stays, from the mesh MCP server.", + Explanation: "The watchdogs, which notice when something expected does not happen, do not answer, so missed " + + "signals are not noticed.", + Resolved: "The watchdogs run again"}, + "probe-failed": {Headline: "A check of the mesh could not run", + Explanation: "One of the mesh's own checks could not run, so what it watches is unknown for now. It runs again " + + "every few minutes.", + Resolved: "The check runs again"}, +} diff --git a/modules/mesh-self-check/cmd/mesh-self-check/probes.go b/modules/mesh-self-check/cmd/mesh-self-check/probes.go new file mode 100644 index 00000000..e4dbce45 --- /dev/null +++ b/modules/mesh-self-check/cmd/mesh-self-check/probes.go @@ -0,0 +1,453 @@ +package main + +// The registry (novox/hq ADR 0564 §6 and the probe table of its Context): the probes that move from the controller, +// each with its assertion unchanged, reading its facts where they live on the bus. A probe whose facts have no home +// on the bus yet answers deferred, naming them, and is never a pass (tracked by issue 1952). Not registered, as the +// table says: D1 (nodes compose their own; a refusal is the watchman's S21), D10 (a want ahead of its have is the +// watchman's pulse), D5, D9 and H-controller (they end with the controller), stuck (the watchman's own finding) and +// the bus step (its home is issue 1953's). + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "slices" + "sort" + "strings" + + "git.novox.be/novox/mesh-sdk/go/buscompose" +) + +// The tools a probe asks: the bus module's reads of its own server, and the watchdogs' seat. +const ( + StreamsTool = "nats.nats_streams" + ClosedTool = "nats.nats_closed_connections" + SignalsVerb = "seat:mesh-watchdogs.signals" + notOnTheBus = "the bus module's tools (nats_streams) answer this only once asked; " + storeOnly = "held only in the controller's store today" + behindBound = 1000 + reconnectMax = 12 +) + +// Invokes are the tools and seat verbs the probes ask, and Reads the states they read: the manifest's `invokes` and +// `reads`, which its test holds to these. +var ( + Invokes = []string{StreamsTool, ClosedTool, SignalsVerb, BannedVerb} + Reads = []string{PlacementRead, SeatHoldersRead, PresenceRead, HealthRead, ReadingsRead, GoodRead, ModulesRead, ManifestsRead} +) + +// Registry is the self-check's registry, in to-be 45's order. **The registry is the design's live form**: a probe +// that moves here is a row added, and the registry test holds it to ADR 0564's table. +func Registry() []Probe { + return []Probe{ + {ID: "D2", Asserts: "every holder of the mesh's resolver answers a machine name for IPv4, and NODATA for IPv6", + From: "issue 262", Kind: "resolver-wrong", + Deferred: "the machines' names and the resolver holders' addresses to ask: the overlay's addresses are " + storeOnly}, + {ID: "D3", Asserts: "every seat on record that serves verbs has a live holder that answers, on every machine that " + + "is heard from", From: "issues 208, 218", Kind: "holder-silent", + Deferred: "the seats on record with their holders per machine, and which machines are heard from: the seat " + + "records are " + storeOnly + ", and the watchman's presence has no read granted to this module"}, + {ID: "D4", Asserts: "every kept archive is held by a manifest", From: "issue 253", Kind: "archives-unheld", + Deferred: "the artifact store's kept archives with the manifests that hold them: the archive index is " + storeOnly}, + {ID: "D6", Asserts: "every durable consumer the mesh expects exists with its definition, and is near its " + + "stream's head", From: "issues 248, 266", Kind: "consumer-wrong", + Raises: []string{"consumer-lost", "consumer-behind"}, Run: probeConsumers}, + {ID: "D7", Asserts: "every stream the mesh defines exists with its definition", From: "issue 208", + Kind: "stream-wrong", Run: probeStreams}, + {ID: "D8", Asserts: "no address the mesh owns — a machine's private address or its endpoint — is in a ban list", + From: "issue 238", Kind: "own-address-banned", Run: probeBans}, + {ID: "D11", Asserts: "no provider holds a consumer retired more than thirty days without a person deciding its " + + "cleanup", From: "ADR 0230", Kind: "cleanup-waiting", + Deferred: "the retirement records of providers' consumers: " + storeOnly}, + {ID: "D12", Asserts: "every consumer of a provision that keeps its data is bound where it was last sent, or moves " + + "by a pin", From: "issue 273, ADR 0232", Kind: "binding-moved", + Deferred: "the bindings of provisions to their consumers: " + storeOnly}, + {ID: "D13", Asserts: "every item of data a machine declares is measured, is there, holds what it held, and is " + + "backed up within its bound", From: "issue 273, ADR 0233", Kind: "data-shrank", + Raises: []string{kindDataMissing, kindDataQuiet, kindBackupStale, kindArrayDegraded, kindProtectionMissing, + kindDataUnmeasured}, + Run: probeData}, + {ID: "D15", Asserts: "no user of the bus had its connection dropped more than twelve times in the last hour: the " + + "bus module's nats_closed_connections", From: "issue 327", Kind: "bus-reconnects", Run: probeReconnects}, + {ID: "DA", Asserts: "every machine that names an agent account has it judged, on its node-engine's newest " + + "statement, unable to become root without a person", From: "ADR 0266, ADR 0259 §8", + Kind: "agent-can-become-root", Run: probeAgentAccounts}, + {ID: "D-root", Asserts: "no agent can become root without a person on a machine where the router or a channel " + + "proving its sender runs", From: "ADR 0259 §8", Kind: "root-not-free", + Deferred: "the node-engines' statements of root and the tools that run as root: " + storeOnly}, + {ID: "D-fill", Asserts: "every module's bucket and log holds less than three quarters of its cap; one said " + + "filling is cleared once it holds less than seven tenths", From: "ADR 0297, issue 501", + Kind: "bucket-or-log-filling", Run: probeFill}, + {ID: "D-heard", Asserts: "no grant of a restricted stream names a module that composes nothing for it", + From: "ADR 0376 §2", Kind: "grant-composes-nothing", + Deferred: "the grants of restricted streams with what each machine composed for them: " + storeOnly}, + {ID: "DW", Asserts: "the watchdogs answer: the watchman holds the seat mesh-watchdogs and answers signals", + From: "ADR 0227 rule 6: the watchers are watched", Kind: "watchdogs-silent", Run: probeWatchdogs}, + {ID: "H-bus", Asserts: "every stream and durable consumer the mesh defines is on the bus, and a request crosses it " + + "to the bus's machine and back", From: "ADR 0236, to-be 45 §8", Kind: "core-unhealthy", Run: probeBusHealth}, + } +} + +// --- the bus's own state, as its module answers it --------------------------------------------------------------- + +// streamsAnswer is what nats_streams answers. +type streamsAnswer struct { + Detail []streamRow `json:"detail"` +} + +type streamRow struct { + Name string `json:"name"` + Subjects []string `json:"subjects"` + Retention string `json:"retention"` + MaxPerSubject int64 `json:"max_per_subject"` + Consumers []consumerRow `json:"consumer_detail"` +} + +type consumerRow struct { + Name string `json:"name"` + Kind string `json:"kind"` + Filters []string `json:"filters"` + Pending int64 `json:"pending"` + Unacked int64 `json:"unacknowledged"` + MaxDeliver int64 `json:"max_deliver"` +} + +// errNoStream is a stream the bus module says it does not hold. +var errNoStream = errors.New("no such stream") + +// streams asks the bus module for every stream, or for one with its consumers. +func (c *Checker) streams(stream string) ([]streamRow, error) { + args := map[string]any{} + if stream != "" { + args["stream"] = stream + } + raw, err := c.Ask(StreamsTool, args) + if err != nil { + if stream != "" && strings.Contains(err.Error(), "no stream called") { + return nil, errNoStream + } + return nil, fmt.Errorf("the bus module did not answer %s: %w", StreamsTool, err) + } + var a streamsAnswer + if err := json.Unmarshal(raw, &a); err != nil { + return nil, fmt.Errorf("the bus module answered %s with something unreadable: %w", StreamsTool, err) + } + return a.Detail, nil +} + +// consumerWords is a consumer as the mesh says it, with its name. +func consumerWords(c buscompose.Consumer) string { + return fmt.Sprintf("the consumer %s on %s", c.Name, c.Stream) +} + +// expectedConsumers are the durable consumers the mesh defines, by stream. +func expectedConsumers() map[string][]buscompose.Consumer { + out := map[string][]buscompose.Consumer{} + for _, c := range buscompose.MeshConsumers() { + out[c.Stream] = append(out[c.Stream], c) + } + return out +} + +// probeConsumers is D6: every durable consumer the mesh defines exists as defined, and is near its stream's head. +func probeConsumers(ctx context.Context, c *Checker) ([]Observation, error) { + var out []Observation + byStream := expectedConsumers() + streams := make([]string, 0, len(byStream)) + for s := range byStream { + streams = append(streams, s) + } + sort.Strings(streams) + for _, stream := range streams { + if ctx.Err() != nil { + return nil, ctx.Err() + } + rows, err := c.streams(stream) + held := map[string]consumerRow{} + switch { + case errors.Is(err, errNoStream): + case err != nil: + return nil, err + default: + for _, r := range rows { + for _, cr := range r.Consumers { + held[cr.Name] = cr + } + } + } + for _, want := range byStream[stream] { + who := want.Stream + "." + want.Name + have, ok := held[want.Name] + if !ok { + out = append(out, Observation{Kind: "consumer-lost", Token: "missing", Severity: Warning, + Subject: Subject{Scope: ScopeBus, ID: who}, + Summary: fmt.Sprintf("%s is not on the bus: what it delivers reaches nobody", consumerWords(want)), + Said: "no consumer " + want.Name + " on " + want.Stream}) + continue + } + if differs := consumerDiffers(want, have); differs != "" { + out = append(out, Observation{Token: "redefined", Severity: Warning, Subject: Subject{Scope: ScopeBus, ID: who}, + Summary: fmt.Sprintf("%s is not as the mesh defines it: %s", consumerWords(want), differs), + Said: differs}) + } + if want.Stream != "NODES" && have.Pending > behindBound { + // One look at a consumer a burst put behind is not one that stays behind: held for the next run, + // so healer H4 resets nothing on a single sample (novox/hq issue 277). + out = append(out, Observation{Kind: "consumer-behind", Token: "behind", Severity: Warning, Confirm: true, + Subject: Subject{Scope: ScopeBus, ID: who}, + Summary: fmt.Sprintf("%s is %d message(s) behind its stream's head", consumerWords(want), have.Pending), + Said: fmt.Sprintf("%d pending, %d handed out and not settled", have.Pending, have.Unacked)}) + } + } + } + return sorted(out), nil +} + +// consumerDiffers is what about a consumer on the bus is not as defined, of what the bus module says of it: its +// filters, its shape (push or pull) and how often it hands a message over. Empty when nothing is. +func consumerDiffers(want buscompose.Consumer, have consumerRow) string { + var differs []string + haveFilters := slices.Clone(have.Filters) + wantFilters := slices.Clone(want.Filters) + sort.Strings(haveFilters) + sort.Strings(wantFilters) + if !slices.Equal(haveFilters, wantFilters) { + differs = append(differs, fmt.Sprintf("filters %v, defined %v", haveFilters, wantFilters)) + } + if want.MaxDeliver != 0 && have.MaxDeliver != int64(want.MaxDeliver) { + differs = append(differs, fmt.Sprintf("hands a message over %d times, defined %d", have.MaxDeliver, want.MaxDeliver)) + } + if wantPush, havePush := want.Push || want.Queue != "", have.Kind == "push"; wantPush != havePush { + differs = append(differs, "delivers by the other shape (push or pull) than defined") + } + return strings.Join(differs, "; ") +} + +// probeStreams is D7: every stream the mesh defines exists as defined. +func probeStreams(ctx context.Context, c *Checker) ([]Observation, error) { + rows, err := c.streams("") + if err != nil { + return nil, err + } + held := map[string]streamRow{} + for _, r := range rows { + held[r.Name] = r + } + var out []Observation + for _, s := range buscompose.MeshStreams() { + have, ok := held[s.Name] + if !ok { + out = append(out, Observation{Token: "stream-missing", Severity: Urgent, Subject: Subject{Scope: ScopeBus, ID: s.Name}, + Summary: fmt.Sprintf("the stream %s is not on the bus: %s", s.Name, firstLine(s.Why)), + Said: "no stream " + s.Name}) + continue + } + if differs := streamDiffers(s, have); differs != "" { + out = append(out, Observation{Token: "stream-redefined", Severity: Warning, Subject: Subject{Scope: ScopeBus, ID: s.Name}, + Summary: fmt.Sprintf("the stream %s is not as the mesh defines it: %s", s.Name, differs), + Said: differs}) + } + } + return sorted(out), nil +} + +// streamDiffers is what about a stream on the bus is not as defined, of what the bus module says of it: its +// subjects, its retention and how many it keeps per subject. Empty when nothing is. +func streamDiffers(want buscompose.Stream, have streamRow) string { + var differs []string + haveSubjects := slices.Clone(have.Subjects) + wantSubjects := slices.Clone(want.Subjects) + sort.Strings(haveSubjects) + sort.Strings(wantSubjects) + if !slices.Equal(haveSubjects, wantSubjects) { + differs = append(differs, fmt.Sprintf("subjects %v, defined %v", haveSubjects, wantSubjects)) + } + retention, perSubject := "limits", int64(want.MaxMsgsPerSubject) + switch want.Retention { + case buscompose.RetentionWorkQueue: + retention = "workqueue" + case buscompose.RetentionLastPerSubject: + perSubject = 1 + } + if have.Retention != "" && have.Retention != retention { + differs = append(differs, fmt.Sprintf("keeps by %s, defined %s", have.Retention, retention)) + } + if perSubject != 0 && have.MaxPerSubject != perSubject { + differs = append(differs, fmt.Sprintf("keeps %d per subject, defined %d", have.MaxPerSubject, perSubject)) + } + return strings.Join(differs, "; ") +} + +// --- D15 ----------------------------------------------------------------------------------------------------- + +// closedAnswer is what nats_closed_connections answers. +type closedAnswer struct { + Hours float64 `json:"hours"` + Reaches bool `json:"reaches"` + Users []struct { + User string `json:"user"` + Closed int `json:"closed"` + Dropped int `json:"dropped"` + Names []struct { + Name string `json:"name"` + Dropped int `json:"dropped"` + Reasons map[string]int `json:"reasons"` + } `json:"names"` + } `json:"users"` +} + +// probeReconnects is D15: each user of the bus whose connections were dropped more than twelve times an hour. +func probeReconnects(_ context.Context, c *Checker) ([]Observation, error) { + raw, err := c.Ask(ClosedTool, map[string]any{"hours": 1}) + if err != nil { + return nil, fmt.Errorf("the bus module did not answer %s: %w", ClosedTool, err) + } + var read closedAnswer + if err := json.Unmarshal(raw, &read); err != nil { + return nil, fmt.Errorf("the bus module answered %s with something unreadable: %w", ClosedTool, err) + } + hours := read.Hours + if hours <= 0 { + hours = 1 + } + var out []Observation + for _, u := range read.Users { + if u.User == "" || strings.HasPrefix(u.User, "(") || float64(u.Dropped)/hours <= reconnectMax { + continue // refused before it logged in is nobody's reconnect loop + } + reasons := map[string]int{} + var names []string + for _, n := range u.Names { + if n.Dropped == 0 { + continue + } + names = append(names, fmt.Sprintf("%q ×%d", n.Name, n.Dropped)) + for r, k := range n.Reasons { + if r != "Client Closed" { + reasons[r] += k + } + } + } + var why []string + for r, k := range reasons { + why = append(why, fmt.Sprintf("%s ×%d", r, k)) + } + sort.Strings(why) + partial := "" + if !read.Reaches { + partial = " (at least: the server's record of closed connections does not reach back the whole hour)" + } + who, machine := busUserWords(u.User) + out = append(out, Observation{Kind: "bus-reconnects", Severity: Warning, + Subject: Subject{Scope: ScopeBus, ID: u.User, Machine: machine}, + Summary: fmt.Sprintf("the bus dropped %s's connection %d times in the last hour%s, more than %d: a client "+ + "reconnecting in a loop", u.User, u.Dropped, partial, reconnectMax), + Said: fmt.Sprintf("%d of %d closed connections dropped in %.0f h; by name %s; why %s", u.Dropped, u.Closed, + hours, strings.Join(names, ", "), strings.Join(why, ", ")), + Headline: capital(who) + " keeps losing the bus", + Explanation: capital(fmt.Sprintf("%s lost its connection to the bus %d times in the last hour and connected "+ + "again each time. While it reconnects, what it says and what it is asked waits.", who, u.Dropped)), + Resolved: "Resolved: " + who + " stays connected"}) + } + return sorted(out), nil +} + +// busUserWords is a bus user as the operator says it, and the machine it is on. +func busUserWords(user string) (string, string) { + switch { + case user == "controller": + return "the controller", "" + case strings.HasPrefix(user, "node."): + m := strings.TrimPrefix(user, "node.") + return "the node-engine on " + m, m + } + if m, module, ok := strings.Cut(user, "."); ok { + if module == "node-tools" { + return "the tool runner on " + m, m + } + return module + " on " + m, m + } + return "a client of the bus", "" +} + +// --- DW and H-bus -------------------------------------------------------------------------------------------- + +// probeWatchdogs is DW: the watchdogs answer. The self-check and the watchdogs watch each other: S10, the +// watchman's pulse on `checked`, is the other half. +func probeWatchdogs(_ context.Context, c *Checker) ([]Observation, error) { + if _, err := c.Ask(SignalsVerb, map[string]any{}); err != nil { + return []Observation{{Token: "silent", Severity: Urgent, Subject: Subject{Scope: ScopeCore, ID: "watchdogs"}, + Summary: "the watchdogs do not answer: the watchman holding mesh-watchdogs did not answer signals, so no " + + "late signal is being said", + Said: firstLine(err.Error())}}, nil + } + return nil, nil +} + +// probeBusHealth is H-bus: every stream and durable consumer the mesh defines is on the bus, and a request crosses it +// to the bus module's machine and back. The question to the bus module is that request: an answer is its round trip, +// and no answer is no pass. +func probeBusHealth(ctx context.Context, c *Checker) ([]Observation, error) { + rows, err := c.streams("") + if err != nil { + return nil, fmt.Errorf("no request crossed the bus to its module and came back: %w", err) + } + held := map[string]bool{} + for _, r := range rows { + held[r.Name] = true + } + var problems []string + for _, s := range buscompose.MeshStreams() { + if !held[s.Name] { + problems = append(problems, "the stream "+s.Name+" is missing") + } + } + byStream := expectedConsumers() + streams := make([]string, 0, len(byStream)) + for s := range byStream { + streams = append(streams, s) + } + sort.Strings(streams) + for _, stream := range streams { + if !held[stream] { + continue // said above + } + if ctx.Err() != nil { + return nil, ctx.Err() + } + detail, err := c.streams(stream) + if err != nil && !errors.Is(err, errNoStream) { + return nil, err + } + have := map[string]bool{} + for _, r := range detail { + for _, cr := range r.Consumers { + have[cr.Name] = true + } + } + for _, want := range byStream[stream] { + if !have[want.Name] { + problems = append(problems, consumerWords(want)+" is missing") + } + } + } + if len(problems) == 0 { + return nil, nil + } + return []Observation{{Token: "unhealthy", Severity: Urgent, Subject: Subject{Scope: ScopeBus, ID: "mesh"}, + Summary: "the bus is not healthy: " + strings.Join(problems, "; "), Said: strings.Join(problems, "; ")}}, nil +} + +// sorted is findings in a stated order, so two runs over one mesh say the same. +func sorted(out []Observation) []Observation { + sort.SliceStable(out, func(i, j int) bool { return out[i].Key() < out[j].Key() }) + return out +} + +func capital(s string) string { + if s == "" { + return s + } + return strings.ToUpper(s[:1]) + s[1:] +} diff --git a/modules/mesh-self-check/cmd/mesh-self-check/probes_data.go b/modules/mesh-self-check/cmd/mesh-self-check/probes_data.go new file mode 100644 index 00000000..98d8146f --- /dev/null +++ b/modules/mesh-self-check/cmd/mesh-self-check/probes_data.go @@ -0,0 +1,392 @@ +package main + +// D13, the data the modules declare (novox/hq ADR 0233, ADR 0571 clause 3, ADR 0564 §6): every item of data a machine +// declares is measured, is there, holds what it held, and is backed up within its bound. The controller composed what +// every machine declares and asked each machine's backup holder what it measured; the facts now live on the bus: the +// readings disk-load keeps (its state `readings`, with the item's seven-day peak), the last good night per item restic +// keeps (its state `good`), and the manifests the indexer holds (for the bounds an item declares). Each kind, key and +// word is the controller's (mesh-controller cmd/mesh-controller/data.go), so the condition the keeper raises from it is +// the one the controller raised. + +import ( + "context" + "encoding/json" + "fmt" + "sort" + "strings" + "time" + + "git.novox.be/novox/mesh-sdk/go/catalogue" + "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/health" +) + +// The states D13 reads (declared under `reads`). +const ( + ReadingsRead = health.ReadingsRead + GoodRead = health.GoodRead + ModulesRead = delivery.IndexerModule + ".modules" + ManifestsRead = delivery.IndexerModule + ".manifests" +) + +// The kinds D13 raises. +const ( + kindDataShrank = "data-shrank" + kindEmptyReplacement = "empty-replacement" + kindDataHeldTwice = "data-held-twice" + kindDataQuiet = "data-quiet" + kindBackupStale = "backup-stale" + kindDataUnmeasured = "data-unmeasured" + kindDataMissing = "data-missing" + kindArrayDegraded = "array-degraded" + kindProtectionMissing = "protection-missing" + kindCleanupWaiting = "cleanup-waiting" + shrinkWindowDays = 7 + cleanupAfter = 30 * 24 * time.Hour + shrinkFloor int64 = 16 << 20 +) + +// dataFacts is everything D13 judges: the readings, the last good nights, each module's manifest, and the machines whose +// backup holder was heard from. +type dataFacts struct { + readings []health.Reading + good map[string]time.Time // by health.GoodKey + manifests map[string]catalogue.Manifest + // unmeasured are the machines that hold node-backup, are heard from, and have no reading of anything. + unmeasured []string + now time.Time +} + +// dataFacts reads them from the bus. +func (c *Checker) dataFacts() (dataFacts, error) { + f := dataFacts{good: map[string]time.Time{}, manifests: map[string]catalogue.Manifest{}, now: c.Now()} + keys, err := c.Store.Keys(ReadingsRead) + if err != nil { + return f, fmt.Errorf("disk-load's readings cannot be read: %w", err) + } + byNode := map[string]int{} + for _, k := range keys { + raw, ok, err := c.Store.Get(ReadingsRead, k) + if err != nil { + return f, fmt.Errorf("the reading %s cannot be read: %w", k, err) + } + if !ok { + continue + } + r, err := health.DecodeReading(raw) + if err != nil { + continue + } + f.readings = append(f.readings, r) + byNode[r.Node]++ + } + gkeys, err := c.Store.Keys(GoodRead) + if err != nil { + return f, fmt.Errorf("restic's last good nights cannot be read: %w", err) + } + for _, k := range gkeys { + raw, ok, err := c.Store.Get(GoodRead, k) + if err != nil { + return f, fmt.Errorf("the last good night %s cannot be read: %w", k, err) + } + if g, derr := health.DecodeGoodRow(raw); ok && derr == nil && !g.LastGood.IsZero() { + f.good[health.GoodKey(g.Node, g.Module, g.Item)] = g.LastGood + } + } + mods, err := c.Store.Keys(ModulesRead) + if err != nil { + return f, fmt.Errorf("the indexer's modules cannot be read: %w", err) + } + for _, name := range mods { + raw, ok, err := c.Store.Get(ModulesRead, name) + if err != nil || !ok { + continue + } + // Only the manifest's digest of the indexer's module, and the manifest's text: what the bounds are read from. + var m struct { + Manifest string `json:"manifest"` + } + if json.Unmarshal(raw, &m) != nil || m.Manifest == "" { + continue + } + mraw, ok, err := c.Store.Get(ManifestsRead, m.Manifest) + if err != nil || !ok { + continue + } + var doc struct { + JSON string `json:"manifest"` + } + if json.Unmarshal(mraw, &doc) != nil { + continue + } + var man catalogue.Manifest + if json.Unmarshal([]byte(doc.JSON), &man) == nil { + f.manifests[name] = man + } + } + heard, err := c.heard() + if err != nil { + return f, err + } + holders, err := c.holdersOf("node-backup") + if err != nil { + return f, err + } + for _, n := range holders { + if heard[n] && byNode[n] == 0 { + f.unmeasured = append(f.unmeasured, n) + } + } + return f, nil +} + +// probeData is D13. +func probeData(ctx context.Context, c *Checker) ([]Observation, error) { + f, err := c.dataFacts() + if err != nil { + return nil, err + } + return dataFindings(f), nil +} + +func severityOf(class string) health.Severity { + if class == catalogue.ClassIrreplaceable { + return Urgent + } + return Warning +} + +func keyOfItem(machine, module, item string) string { return machine + "." + module + "." + item } + +func comparable(precision string) bool { + return precision == "" || precision == "exact" || strings.HasPrefix(precision, "dataset ") +} + +func datasetOf(precision string) string { + rest, ok := strings.CutPrefix(precision, "dataset ") + if !ok { + return "" + } + name, _, _ := strings.Cut(rest, ":") + return name +} + +func orUnknownPath(p string) string { + if p == "" { + return "a path its backup holder never named" + } + return p +} + +func sizeWords(size *int64) string { + if size == nil || *size < 0 { + return "size unknown" + } + b := float64(*size) + for _, unit := range []string{"B", "KB", "MB", "GB"} { + if b < 1024 || unit == "GB" { + if unit == "B" { + return fmt.Sprintf("%d B", *size) + } + return fmt.Sprintf("%.1f %s", b, unit) + } + b /= 1024 + } + return "" +} + +// firstSeen is when an item was first measured on its node (the reading keeps it); for a reading written before it did, +// the oldest day its seven-day peak is made of, else its measurement: a lower bound on how long it has been declared. +func firstSeen(r health.Reading, now time.Time) time.Time { + if !r.FirstSeen.IsZero() { + return r.FirstSeen + } + if len(r.Days) > 0 { + if t, err := time.Parse("2006-01-02", r.Days[0].Day); err == nil { + return t + } + } + if r.MeasuredAt != nil { + return *r.MeasuredAt + } + return now +} + +func obs(kind string, id, machine string, sev health.Severity, summary, said string) Observation { + return Observation{Kind: kind, Token: kind, Severity: sev, Summary: summary, Said: said, + Subject: Subject{Scope: ScopeMachine, ID: id, Machine: machine}} +} + +// dataFindings is every condition the data on record raises now: a function of what is known, as the controller's was. +func dataFindings(f dataFacts) []Observation { + var out []Observation + arrays := map[string][]health.Reading{} + type shrunk struct { + machine, dataset, class string + size, peak int64 + items []string + } + shrunkDatasets := map[string]shrunk{} + byItem := map[string][]health.Reading{} + retiredCopies := map[string][]health.Reading{} + for _, m := range f.unmeasured { + o := obs(kindDataUnmeasured, m, m, Warning, fmt.Sprintf("%s's backup holder did not say what it measured of the data "+ + "declared there, so nothing about that data is known", m), "disk-load keeps no reading of this machine's data") + o.Confirm = true + out = append(out, o) + } + rs := append([]health.Reading(nil), f.readings...) + sort.Slice(rs, func(i, j int) bool { + return keyOfItem(rs[i].Node, rs[i].Module, rs[i].ID) < keyOfItem(rs[j].Node, rs[j].Module, rs[j].ID) + }) + for _, r := range rs { + class := r.Class + item, declared := f.manifests[r.Module].DataItem(r.ID) + id := keyOfItem(r.Node, r.Module, r.ID) + if !r.RetiredAt.IsZero() { + retiredCopies[r.Module+"/"+r.ID] = append(retiredCopies[r.Module+"/"+r.ID], r) + if f.now.Sub(r.RetiredAt) > cleanupAfter && catalogue.Watched(class) { + out = append(out, obs(kindCleanupWaiting, id, r.Node, Warning, + fmt.Sprintf("%s of %s on %s (%s, %s) has been retired %d days — kept where it was since %s; "+ + "`cleanup delete %s %s %s --why …` once a person has decided, or assign %s there again", + r.ID, r.Module, r.Node, r.Class, sizeWords(r.SizeBytes), int(f.now.Sub(r.RetiredAt).Hours()/24), + r.RetiredWhy, r.Node, r.Module, r.ID, r.Module), "kept at "+orUnknownPath(r.Path))) + out[len(out)-1].Token = "cleanup" + } + continue + } + byItem[r.Module+"/"+r.ID] = append(byItem[r.Module+"/"+r.ID], r) + if !catalogue.Watched(class) { + continue + } + severity := severityOf(class) + if r.Redundancy != nil { + where := r.Node + "/" + r.Redundancy.Kind + ":" + r.Redundancy.Where + arrays[where] = append(arrays[where], r) + } else if declared && item.Redundancy != "" && r.MeasuredAt != nil && r.Error == "" { + out = append(out, obs(kindProtectionMissing, id, r.Node, severity, + fmt.Sprintf("%s of %s on %s (%s) is said to be protected by the redundancy of the storage it is on, "+ + "and it is on nothing the backup holder can read as redundant: it has no protection the mesh can see", + r.ID, r.Module, r.Node, class), + orUnknownPath(r.Path)+" is on no redundant storage the backup holder can read")) + } + peak := r.Peak + shrank := peak != nil && r.SizeBytes != nil && comparable(r.Precision) && *r.SizeBytes*2 < *peak && *peak-*r.SizeBytes >= shrinkFloor + switch { + case r.Error != "" && strings.Contains(r.Error, "does not exist"): + out = append(out, obs(kindDataMissing, id, r.Node, severity, + fmt.Sprintf("%s of %s on %s (%s) is gone: where it was declared, nothing exists any more", r.ID, r.Module, r.Node, class), + orUnknownPath(r.Path)+" does not exist: "+firstLine(r.Error))) + case shrank && datasetOf(r.Precision) != "": + // Several items on one dataset share its size: one condition for the dataset, as loud as the most + // precious item on it. + k := r.Node + "/" + datasetOf(r.Precision) + ds := shrunkDatasets[k] + ds.machine, ds.dataset, ds.size, ds.peak = r.Node, datasetOf(r.Precision), *r.SizeBytes, *peak + ds.class = catalogue.StricterClass(ds.class, class) + ds.items = append(ds.items, r.Module+"/"+r.ID) + shrunkDatasets[k] = ds + case shrank: + out = append(out, obs(kindDataShrank, id, r.Node, severity, + fmt.Sprintf("%s of %s on %s (%s) shrank to %s from %s within %d days — more than half of what it "+ + "held is gone. If that was meant, silence this with why; if not, `node-backup.restore` puts the last "+ + "good copy beside it", r.ID, r.Module, r.Node, class, sizeWords(r.SizeBytes), sizeWords(peak), shrinkWindowDays), "")) + } + if !declared { + continue + } + if within := item.ActiveWithin(); within > 0 && r.LastWrite != nil && f.now.Sub(*r.LastWrite) > within { + out = append(out, obs(kindDataQuiet, id, r.Node, severity, + fmt.Sprintf("%s of %s on %s is written all the time, and has not been since %s (its bound is %s): "+ + "whatever writes it has stopped", r.ID, r.Module, r.Node, r.LastWrite.UTC().Format(time.RFC3339), within), "")) + } + // A backup is required of what is irreplaceable and copied; of what is valuable it is the standard + // plan, said only where the machine was measured — where a holder is there to take it. + if item.BackedUp() && (class == catalogue.ClassIrreplaceable || r.MeasuredAt != nil) { + within := item.BackupWithin() + last, has := f.good[health.GoodKey(r.Node, r.Module, r.ID)] + switch { + case !has && f.now.Sub(firstSeen(r, f.now)) > within: + out = append(out, obs(kindBackupStale, id, r.Node, severity, + fmt.Sprintf("%s of %s on %s is %s and has no good backup on record, %s after it was first "+ + "declared — is node-backup held there, and do its nights succeed? (`node-backup.backed-up`)", + r.ID, r.Module, r.Node, class, f.now.Sub(firstSeen(r, f.now)).Round(time.Hour)), "")) + case has && f.now.Sub(last) > within: + out = append(out, obs(kindBackupStale, id, r.Node, severity, + fmt.Sprintf("%s of %s on %s is %s and its newest good backup is from %s, older than its bound "+ + "of %s", r.ID, r.Module, r.Node, class, last.UTC().Format(time.RFC3339), within), "")) + } + } + } + keys := make([]string, 0, len(shrunkDatasets)) + for k := range shrunkDatasets { + keys = append(keys, k) + } + sort.Strings(keys) + for _, k := range keys { + ds := shrunkDatasets[k] + out = append(out, obs(kindDataShrank, ds.machine+".dataset."+strings.ReplaceAll(ds.dataset, "/", "-"), ds.machine, + severityOf(ds.class), fmt.Sprintf("the dataset on %s that holds %s shrank to %s from %s within %d days — more than "+ + "half of what it held is gone", ds.machine, strings.Join(ds.items, ", "), sizeWords(&ds.size), sizeWords(&ds.peak), + shrinkWindowDays), "the dataset "+ds.dataset)) + } + // The redundant storage watched data is on: one condition per array, as loud as the most precious item on it — + // the array, not each item, is what degrades. + wheres := make([]string, 0, len(arrays)) + for w := range arrays { + wheres = append(wheres, w) + } + sort.Strings(wheres) + for _, where := range wheres { + items := arrays[where] + red := items[0].Redundancy + if red.Healthy != nil && *red.Healthy { + continue + } + class, machine := catalogue.ClassValuable, items[0].Node + var names []string + for _, r := range items { + class = catalogue.StricterClass(class, r.Class) + names = append(names, r.Module+"/"+r.ID) + } + state := "could not be read" + if red.Healthy != nil { + state = "is NOT healthy" + } + out = append(out, obs(kindArrayDegraded, machine+".array."+strings.NewReplacer("/", "-", ":", "-").Replace(red.Kind+"-"+red.Where), + machine, severityOf(class), fmt.Sprintf("the %s storage on %s that protects %s %s", red.Kind, machine, + strings.Join(names, ", "), state), fmt.Sprintf("the %s storage %s %s: %s", red.Kind, red.Where, state, firstLine(red.Said)))) + } + // The same item on several machines: a copy that is in use and far smaller than one kept retired elsewhere is an + // empty replacement. Only against a retired copy — a module running on two machines on purpose keeps two different + // sets of data. + items := make([]string, 0, len(byItem)) + for k := range byItem { + items = append(items, k) + } + sort.Strings(items) + for _, key := range items { + for _, a := range byItem[key] { + if a.SizeBytes == nil || !catalogue.Watched(a.Class) || !comparable(a.Precision) { + continue + } + for _, o := range retiredCopies[key] { + if o.Node == a.Node || o.SizeBytes == nil || !comparable(o.Precision) || !replacedByLess(*a.SizeBytes, *o.SizeBytes) { + continue + } + ob := obs(kindEmptyReplacement, keyOfItem(a.Node, a.Module, a.ID), a.Node, severityOf(a.Class), + fmt.Sprintf("%s of %s on %s holds %s, and the copy %s kept on %s holds %s: %s is running on "+ + "an empty replacement of its data. Move the data, or assign it back where its data is", + a.ID, a.Module, a.Node, sizeWords(a.SizeBytes), o.Module, o.Node, sizeWords(o.SizeBytes), a.Module), "") + ob.Subject.Also = []string{o.Node} + out = append(out, ob) + break + } + } + } + return out +} + +// replacedByLess is whether a copy in use is an empty replacement of a copy kept elsewhere: less than half of it, and at +// least shrinkFloor less. +func replacedByLess(inUse, kept int64) bool { return inUse*2 < kept && kept-inUse >= shrinkFloor } diff --git a/modules/mesh-self-check/cmd/mesh-self-check/probes_fill.go b/modules/mesh-self-check/cmd/mesh-self-check/probes_fill.go new file mode 100644 index 00000000..a868ab33 --- /dev/null +++ b/modules/mesh-self-check/cmd/mesh-self-check/probes_fill.go @@ -0,0 +1,112 @@ +package main + +// D-fill (novox/hq ADR 0297 §6, issue 501): a module's bucket or log filling toward its cap is said before it is full. +// Every module's bucket (`KV__`) and log (`LOG__`) is read from the bus module's +// nats_streams, which answers each stream's size and its cap (max_bytes, mesh-catalog cab031bf). Said at three +// quarters of its cap, cleared below seven tenths: between the two a finding already written is kept, so one at its +// threshold is not raised and cleared on every run. The kind, key and words are the controller's +// (mesh-controller bucket_log_fill.go). + +import ( + "context" + "encoding/json" + "fmt" + "sort" + "strings" + + "git.novox.be/novox/mesh-sdk/go/buscompose" +) + +// The fill thresholds, in percent of a bucket's or log's cap. +const ( + fillRaiseAt = 75 + fillClearBelow = 70 +) + +// fillRow is what nats_streams answers of a stream's size and cap. +type fillRow struct { + Name string `json:"name"` + Bytes uint64 `json:"bytes"` + MaxBytes uint64 `json:"max_bytes"` +} + +// bucketOrLog is a module's bucket or log by its stream: the module, `bucket` or `log`, and its local name. False for +// a stream that is no module's bucket or log (the mesh's own streams). +func bucketOrLog(stream string) (module, kind, name string, ok bool) { + var rest string + switch { + case strings.HasPrefix(stream, "KV_"): + kind, rest = "bucket", strings.TrimPrefix(stream, "KV_") + case strings.HasPrefix(stream, buscompose.LogStreamPrefix): + kind, rest = "log", strings.TrimPrefix(stream, buscompose.LogStreamPrefix) + default: + return "", "", "", false + } + module, name, ok = strings.Cut(rest, "_") + if !ok || module == "" || name == "" { + return "", "", "", false + } + return module, kind, name, true +} + +// mibWords is a size as a person reads it, in MiB with one decimal. +func mibWords(b uint64) string { return fmt.Sprintf("%.1f MiB", float64(b)/(1024*1024)) } + +// fillKey is where a bucket's or log's fill finding is kept. +func fillKey(stream string) string { + return Observation{Subject: Subject{Scope: ScopeBus, ID: stream}, Token: "filling"}.Key() +} + +// fillObservation is what one bucket's or log's fill says: a finding at fillRaiseAt or above, and, while its finding +// is written, at fillClearBelow or above too; nothing otherwise, which is what clears it. +func fillObservation(stream string, r fillRow, open bool) (Observation, bool) { + module, kind, name, ok := bucketOrLog(stream) + if !ok || r.MaxBytes == 0 { + return Observation{}, false // no module's, or with no cap: it cannot fill + } + // Compared in bytes, not rounded percent: 74.9% is not 75%. + atRaise := r.Bytes*100 >= r.MaxBytes*fillRaiseAt + aboveClear := r.Bytes*100 >= r.MaxBytes*fillClearBelow + if !atRaise && !(open && aboveClear) { + return Observation{}, false + } + pct := r.Bytes * 100 / r.MaxBytes + return Observation{Kind: "bucket-or-log-filling", Token: "filling", Severity: Warning, + Subject: Subject{Scope: ScopeBus, ID: stream}, + Summary: fmt.Sprintf("%s's %s %s holds %s of %s, %d%%: at its cap the bus refuses the module's next write", + module, kind, name, mibWords(r.Bytes), mibWords(r.MaxBytes), pct), + Said: fmt.Sprintf("module %s, %s %s (stream %s): %d of %d bytes, %d%%", module, kind, name, stream, r.Bytes, + r.MaxBytes, pct), + Headline: "A module's bucket or log on the bus is filling up", + Needs: "decide whether to raise its cap or have the module keep less.", + Explanation: "A bucket or log a module keeps on the bus holds three quarters of its cap or more. When it is " + + "full, the bus refuses what the module writes there.", + Resolved: "It has room again"}, true +} + +// probeFill is D-fill: every module's bucket and log on the bus judged against its cap. The bus module not answering +// is the probe failing to run, which clears nothing; a finding already written counts as open. +func probeFill(_ context.Context, c *Checker) ([]Observation, error) { + raw, err := c.Ask(StreamsTool, map[string]any{}) + if err != nil { + return nil, fmt.Errorf("the bus module did not answer %s: %w", StreamsTool, err) + } + var a struct { + Detail []fillRow `json:"detail"` + } + if err := json.Unmarshal(raw, &a); err != nil { + return nil, fmt.Errorf("the bus module answered %s with something unreadable: %w", StreamsTool, err) + } + written, err := c.written() + if err != nil { + return nil, fmt.Errorf("its own findings cannot be read, so a fill between the thresholds cannot be judged: %w", err) + } + sort.Slice(a.Detail, func(i, j int) bool { return a.Detail[i].Name < a.Detail[j].Name }) + var out []Observation + for _, r := range a.Detail { + if o, said := fillObservation(r.Name, r, written.has(fillKey(r.Name))); said { + out = append(out, o) + } + } + return sorted(out), nil +} diff --git a/modules/mesh-self-check/cmd/mesh-self-check/probes_mesh.go b/modules/mesh-self-check/cmd/mesh-self-check/probes_mesh.go new file mode 100644 index 00000000..a25fab0a --- /dev/null +++ b/modules/mesh-self-check/cmd/mesh-self-check/probes_mesh.go @@ -0,0 +1,219 @@ +package main + +// The probes of the mesh's machines (novox/hq ADR 0564 §6, issue 1952): each reads its facts where they live on the +// bus — placement (the seat mesh-placement's state), who holds which seat (its seat-holders), whether a node is heard +// from (the watchman's presence) — and asks each node's seat holder what only that node knows. Its assertion, kind, +// key and words are the controller's (mesh-controller probes.go), so the condition the keeper raises from it is the one +// the controller raised. + +import ( + "context" + "encoding/json" + "fmt" + "net" + "regexp" + "slices" + "sort" + "strings" + "sync" + + "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/health" +) + +// The states the machine probes read (declared under `reads`), and the verb D8 asks each node. +const ( + PlacementRead = delivery.AssignerModule + "." + "placement" + SeatHoldersRead = delivery.SeatHoldersRead + PresenceRead = delivery.WatchmanModule + ".presence" + BannedSeat = "node-intrusion-prevention" + BannedVerb = "seat:" + BannedSeat + ".banned" + ScopeMachine = health.ScopeMachine +) + +// placements is every node's placement, by node. +func (c *Checker) placements() (map[string]delivery.Placement, error) { + keys, err := c.Store.Keys(PlacementRead) + if err != nil { + return nil, fmt.Errorf("the placement cannot be read: %w", err) + } + out := map[string]delivery.Placement{} + for _, k := range keys { + raw, ok, err := c.Store.Get(PlacementRead, k) + if err != nil { + return nil, fmt.Errorf("the placement of %s cannot be read: %w", k, err) + } + if !ok { + continue + } + var p delivery.Placement + if err := json.Unmarshal(raw, &p); err != nil { + return nil, fmt.Errorf("the placement of %s is not readable: %w", k, err) + } + if p.Node == "" { + p.Node = k + } + out[p.Node] = p + } + return out, nil +} + +// heard is each node the watchman says is online. +func (c *Checker) heard() (map[string]bool, error) { + keys, err := c.Store.Keys(PresenceRead) + if err != nil { + return nil, fmt.Errorf("the watchman's presence cannot be read: %w", err) + } + out := map[string]bool{} + for _, k := range keys { + node, _, _, err := delivery.ParsePresenceKey(k) + if err != nil || node == "" { + continue + } + raw, ok, err := c.Store.Get(PresenceRead, k) + if err != nil { + return nil, fmt.Errorf("the presence of %s cannot be read: %w", node, err) + } + var p delivery.Presence + if ok && json.Unmarshal(raw, &p) == nil && p.Online { + out[node] = true + } + } + return out, nil +} + +// holdersOf is each node on which a module holds the node seat seat, sorted. +func (c *Checker) holdersOf(seat string) ([]string, error) { + keys, err := c.Store.Keys(SeatHoldersRead) + if err != nil { + return nil, fmt.Errorf("who holds which seat cannot be read: %w", err) + } + var out []string + for _, k := range keys { + if !strings.HasPrefix(k, seat+".") { + continue + } + raw, ok, err := c.Store.Get(SeatHoldersRead, k) + if err != nil { + return nil, fmt.Errorf("the holders of %s cannot be read: %w", k, err) + } + var sh delivery.SeatHolders + if !ok || json.Unmarshal(raw, &sh) != nil || sh.Seat != seat || sh.Node == "" || len(sh.Holders) == 0 { + continue + } + if !slices.Contains(out, sh.Node) { + out = append(out, sh.Node) + } + } + sort.Strings(out) + return out, nil +} + +// --- D8 ---------------------------------------------------------------------------------------------------------- + +// ipLike finds the addresses in a ban list, whatever shape its holder answers in. +var ipLike = regexp.MustCompile(`\b(?:\d{1,3}\.){3}\d{1,3}\b`) + +// lookupHost resolves an endpoint's name; replaced in tests. +var lookupHost = func(ctx context.Context, host string) ([]string, error) { + return net.DefaultResolver.LookupHost(ctx, host) +} + +// probeBans is D8: no address the mesh owns is in any machine's ban list. The mesh's own addresses are every machine's +// private address and the address its endpoint names, from placement; the ban lists are asked of each machine's holder +// of node-intrusion-prevention that is heard from. +func probeBans(ctx context.Context, c *Checker) ([]Observation, error) { + places, err := c.placements() + if err != nil { + return nil, err + } + // address → whose, in machine names: what the summary says. The address, and the endpoint's name it was found + // by, are the evidence (ADR 0234 §6). + owned, via := map[string]string{}, map[string]string{} + nodes := make([]string, 0, len(places)) + for n := range places { + nodes = append(nodes, n) + } + sort.Strings(nodes) + for _, n := range nodes { + d := places[n].Decided + if a, _, _ := strings.Cut(d.Address, "/"); a != "" { + owned[a] = n + "'s private address" + } + if host, _, err := net.SplitHostPort(d.Endpoint); err == nil && host != "" { + if ip := net.ParseIP(host); ip != nil { + owned[ip.String()] = n + "'s endpoint" + } else if addrs, err := lookupHost(ctx, host); err == nil { + for _, a := range addrs { + owned[a], via[a] = n+"'s endpoint", host + } + } + } + } + heard, err := c.heard() + if err != nil { + return nil, err + } + all, err := c.holdersOf(BannedSeat) + if err != nil { + return nil, err + } + var holders []string + for _, n := range all { + if heard[n] { + holders = append(holders, n) + } + } + // Every machine asked at once: one after another, four holders that each wait their bound outlast the probe's + // thirty seconds, and the probe says nothing about any of them. + answers := make([]json.RawMessage, len(holders)) + errs := make([]error, len(holders)) + var wg sync.WaitGroup + for i, node := range holders { + wg.Add(1) + go func(i int, node string) { + defer wg.Done() + answers[i], errs[i] = c.Ask(BannedVerb+"@"+node, map[string]any{}) + }(i, node) + } + wg.Wait() + var out []Observation + var unasked []string + for i, node := range holders { + if errs[i] != nil { + unasked = append(unasked, errs[i].Error()) + continue + } + var banned, whose []string + for _, ip := range ipLike.FindAllString(string(answers[i]), -1) { + who, ours := owned[ip] + said := ip + " (" + who + ")" + if h := via[ip]; h != "" { + said = ip + " (" + who + ", " + h + ")" + } + if !ours || slices.Contains(banned, said) { + continue + } + banned = append(banned, said) + if !slices.Contains(whose, who) { + whose = append(whose, who) + } + } + if len(banned) > 0 { + out = append(out, Observation{Kind: "own-address-banned", Token: "bans-the-mesh", Severity: Urgent, + Subject: Subject{Scope: ScopeMachine, ID: node, Machine: node}, + Summary: fmt.Sprintf("%s's ban list holds %d of the mesh's own address(es) — %s: the mesh is locked out "+ + "of itself there (ADR 0186)", node, len(banned), strings.Join(whose, ", ")), + Said: strings.Join(banned, ", "), + Headline: node + " has banned the mesh", + Needs: fmt.Sprintf("lift the ban on %s, or wait for it to expire.", node), + Explanation: fmt.Sprintf("The intrusion protection on %s banned one of the mesh's own addresses, so the "+ + "mesh is locked out of it.", node), + Resolved: node + " no longer bans the mesh"}) + } + } + if len(unasked) > 0 { + return nil, fmt.Errorf("a ban list could not be read: %s", strings.Join(unasked, "; ")) + } + return sorted(out), nil +} diff --git a/modules/mesh-self-check/cmd/mesh-self-check/probes_root.go b/modules/mesh-self-check/cmd/mesh-self-check/probes_root.go new file mode 100644 index 00000000..2d46fba4 --- /dev/null +++ b/modules/mesh-self-check/cmd/mesh-self-check/probes_root.go @@ -0,0 +1,254 @@ +package main + +// The account agents run as (novox/hq ADR 0266) and root where the trusted parties run (ADR 0259 §8), judged on the +// machine's own word: each node-engine says, in its health report (mesh-host.health, key ), its verdict on every +// account a module declares never to become root without a person. The judgement, its kind, key and words are the +// controller's (mesh-controller agent_account.go). Whether the verdict is current is the watchman's presence: the +// report is written on a change only, so a machine not heard from has no verdict on now. + +import ( + "context" + "fmt" + "sort" + "strings" + "sync" + "time" + + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +// The engine's words and bounds the judgement reads (mesh-host rootsearch). +const ( + // HealthRead is the node-engines' health reports. + HealthRead = delivery.NodeHealthRead + // rootPending starts a verdict's reason while the engine's setuid search runs and no complete one judges. + rootPending = "not judged yet (search running)" + // rootQuiet is how long nothing is raised while an agent account waits for that search, counted from when this + // self-check first saw it waiting: past it, waiting is itself the urgent condition. It never makes it confined. + rootQuiet = 12 * time.Hour + stateHealthy = "healthy" + stateUnhealthy = "unhealthy" + stateUnknown = "unknown" +) + +// report is a node's health report, and whether it has one. +func (c *Checker) report(node string) (delivery.NodeHealth, bool, error) { + raw, ok, err := c.Store.Get(HealthRead, node) + if err != nil { + return delivery.NodeHealth{}, false, fmt.Errorf("the health report of %s cannot be read: %w", node, err) + } + if !ok { + return delivery.NodeHealth{}, false, nil + } + h, err := delivery.DecodeNodeHealth(raw) + if err != nil { + return delivery.NodeHealth{}, false, fmt.Errorf("the health report of %s: %w", node, err) + } + return h, true, nil +} + +// pendingSince is, by node, when this self-check first saw its agent account waiting for the setuid search: kept while +// it waits, forgotten once a verdict is complete. In this process alone: a restart of the node-engine does not restart +// it, and a restart of the self-check gives a waiting account at most one more quiet. +type pendingSince struct { + mu sync.Mutex + seen map[string]time.Time +} + +func (p *pendingSince) since(node string, now time.Time) time.Time { + p.mu.Lock() + defer p.mu.Unlock() + if p.seen == nil { + p.seen = map[string]time.Time{} + } + if at, ok := p.seen[node]; ok { + return at + } + p.seen[node] = now + return now +} + +func (p *pendingSince) judged(node string) { + p.mu.Lock() + defer p.mu.Unlock() + delete(p.seen, node) +} + +// The kinds of an agent account's verdict, for the quiet a search earns. +const ( + verdictOther = iota // anything else: not heard, an older engine, no verdict, another unknown + verdictPending // waiting for the search, and otherwise healthy + verdictComplete // judged: healthy, or a way to root found +) + +// judgedConfined is the controller's judgement over one machine's report: confined only when the machine is heard +// from, its engine judges root, and every verdict on the agent account is healthy. why is said either way. +func judgedConfined(agent string, h delivery.NodeHealth, had, heard bool) (bool, string) { + switch { + case !had: + return false, fmt.Sprintf("the agent account %s is not judged: the machine's node-engine has stated "+ + "nothing of what it runs", agent) + case !heard: + return false, fmt.Sprintf("the agent account %s is not judged: the machine is not heard from, and a verdict "+ + "from before is not a verdict on now", agent) + case h.Contract < delivery.RootContract: + return false, fmt.Sprintf("the agent account %s is not judged: the machine's node-engine is older than "+ + "the judging of an account's root (its statement's contract is %d, the judging is %d)", + agent, h.Contract, delivery.RootContract) + } + verdicts := verdictsOn(agent, h) + if len(verdicts) == 0 { + return false, fmt.Sprintf("the agent account %s is not judged: the machine's newest statement holds no "+ + "verdict on it — no module there declares it never to become root, or the declaration naming it "+ + "has not been applied", agent) + } + for _, v := range verdicts { + switch v.State { + case stateHealthy: + case stateUnhealthy: + return false, fmt.Sprintf("the agent account %s %s (said by %s's %s)", agent, orNoneKnown(v.Reason), + v.Module, v.Resource) + default: + return false, fmt.Sprintf("the agent account %s is not judged: %s (%s's %s, %s)", agent, + orNoneKnown(v.Reason), v.Module, v.Resource, v.State) + } + } + return true, fmt.Sprintf("the agent account %s cannot become root without a person (judged %s)", agent, + h.At.Local().Format("2006-01-02 15:04")) +} + +// verdictsOn is every verdict on the account, sorted by module and resource. +func verdictsOn(agent string, h delivery.NodeHealth) []delivery.RootVerdict { + var out []delivery.RootVerdict + for _, v := range h.Root { + if v.Account == agent { + out = append(out, v) + } + } + sort.Slice(out, func(i, j int) bool { return out[i].Module+out[i].Resource < out[j].Module+out[j].Resource }) + return out +} + +// rootVerdictKind reads a report for the agent account: pending when every verdict on it is healthy or waits for the +// setuid search, at least one of them that; complete when every verdict is healthy or one found a way to root. +func rootVerdictKind(agent string, h delivery.NodeHealth, had, heard bool) int { + if !had || !heard || h.Contract < delivery.RootContract { + return verdictOther + } + pending, any := false, false + for _, v := range verdictsOn(agent, h) { + any = true + switch { + case v.State == stateHealthy: + case v.State == stateUnhealthy: + return verdictComplete + case v.State == stateUnknown && strings.HasPrefix(v.Reason, rootPending): + pending = true + default: + return verdictOther + } + } + switch { + case !any: + return verdictOther + case pending: + return verdictPending + } + return verdictComplete +} + +// agentQuiet says the one thing keeping a machine's agent account unjudged is the engine's setuid search, first seen +// waiting less than rootQuiet ago. +func (c *Checker) agentQuiet(node, agent string, h delivery.NodeHealth, had, heard bool) bool { + switch rootVerdictKind(agent, h, had, heard) { + case verdictComplete: + c.pending.judged(node) + case verdictPending: + now := c.clock() + return now.Sub(c.pending.since(node, now)) <= rootQuiet + } + return false +} + +// clock is the checker's clock, the wall clock where it has none. +func (c *Checker) clock() time.Time { + if c.Now != nil { + return c.Now() + } + return time.Now() +} + +// confinement is one machine's agent account judged: whether it names one, whether it is confined, why, and whether +// it is unjudged only because its search runs within the quiet. +type confinement struct { + Named, Confined, Quiet bool + Why string +} + +// confined judges node's agent account from its placement, its report and its presence. +func (c *Checker) confined(node string, place delivery.Placement, heard bool) (confinement, error) { + agent := place.Decided.AgentAccount + if agent == "" { + return confinement{Why: fmt.Sprintf("%s names no agent account: agents run as the operator account (%s)", + node, orNoneKnown(place.Decided.Account))}, nil + } + h, had, err := c.report(node) + if err != nil { + return confinement{Named: true}, err + } + ok, why := judgedConfined(agent, h, had, heard) + return confinement{Named: true, Confined: ok, Why: why, Quiet: !ok && c.agentQuiet(node, agent, h, had, heard)}, nil +} + +// probeAgentAccounts is DA: every machine that names an agent account has it judged, on its node-engine's newest +// statement, unable to become root without a person (ADR 0266). +func probeAgentAccounts(_ context.Context, c *Checker) ([]Observation, error) { + places, err := c.placements() + if err != nil { + return nil, err + } + heard, err := c.heard() + if err != nil { + return nil, err + } + nodes := make([]string, 0, len(places)) + for n := range places { + nodes = append(nodes, n) + } + sort.Strings(nodes) + var out []Observation + for _, n := range nodes { + if places[n].Decided.AgentAccount == "" { + continue + } + v, err := c.confined(n, places[n], heard[n]) + if err != nil { + return nil, err + } + // Not judged yet only because the engine's search runs: not the urgent condition after every restart. Loud + // again once the search fails, waits past the quiet, or the machine is not heard from. + if v.Confined || v.Quiet { + continue + } + out = append(out, Observation{Kind: "agent-can-become-root", Token: "agent-root", Severity: Urgent, + Subject: Subject{Scope: ScopeMachine, ID: n, Machine: n}, + Summary: fmt.Sprintf("on %s, %s (ADR 0266): an agent there may become root without a person, and no "+ + "answer from a channel authorises an act there (ADR 0259 §8)", n, v.Why), + Said: v.Why, + Headline: "Sessions on " + n + " could become root", + Needs: "take the sessions' own account out of every group and rule that grants root; the details say which.", + Explanation: fmt.Sprintf("Assistant sessions on %s run under an account of their own, so that none "+ + "can take over the machine without you. The machine cannot show that this holds now, so an answer "+ + "from your phone authorises nothing there until it does.", n), + Resolved: "Sessions on " + n + " cannot become root again"}) + } + return sorted(out), nil +} + +// orNoneKnown is a value, or that none is known. +func orNoneKnown(s string) string { + if strings.TrimSpace(s) == "" { + return "none known" + } + return s +} diff --git a/modules/mesh-self-check/cmd/mesh-self-check/probes_test.go b/modules/mesh-self-check/cmd/mesh-self-check/probes_test.go new file mode 100644 index 00000000..03eb637a --- /dev/null +++ b/modules/mesh-self-check/cmd/mesh-self-check/probes_test.go @@ -0,0 +1,455 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "slices" + "strings" + "testing" + "time" + + "git.novox.be/novox/mesh-sdk/go/buscompose" + "git.novox.be/novox/mesh-sdk/go/catalogue" + "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/health" +) + +// bus is the bus module's answers as a probe asks them: every mesh stream and consumer as defined, unless changed. +type bus struct { + drop map[string]bool // stream or stream.consumer missing + change func(rows map[string]*streamRow) + closed string + signals error + asksFail error + // banned is each node's answer to node-intrusion-prevention.banned; a node absent does not answer. + banned map[string]string + // fills are rows nats_streams answers beside the mesh's streams: a module's bucket or log with its size and cap. + fills []fillRow +} + +func (b bus) rows() map[string]*streamRow { + rows := map[string]*streamRow{} + for _, s := range buscompose.MeshStreams() { + if b.drop[s.Name] { + continue + } + retention := "limits" + per := int64(s.MaxMsgsPerSubject) + switch s.Retention { + case buscompose.RetentionWorkQueue: + retention = "workqueue" + case buscompose.RetentionLastPerSubject: + per = 1 + } + rows[s.Name] = &streamRow{Name: s.Name, Subjects: s.Subjects, Retention: retention, MaxPerSubject: per} + } + for _, c := range buscompose.MeshConsumers() { + r := rows[c.Stream] + if r == nil || b.drop[c.Stream+"."+c.Name] { + continue + } + kind := "pull" + if c.Push || c.Queue != "" { + kind = "push" + } + r.Consumers = append(r.Consumers, consumerRow{Name: c.Name, Kind: kind, Filters: c.Filters, MaxDeliver: int64(c.MaxDeliver)}) + } + if b.change != nil { + b.change(rows) + } + return rows +} + +func (b bus) ask(key string, body any) (json.RawMessage, error) { + if b.asksFail != nil { + return nil, b.asksFail + } + switch key { + case StreamsTool: + stream, _ := body.(map[string]any)["stream"].(string) + var detail []streamRow + for name, r := range b.rows() { + if stream == "" { + row := *r + row.Consumers = nil + detail = append(detail, row) + } else if name == stream { + detail = append(detail, *r) + } + } + if stream == "" && len(b.fills) > 0 { + var rows []any + for _, r := range detail { + rows = append(rows, r) + } + for _, f := range b.fills { + rows = append(rows, f) + } + return json.Marshal(map[string]any{"detail": rows}) + } + if stream != "" && len(detail) == 0 { + return nil, errors.New("no stream called " + stream) + } + return json.Marshal(map[string]any{"detail": detail}) + case ClosedTool: + if b.closed == "" { + return json.RawMessage(`{"hours":1,"reaches":true,"users":[]}`), nil + } + return json.RawMessage(b.closed), nil + case SignalsVerb: + if b.signals != nil { + return nil, b.signals + } + return json.RawMessage(`{"signals":[]}`), nil + } + if node, ok := strings.CutPrefix(key, BannedVerb+"@"); ok { + if a, ok := b.banned[node]; ok { + return json.RawMessage(a), nil + } + } + return nil, errors.New("nats: no responders available for request") +} + +func keysOf(found []Observation) []string { + var out []string + for _, f := range found { + out = append(out, f.Key()) + } + return out +} + +// Each probe that moved judges the bus module's answers with its assertion unchanged (ADR 0564 §6; issue 1952 SC2). +func TestTheBusProbes(t *testing.T) { + cases := []struct { + name string + probe func(context.Context, *Checker) ([]Observation, error) + bus bus + want []string + err string + mesh *memStore // the bus states the machine probes read; nil reads none + }{ + {"D6 all as defined", probeConsumers, bus{}, nil, "", nil}, + {"D6 a consumer gone", probeConsumers, bus{drop: map[string]bool{"DEAD_LETTER_NOTICES.controller": true}}, + []string{"bus.DEAD_LETTER_NOTICES.controller.missing"}, "", nil}, + {"D6 a stream gone takes its consumers", probeConsumers, bus{drop: map[string]bool{"CONTROL": true}}, + []string{"bus.CONTROL.controller.missing"}, "", nil}, + {"D6 redefined and far behind", probeConsumers, bus{change: func(rows map[string]*streamRow) { + for i, c := range rows["EVENTS"].Consumers { + if c.Name == "controller" { + rows["EVENTS"].Consumers[i].MaxDeliver, rows["EVENTS"].Consumers[i].Pending = 9, 5000 + } + } + }}, []string{"bus.EVENTS.controller.behind", "bus.EVENTS.controller.redefined"}, "", nil}, + {"D6 the bus module silent", probeConsumers, bus{asksFail: errors.New("no responders")}, nil, "did not answer", nil}, + {"D7 all as defined", probeStreams, bus{}, nil, "", nil}, + {"D7 missing and redefined", probeStreams, bus{drop: map[string]bool{"NODES": true}, change: func(rows map[string]*streamRow) { + rows["EVENTS"].Retention = "workqueue" + }}, []string{"bus.EVENTS.stream-redefined", "bus.NODES.stream-missing"}, "", nil}, + {"D15 quiet", probeReconnects, bus{}, nil, "", nil}, + {"D15 a loop", probeReconnects, bus{closed: `{"hours":1,"reaches":true,"users":[ + {"user":"g14.node-tools","closed":30,"dropped":20,"names":[{"name":"node-tools","dropped":20,"reasons":{"Stale Connection":20}}]}, + {"user":"ace.records","closed":5,"dropped":3},{"user":"(unauthenticated)","closed":90,"dropped":90}]}`}, + []string{"bus.g14.node-tools.bus-reconnects"}, "", nil}, + {"DW answers", probeWatchdogs, bus{}, nil, "", nil}, + {"DW silent", probeWatchdogs, bus{signals: errors.New("no responders")}, []string{"core.watchdogs.silent"}, "", nil}, + {"H-bus healthy", probeBusHealth, bus{}, nil, "", nil}, + {"H-bus a stream and a consumer missing", probeBusHealth, + bus{drop: map[string]bool{"NODES": true, "EVENTS.controller-merges": true}}, []string{"bus.mesh.unhealthy"}, "", nil}, + {"H-bus no round trip", probeBusHealth, bus{asksFail: errors.New("timeout")}, nil, "no request crossed", nil}, + // D8 (issue 238): placement's addresses and endpoints against each heard holder's ban list. + {"D8 nothing of the mesh banned", probeBans, bus{banned: map[string]string{"ace": `{"banned":["198.51.100.7"]}`, + "novox": `{"banned":[]}`}}, nil, "", machines()}, + {"D8 a private address and an endpoint banned", probeBans, bus{banned: map[string]string{ + "ace": `{"jails":{"sshd":["10.10.0.1","198.51.100.7"]}}`, "novox": `{"banned":["203.0.113.5"]}`}}, + []string{"machine.ace.bans-the-mesh", "machine.novox.bans-the-mesh"}, "", machines()}, + {"D8 a machine not heard from is not asked", probeBans, bus{banned: map[string]string{"novox": `{"banned":[]}`, + "ace": `{"banned":[]}`}}, nil, "", machines()}, + {"D8 a heard holder that does not answer is no pass", probeBans, bus{banned: map[string]string{}}, nil, + "a ban list could not be read", machines()}, + // D13 (issue 273, ADR 0233): the readings disk-load keeps and the last good nights restic keeps, against the bounds + // the manifests declare; the kinds, keys and words are the controller's. + {"D13 all held", probeData, bus{}, nil, "", dataMesh(func(*dataRig) {})}, + {"D13 shrank, gone, quiet, unprotected and stale", probeData, bus{}, []string{ + "machine.ace.pg.data.data-quiet", "machine.ace.pg.data.backup-stale", "machine.ace.pg.dumps.data-missing", + "machine.ace.plex.movies.protection-missing", "machine.ace.plex.thumbs.data-shrank", + "machine.ace.array.md-md127.array-degraded"}, + "", dataMesh(func(r *dataRig) { + r.size("ace", "pg", "data", 100<<20, 100<<20).write("ace", "pg", "data", -80*time.Hour) + r.gone("ace", "pg", "dumps") + r.size("ace", "plex", "thumbs", 10<<20, 600<<20) + r.size("ace", "plex", "movies", 1<<30, 1<<30).noArray("ace", "plex", "movies") + r.size("ace", "plex", "library", 1<<30, 1<<30).array("ace", "plex", "library", "md", "md127", false) + r.lastGood("ace", "pg", "data", -100*time.Hour) + })}, + {"D13 a dataset shrank once for its items", probeData, bus{}, []string{"machine.ace.dataset.storage-media.data-shrank"}, + "", dataMesh(func(r *dataRig) { + r.size("ace", "plex", "thumbs", 10<<20, 600<<20).dataset("ace", "plex", "thumbs", "storage/media") + r.size("ace", "plex", "meta", 10<<20, 600<<20).dataset("ace", "plex", "meta", "storage/media") + })}, + {"D13 a retired item is kept, not judged, and cleanup waits after thirty days", probeData, bus{}, []string{ + "machine.ace.pg.dumps.cleanup"}, "", dataMesh(func(r *dataRig) { + r.size("ace", "pg", "dumps", 1<<20, 1<<30).retired("ace", "pg", "dumps", -31*24*time.Hour) + r.size("ace", "pg", "old", 1<<20, 1<<30).retired("ace", "pg", "old", -2*24*time.Hour) + })}, + {"D13 a copy far smaller than a retired one elsewhere is an empty replacement", probeData, bus{}, + []string{"machine.ace.plex.thumbs.empty-replacement"}, "", dataMesh(func(r *dataRig) { + r.size("ace", "plex", "thumbs", 10<<20, 10<<20) + r.size("g14", "plex", "thumbs", 600<<20, 600<<20).retired("g14", "plex", "thumbs", -24*time.Hour) + })}, + {"D13 no good backup is counted from when the item was first measured", probeData, bus{}, + []string{"machine.ace.pg.data.backup-stale"}, "", dataMesh(func(r *dataRig) { + delete(r.good, health.ReadingKey("ace", "pg", "data")) + delete(r.good, health.ReadingKey("ace", "plex", "meta")) + r.firstSeen("ace", "pg", "data", -100*time.Hour).firstSeen("ace", "plex", "meta", -time.Hour) + })}, + {"D13 a machine with a backup holder and no reading", probeData, bus{}, []string{"machine.g14.data-unmeasured"}, "", + dataMesh(func(r *dataRig) { r.holder("g14") })}, + // D-fill (ADR 0297 §6): a module's bucket or log at three quarters of its cap, kept until under seven tenths. + {"D-fill all with room", probeFill, bus{fills: []fillRow{{Name: "KV_mesh-issues_issues", Bytes: 74, MaxBytes: 100}, + {Name: "LOG_mesh-delivery-builder_build-log", Bytes: 10, MaxBytes: 100}, {Name: "KV_nocap_x", Bytes: 999}}}, nil, "", nil}, + {"D-fill a bucket and a log filling", probeFill, bus{fills: []fillRow{{Name: "KV_mesh-issues_issues", Bytes: 75, MaxBytes: 100}, + {Name: "LOG_mesh-delivery-builder_build-log", Bytes: 99, MaxBytes: 100}}}, + []string{"bus.KV_mesh-issues_issues.filling", "bus.LOG_mesh-delivery-builder_build-log.filling"}, "", nil}, + {"D-fill between the thresholds raises nothing new", probeFill, + bus{fills: []fillRow{{Name: "KV_mesh-issues_issues", Bytes: 72, MaxBytes: 100}}}, nil, "", nil}, + {"D-fill between the thresholds keeps what is said", probeFill, + bus{fills: []fillRow{{Name: "KV_mesh-issues_issues", Bytes: 72, MaxBytes: 100}}}, + []string{"bus.KV_mesh-issues_issues.filling"}, "", filling("KV_mesh-issues_issues")}, + {"D-fill the bus module silent", probeFill, bus{asksFail: errors.New("no responders")}, nil, "did not answer", nil}, + // DA (ADR 0266): the agent account each machine names, judged on its engine's report while it is heard from. + {"DA confined", probeAgentAccounts, bus{}, nil, "", agents(map[string]string{"novox": stateHealthy})}, + {"DA a way to root found", probeAgentAccounts, bus{}, []string{"machine.novox.agent-root"}, "", + agents(map[string]string{"novox": stateUnhealthy})}, + {"DA no verdict on it", probeAgentAccounts, bus{}, []string{"machine.novox.agent-root"}, "", + agents(map[string]string{"novox": ""})}, + {"DA the search runs, within its quiet", probeAgentAccounts, bus{}, nil, "", + agents(map[string]string{"novox": "pending"})}, + {"DA a machine not heard from has no verdict on now", probeAgentAccounts, bus{}, []string{"machine.g14.agent-root"}, + "", agents(map[string]string{"novox": stateHealthy, "g14": stateHealthy})}, + {"DA an engine older than the judging", probeAgentAccounts, bus{}, []string{"machine.novox.agent-root"}, "", + agents(map[string]string{"novox": "old"})}, + } + for _, tc := range cases { + c := &Checker{Ask: tc.bus.ask, Store: tc.mesh, Now: func() time.Time { return dataNow }} + if tc.mesh == nil { + c.Store = newStore() + } + found, err := tc.probe(context.Background(), c) + if tc.err != "" { + if err == nil || !strings.Contains(err.Error(), tc.err) { + t.Errorf("%s: error %v, want %q", tc.name, err, tc.err) + } + continue + } + if err != nil { + t.Errorf("%s: %v", tc.name, err) + continue + } + if got := keysOf(found); !slices.Equal(got, tc.want) { + t.Errorf("%s: found %v, want %v", tc.name, got, tc.want) + } + } +} + +// machines is a mesh of three: novox the hub with its endpoint, ace and g14 on the private network; ace and novox +// hold node-intrusion-prevention and are heard from, g14 holds it and is not. +func machines() *memStore { + s := newStore() + for _, p := range []struct { + node, address, endpoint string + online bool + }{{"novox", "10.10.0.1/16", "203.0.113.5:51820", true}, {"ace", "10.10.0.2/16", "", true}, {"g14", "10.10.0.3/16", "", false}} { + _ = s.Put(PlacementRead, p.node, delivery.Placement{V: 1, Node: p.node, + Decided: delivery.NodeDecided{Address: p.address, Endpoint: p.endpoint, Hub: p.endpoint != ""}}) + _ = s.Put(PresenceRead, p.node, delivery.Presence{V: 1, Online: p.online}) + key, _ := delivery.SeatHoldersKey(BannedSeat, p.node) + _ = s.Put(SeatHoldersRead, key, delivery.SeatHolders{V: 1, Seat: BannedSeat, Scope: "node", Node: p.node, + Holders: []delivery.SeatHolder{{Module: "fail2ban", Node: p.node}}}) + } + return s +} + +// filling is a store holding the self-check's written fill finding of a stream: the condition open. +func filling(stream string) *memStore { + s := newStore() + o := Observation{Kind: "bucket-or-log-filling", Token: "filling", Severity: Warning, Subject: Subject{Scope: ScopeBus, ID: stream}, + Summary: "filling", Source: "D-fill"} + raw, _ := health.EncodeFinding(o.Finding("run", time.Now())) + _ = s.Put(FindingsState, o.Key(), json.RawMessage(raw)) + return s +} + +// agents is machines() where each node named names the agent account "agent" and its report holds the verdict on it: +// a state, "" for none, "pending" for one waiting for its search, "old" for an engine older than the judging. +func agents(verdicts map[string]string) *memStore { + s := machines() + for node, v := range verdicts { + raw, _, _ := s.Get(PlacementRead, node) + var p delivery.Placement + _ = json.Unmarshal(raw, &p) + p.Decided.AgentAccount = "agent" + _ = s.Put(PlacementRead, node, p) + h := delivery.NodeHealth{V: 1, Node: node, Contract: 13} + switch v { + case "": + case "old": + h.Contract = 2 + h.Root = []delivery.RootVerdict{{Account: "agent", Module: "claude-code", Resource: "claude-code.agent", State: stateHealthy}} + case "pending": + h.Root = []delivery.RootVerdict{{Account: "agent", Module: "claude-code", Resource: "claude-code.agent", + State: stateUnknown, Reason: rootPending + ": 3 of 9 trees"}} + default: + h.Root = []delivery.RootVerdict{{Account: "agent", Module: "claude-code", Resource: "claude-code.agent", State: v, + Reason: "can become root without a person: a sudo rule"}} + } + raw, _ = delivery.EncodeNodeHealth(h) + _ = s.Put(HealthRead, node, json.RawMessage(raw)) + } + return s +} + +// The registry holds exactly the probes ADR 0564's table moves here, each with a kind, and a deferred one names what +// it lacks; the probes that end with the controller or moved to the watchman are not in it (issue 1952 SC4). +func TestTheRegistryIsADR0564sTable(t *testing.T) { + moved := []string{"D2", "D3", "D4", "D6", "D7", "D8", "D11", "D12", "D13", "D15", "DA", "D-root", "D-fill", "D-heard", + "DW", "H-bus"} + var ids []string + for _, p := range Registry() { + ids = append(ids, p.ID) + if p.Kind == "" || p.Asserts == "" { + t.Errorf("%s has no kind or assertion", p.ID) + } + if (p.Run == nil) == (p.Deferred == "") { + t.Errorf("%s is neither run nor deferred naming what it lacks", p.ID) + } + } + slices.Sort(ids) + slices.Sort(moved) + if !slices.Equal(ids, moved) { + t.Errorf("the registry holds %v, ADR 0564 moves %v", ids, moved) + } +} + +// ---- D13's rig ------------------------------------------------------------------------------------------------ + +var dataNow = time.Date(2026, 10, 29, 12, 0, 0, 0, time.UTC) + +// dataRig is the mesh D13 reads: a manifest per module with the items it declares, a reading per item, a last good night. +type dataRig struct { + s *memStore + readings map[string]*health.Reading + good map[string]time.Time +} + +func (r *dataRig) key(node, module, item string) string { return health.ReadingKey(node, module, item) } + +func (r *dataRig) reading(node, module, item string) *health.Reading { + k := r.key(node, module, item) + if r.readings[k] == nil { + at := dataNow + r.readings[k] = &health.Reading{V: health.ReadingVersion, Node: node, PlacedDataItem: catalogue.PlacedDataItem{Module: module, + ID: item, Class: catalogue.ClassValuable, Path: "/srv/" + module + "/" + item}, Measured: health.Measured{MeasuredAt: &at, Precision: "exact"}} + } + return r.readings[k] +} + +func (r *dataRig) size(node, module, item string, size, peak int64) *dataRig { + x := r.reading(node, module, item) + x.SizeBytes, x.Peak = &size, &peak + return r +} + +func (r *dataRig) write(node, module, item string, ago time.Duration) *dataRig { + t := dataNow.Add(ago) + r.reading(node, module, item).LastWrite = &t + return r +} + +func (r *dataRig) gone(node, module, item string) *dataRig { + r.reading(node, module, item).Error = "/srv/" + module + "/" + item + " does not exist" + return r +} + +func (r *dataRig) dataset(node, module, item, name string) *dataRig { + r.reading(node, module, item).Precision = "dataset " + name + ": its whole size" + return r +} + +func (r *dataRig) array(node, module, item, kind, where string, healthy bool) *dataRig { + r.reading(node, module, item).Redundancy = &health.Redundancy{Kind: kind, Where: where, Healthy: &healthy, Said: "a member is missing"} + return r +} + +func (r *dataRig) noArray(node, module, item string) *dataRig { + r.reading(node, module, item).Redundancy = nil + return r +} + +func (r *dataRig) lastGood(node, module, item string, ago time.Duration) *dataRig { + r.good[r.key(node, module, item)] = dataNow.Add(ago) + return r +} + +func (r *dataRig) retired(node, module, item string, ago time.Duration) *dataRig { + x := r.reading(node, module, item) + x.RetiredAt, x.RetiredWhy = dataNow.Add(ago), "no longer declared on "+node + return r +} + +func (r *dataRig) firstSeen(node, module, item string, ago time.Duration) *dataRig { + r.reading(node, module, item).FirstSeen = dataNow.Add(ago) + return r +} + +func (r *dataRig) holder(node string) { + key, _ := delivery.SeatHoldersKey("node-backup", node) + _ = r.s.Put(SeatHoldersRead, key, delivery.SeatHolders{V: 1, Seat: "node-backup", Scope: "node", Node: node, + Holders: []delivery.SeatHolder{{Module: "restic", Node: node}}}) + _ = r.s.Put(PresenceRead, node, delivery.Presence{V: 1, Online: true}) +} + +// dataMesh builds the mesh: pg declares data (backed up within 48 h, written all the time within 24 h) and dumps; plex +// declares movies (protected by redundancy), thumbs, meta and library (all valuable); every item is read as held, and what +// is given changes it. +func dataMesh(change func(*dataRig)) *memStore { + r := &dataRig{s: newStore(), readings: map[string]*health.Reading{}, good: map[string]time.Time{}} + r.holder("ace") + declare := func(module string, items map[string]catalogue.DataItem) { + var man catalogue.Manifest + man.Module = module + man.Data = &catalogue.Data{} + for id, it := range items { + it.ID = id + man.Data.Own = append(man.Data.Own, it) + } + raw, _ := json.Marshal(man) + digest := delivery.ManifestDigest(raw) + _ = r.s.Put(ManifestsRead, digest, delivery.Manifest{V: 1, Module: module, JSON: string(raw)}) + _ = r.s.Put(ModulesRead, module, delivery.Module{V: 1, Manifest: digest}) + } + declare("pg", map[string]catalogue.DataItem{ + "data": {Class: catalogue.ClassValuable, Within: "48h", Active: "24h"}, + "dumps": {Class: catalogue.ClassValuable}}) + declare("plex", map[string]catalogue.DataItem{ + "movies": {Class: catalogue.ClassValuable, Redundancy: "the array"}, + "thumbs": {Class: catalogue.ClassValuable}, + "meta": {Class: catalogue.ClassValuable}, + "library": {Class: catalogue.ClassValuable, Redundancy: "the array"}}) + for _, i := range [][2]string{{"pg", "data"}, {"pg", "dumps"}, {"plex", "movies"}, {"plex", "thumbs"}, {"plex", "meta"}, {"plex", "library"}} { + r.size("ace", i[0], i[1], 1<<30, 1<<30).lastGood("ace", i[0], i[1], -time.Hour).write("ace", i[0], i[1], -time.Minute) + if i[0] == "plex" && (i[1] == "movies" || i[1] == "library") { + r.array("ace", i[0], i[1], "md", "md127", true) + } + } + change(r) + for k, x := range r.readings { + _ = r.s.Put(ReadingsRead, k, x) + } + for k, t := range r.good { + node, rest, _ := strings.Cut(k, ".") + module, item, _ := strings.Cut(rest, ".") + _ = r.s.Put(GoodRead, k, health.GoodRow{V: health.GoodVersion, Node: node, Module: module, Item: item, CoveredBy: "/x", LastGood: t}) + } + return r.s +} diff --git a/modules/mesh-self-check/cmd/mesh-self-check/reads-outside b/modules/mesh-self-check/cmd/mesh-self-check/reads-outside new file mode 100644 index 00000000..cb8f579b --- /dev/null +++ b/modules/mesh-self-check/cmd/mesh-self-check/reads-outside @@ -0,0 +1,2 @@ +# What these tests read outside their package, as paths from the repository's root (issue 1936). +modules/mesh-self-check/module.json diff --git a/modules/mesh-self-check/cmd/mesh-self-check/selfcheck.go b/modules/mesh-self-check/cmd/mesh-self-check/selfcheck.go new file mode 100644 index 00000000..174ff884 --- /dev/null +++ b/modules/mesh-self-check/cmd/mesh-self-check/selfcheck.go @@ -0,0 +1,417 @@ +package main + +// The self-check (novox/hq ADR 0564 §6): the registry of probes, each a live invariant of a design, run every five +// minutes, each probe within thirty seconds. What a probe finds is written as a finding in this module's bucket +// `findings`, which the condition keeper raises and clears conditions from (ADR 0564 §2); every run is kept in the +// bucket `verdicts` for a week and the last as the key `last` of `verdict`, and is said as `checked`, which the +// watchman and mesh-watcher hear (S10, ADR 0227 §6). A probe that errors or does not finish is never a pass: it is +// said in the verdict at once and written as a finding of its own when the next run cannot run it either. + +import ( + "context" + "encoding/json" + "fmt" + "slices" + "sort" + "strings" + "sync" + "time" + + "git.novox.be/novox/mesh-sdk/go/health" +) + +// Module and Seat are this module and the seat it holds once for the mesh. +const ( + Module = "mesh-self-check" + Seat = "mesh-self-check" +) + +// The states it keeps, and the event that ends every run. +const ( + FindingsState = health.FindingsState + VerdictState = "verdict" + VerdictsState = "verdicts" + LastKey = "last" + CheckedEvent = health.EventChecked + // CheckedSubject is where the seat says it: a seat's events are on mesh.seat..event., which its + // consumers hear (catalogue.SeatSays). + CheckedSubject = "mesh.seat." + Seat + ".event." + CheckedEvent +) + +// The clocks: the self-check's run is the one timer among the four holders (ADR 0564 §1). +var ( + Every = 5 * time.Minute + FirstAfter = time.Minute + Within = 30 * time.Second +) + +// The verdicts a probe can have, as mesh-sdk's health names them. +const ( + VerdictPass = health.VerdictPassed + VerdictFail = health.VerdictFailed + VerdictFailedToRun = health.VerdictFailedToRun + VerdictDeferred = health.VerdictDeferred +) + +// sourceSelf is what writes a probe's failure to run: the self-check itself. +const sourceSelf = "doctor" + +// Store is the bus's buckets as this module reaches them, through its runtime. +type Store interface { + Get(state, key string) ([]byte, bool, error) + Put(state, key string, value any) error + Delete(state, key string) error + Keys(state string) ([]string, error) +} + +// On says whether the setting lists the probe. +func (c *Checker) On(id string) bool { + if c.Enabled == nil { + return true + } + return slices.Contains(c.Enabled(), id) +} + +// DefaultProbes are the probes that run unless the setting says otherwise. +var DefaultProbes = []string{"D13", "D-fill"} + +// Probe is one live invariant: its id, what it asserts, the kind it raises, and the run; or why it waits. +type Probe struct { + ID string + Asserts string + From string + Kind string + Raises []string + // Deferred names the facts it lacks on the bus, for a probe whose facts have no home there yet (ADR 0564 §6): + // answered deferred, never a pass. + Deferred string + Run func(ctx context.Context, c *Checker) ([]Observation, error) +} + +// ProbeVerdict, Counts and Run are a run as mesh-sdk's health holds it: the body of `checked`, the controller's +// doctor-heartbeat body unchanged (ADR 0564 §6), Controller naming the machine that ran it. +type ( + ProbeVerdict = health.ProbeVerdict + Counts = health.CheckedCounts + Run = health.Checked +) + +// Checker is the self-check: the registry and what its probes reach. +type Checker struct { + Store Store + // Enabled is the setting `probes`: the ids of the probes that run. Nil runs every probe in the registry; one not + // listed runs nothing and raises nothing (the operator, 2026-10-30). + Enabled func() []string + Ask func(key string, body any) (json.RawMessage, error) + // Publish says a seat event on its subject, de-duplicated by id (stdio.SeatPublish). + Publish func(subject string, body any, id string) error + Now func() time.Time + Logf func(format string, args ...any) + Host string + Registry []Probe + + running sync.Mutex + mu sync.Mutex + n uint64 + last *Run + // seen are the keys the run before saw, confirmed or not: a finding a single look can be wrong about is + // written only when two runs in a row see it, or while it is already written (novox/hq issue 277). + seen map[string]bool + // unsaid counts the runs whose `checked` could not be said since the start. + unsaid int + // pending is when each machine's agent account was first seen waiting for its setuid search (DA, D-root). + pending pendingSince +} + +// LastRun is the last run, nil before the first. +func (c *Checker) LastRun() *Run { + c.mu.Lock() + defer c.mu.Unlock() + return c.last +} + +type result struct { + found []Observation + err error + took time.Duration +} + +// RunOnce runs every probe of the registry, writes what changed in `findings`, keeps the verdict and says `checked`. +// One run at a time: a person's run during a scheduled one waits for it. +func (c *Checker) RunOnce(ctx context.Context, why string) Run { + c.running.Lock() + defer c.running.Unlock() + c.mu.Lock() + c.n++ + n := c.n + c.mu.Unlock() + started := c.Now() + run := Run{Run: fmt.Sprintf("doctor-%d-%d", started.Unix(), n), Started: started.UTC(), + IntervalSeconds: int(Every / time.Second), Controller: c.Host, Why: why} + + results := make([]result, len(c.Registry)) + var wg sync.WaitGroup + for i, p := range c.Registry { + if p.Run == nil || !c.On(p.ID) { + continue + } + wg.Add(1) + go func(i int, p Probe) { + defer wg.Done() + results[i] = c.probe(ctx, p) + }(i, p) + } + wg.Wait() + + written, readErr := c.written() + seen := map[string]bool{} + want := map[string]Observation{} // every finding this run writes, by key + ran := map[string]bool{} // the sources whose findings this run judged whole + var blind []Observation + for i, p := range c.Registry { + if !c.On(p.ID) { + // Off by the setting: it ran nothing and sees nothing, so what it raised before is cleared, and it has no + // verdict in the run. + ran[p.ID] = true + continue + } + v := ProbeVerdict{ID: p.ID} + r := results[i] + switch { + case p.Run == nil: + v.Verdict = VerdictDeferred + v.Error = "deferred: " + p.Deferred + run.Counts.Deferred++ + case r.err != nil: + v.Verdict, v.Error = VerdictFailedToRun, r.err.Error() + run.Counts.FailedToRun++ + blind = append(blind, Observation{Kind: "probe-failed", Token: "failed", Severity: Warning, Confirm: true, + Subject: Subject{Scope: ScopeProbe, ID: p.ID}, Source: sourceSelf, + Summary: fmt.Sprintf("the probe %s (%s) could not run: what it checks is not known — never a pass", p.ID, p.Asserts), + Said: firstLine(r.err.Error())}) + default: + ran[p.ID] = true + for _, f := range r.found { + if f.Kind == "" { + f.Kind = p.Kind + } + f.Source = p.ID + f = worded(f) + key := f.Key() + seen[key] = true + if f.Confirm && !c.seenBefore(key) && !written.has(key) { + v.Unconfirmed = append(v.Unconfirmed, f.Summary) + continue + } + want[key] = f + v.Found = append(v.Found, f.Summary) + } + if len(v.Found) == 0 { + v.Verdict = VerdictPass + run.Counts.Passed++ + } else { + v.Verdict = VerdictFail + run.Counts.Failed++ + } + } + if p.Run != nil { + v.Took = r.took.Round(time.Millisecond).String() + } + run.Probes = append(run.Probes, v) + } + ran[sourceSelf] = true + for _, f := range blind { + f = worded(f) + key := f.Key() + seen[key] = true + if c.seenBefore(key) || written.has(key) { + want[key] = f + } + } + if readErr != nil { + // Blind to its own bucket: nothing is written or deleted (ADR 0227 §4), and the log says so. + c.Logf("[%s] its findings cannot be read, so none is written or cleared this run: %v", Module, readErr) + } else if err := c.write(written, want, ran, run.Run); err != nil { + c.Logf("[%s] what this run found is not all written: %v", Module, err) + } + c.mu.Lock() + c.seen = seen + c.mu.Unlock() + + run.At = c.Now().UTC() + run.Took = run.At.Sub(started).Round(time.Millisecond).String() + c.mu.Lock() + run.Unsaid = c.unsaid + c.mu.Unlock() + c.keep(run) + return run +} + +// probe runs one probe within its bound; a panic or an overrun is an error, never a pass. +func (c *Checker) probe(ctx context.Context, p Probe) result { + probing, cancel := context.WithTimeout(ctx, Within) + defer cancel() + began := c.Now() + done := make(chan result, 1) + go func() { + defer func() { + if r := recover(); r != nil { + done <- result{err: fmt.Errorf("the probe panicked: %v", r)} + } + }() + found, err := p.Run(probing, c) + done <- result{found: found, err: err} + }() + select { + case r := <-done: + r.took = c.Now().Sub(began) + return r + case <-probing.Done(): + return result{err: fmt.Errorf("it did not finish within %s", Within), took: c.Now().Sub(began)} + } +} + +func (c *Checker) seenBefore(key string) bool { + c.mu.Lock() + defer c.mu.Unlock() + return c.seen[key] +} + +// writtenFindings are the findings in the bucket, by key. +type writtenFindings map[string]health.Finding + +func (w writtenFindings) has(key string) bool { _, ok := w[key]; return ok } + +// written reads every finding this module has written. +func (c *Checker) written() (writtenFindings, error) { + keys, err := c.Store.Keys(FindingsState) + if err != nil { + return writtenFindings{}, err + } + out := writtenFindings{} + for _, k := range keys { + raw, found, err := c.Store.Get(FindingsState, k) + if err != nil { + return out, err + } + if !found { + continue + } + f, err := health.DecodeFinding(raw) + if err != nil { + return out, fmt.Errorf("the finding %s is not readable: %w", k, err) + } + out[k] = f + } + return out, nil +} + +// write puts each finding whose words changed and deletes each one a source that ran whole no longer sees. A finding +// of a source that could not run is left as it is: a blind observer clears nothing (ADR 0227 §4). +func (c *Checker) write(written writtenFindings, want map[string]Observation, ran map[string]bool, run string) error { + var errs []string + keys := make([]string, 0, len(want)) + for k := range want { + keys = append(keys, k) + } + sort.Strings(keys) + now := c.Now() + for _, k := range keys { + f := want[k].Finding(run, now) + if was, ok := written[k]; ok && says(was) == says(f) { + continue + } + raw, err := health.EncodeFinding(f) + if err == nil { + err = c.Store.Put(FindingsState, k, json.RawMessage(raw)) + } + if err != nil { + errs = append(errs, k+": "+err.Error()) + } + } + for k, f := range written { + if _, still := want[k]; still || !ran[f.Source] { + continue + } + if err := c.Store.Delete(FindingsState, k); err != nil { + errs = append(errs, k+": "+err.Error()) + } + } + if len(errs) > 0 { + return fmt.Errorf("%s", strings.Join(errs, "; ")) + } + return nil +} + +// keep keeps the run as the last verdict and in the week's verdicts, and says it as `checked`. Not said is said in +// the log: the watchman's S10 says the self-check silent when `checked` stops. +func (c *Checker) keep(run Run) { + c.mu.Lock() + c.last = &run + c.mu.Unlock() + if err := c.Store.Put(VerdictState, LastKey, run); err != nil { + c.Logf("[%s] the run %s is not kept as the last verdict: %v", Module, run.Run, err) + } + if err := c.Store.Put(VerdictsState, run.Run, Overview(run)); err != nil { + c.Logf("[%s] the run %s is not kept among the week's verdicts: %v", Module, run.Run, err) + } + if c.Publish == nil { + return + } + if err := c.Publish(CheckedSubject, run, run.Run); err != nil { + c.mu.Lock() + c.unsaid++ + c.mu.Unlock() + c.Logf("[%s] the run %s could NOT be said as checked, so the watchers will say the self-check is silent: %v", + Module, run.Run, err) + } +} + +// overviewFindings is how many findings of one probe an overview carries (novox/hq issue 314). +const overviewFindings = 20 + +// Overview is a run with each probe's findings at most overviewFindings, with their count and how to read them all. +func Overview(run Run) Run { + probes := make([]ProbeVerdict, len(run.Probes)) + for i, p := range run.Probes { + total := len(p.Found) + len(p.Unconfirmed) + if total > overviewFindings { + p.FoundCount = len(p.Found) + if len(p.Found) > overviewFindings { + p.Found = p.Found[:overviewFindings:overviewFindings] + } + if room := overviewFindings - len(p.Found); len(p.Unconfirmed) > room { + p.Unconfirmed = p.Unconfirmed[:room:room] + } + p.More = fmt.Sprintf("%d found and %d unconfirmed, %d of them shown — `%s.doctor probe=%s` gives them all", + p.FoundCount, total-p.FoundCount, len(p.Found)+len(p.Unconfirmed), Seat, p.ID) + } + probes[i] = p + } + run.Probes = probes + return run +} + +// Keep runs the registry on its schedule until ctx ends: the first run FirstAfter the start, then every Every. +func (c *Checker) Keep(ctx context.Context) { + select { + case <-ctx.Done(): + return + case <-time.After(FirstAfter): + } + tick := time.NewTicker(Every) + defer tick.Stop() + for { + c.RunOnce(ctx, "the schedule") + select { + case <-ctx.Done(): + return + case <-tick.C: + } + } +} + +func firstLine(s string) string { + line, _, _ := strings.Cut(s, "\n") + return line +} diff --git a/modules/mesh-self-check/cmd/mesh-self-check/selfcheck_test.go b/modules/mesh-self-check/cmd/mesh-self-check/selfcheck_test.go new file mode 100644 index 00000000..50557340 --- /dev/null +++ b/modules/mesh-self-check/cmd/mesh-self-check/selfcheck_test.go @@ -0,0 +1,295 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "slices" + "sort" + "strings" + "sync" + "testing" + "time" + + "git.novox.be/novox/mesh-sdk/go/health" +) + +// memStore is the bus's buckets in memory, counting the writes. +type memStore struct { + mu sync.Mutex + kv map[string]map[string][]byte + puts map[string]int + keysErr error +} + +func newStore() *memStore { + return &memStore{kv: map[string]map[string][]byte{}, puts: map[string]int{}} +} + +func (s *memStore) Get(state, key string) ([]byte, bool, error) { + s.mu.Lock() + defer s.mu.Unlock() + v, ok := s.kv[state][key] + return v, ok, nil +} + +func (s *memStore) Put(state, key string, value any) error { + raw, err := json.Marshal(value) + if err != nil { + return err + } + s.mu.Lock() + defer s.mu.Unlock() + if s.kv[state] == nil { + s.kv[state] = map[string][]byte{} + } + s.kv[state][key] = raw + s.puts[state+"/"+key]++ + return nil +} + +func (s *memStore) Delete(state, key string) error { + s.mu.Lock() + defer s.mu.Unlock() + delete(s.kv[state], key) + return nil +} + +func (s *memStore) Keys(state string) ([]string, error) { + s.mu.Lock() + defer s.mu.Unlock() + if s.keysErr != nil && state == FindingsState { + return nil, s.keysErr + } + var out []string + for k := range s.kv[state] { + out = append(out, k) + } + sort.Strings(out) + return out, nil +} + +func (s *memStore) findingKeys() []string { + keys, _ := s.Keys(FindingsState) + return keys +} + +// checker is a self-check over memory, with a registry the test drives. +func checker(store *memStore, registry ...Probe) (*Checker, *[]string) { + var said []string + now := time.Date(2026, 10, 29, 21, 0, 0, 0, time.UTC) + c := &Checker{Store: store, Registry: registry, Host: "novox", Logf: func(string, ...any) {}, + Now: func() time.Time { now = now.Add(time.Second); return now }, + Publish: func(subject string, body any, id string) error { + raw, _ := json.Marshal(body) + said = append(said, subject+" "+id+" "+string(raw)) + return nil + }} + return c, &said +} + +// A run writes what a probe finds, writes nothing it already wrote, and deletes what it no longer sees; a probe that +// could not run clears nothing and is written as its own finding at the second run in a row; a finding one look can +// be wrong about waits for the second run; a deferred probe is never a pass (ADR 0564 §2, §6; issue 1952 SC1). +func TestARunWritesFindingsAsTheyChange(t *testing.T) { + var found []Observation + var fail error + steady := Probe{ID: "P1", Kind: "stream-wrong", Run: func(context.Context, *Checker) ([]Observation, error) { return found, fail }} + deferred := Probe{ID: "P2", Kind: "resolver-wrong", Deferred: "the overlay's addresses"} + missing := Observation{Token: "stream-missing", Severity: Urgent, Subject: Subject{Scope: ScopeBus, ID: "EVENTS"}, + Summary: "the stream EVENTS is not on the bus"} + behind := Observation{Kind: "consumer-behind", Token: "behind", Severity: Warning, Confirm: true, + Subject: Subject{Scope: ScopeBus, ID: "EVENTS.controller"}, Summary: "far behind"} + store := newStore() + c, said := checker(store, steady, deferred) + + steps := []struct { + name string + found []Observation + fail error + want []string // finding keys written after the run + verdict string + missingPs int // puts of the missing finding so far + }{ + {"a finding is written", []Observation{missing}, nil, []string{"bus.EVENTS.stream-missing"}, VerdictFail, 1}, + {"the same again is no write", []Observation{missing}, nil, []string{"bus.EVENTS.stream-missing"}, VerdictFail, 1}, + {"one look can be wrong: held", []Observation{missing, behind}, nil, []string{"bus.EVENTS.stream-missing"}, VerdictFail, 1}, + {"seen twice: written", []Observation{missing, behind}, nil, + []string{"bus.EVENTS.controller.behind", "bus.EVENTS.stream-missing"}, VerdictFail, 1}, + {"blind once: clears nothing, held", nil, errors.New("timed out"), + []string{"bus.EVENTS.controller.behind", "bus.EVENTS.stream-missing"}, VerdictFailedToRun, 1}, + {"blind twice: its own finding", nil, errors.New("timed out"), + []string{"bus.EVENTS.controller.behind", "bus.EVENTS.stream-missing", "probe.P1.failed"}, VerdictFailedToRun, 1}, + {"no longer seen: cleared", nil, nil, nil, VerdictPass, 1}, + } + for _, s := range steps { + found, fail = s.found, s.fail + run := c.RunOnce(context.Background(), "the test") + if got := store.findingKeys(); !slices.Equal(got, s.want) { + t.Errorf("%s: findings %v, want %v", s.name, got, s.want) + } + if run.Probes[0].Verdict != s.verdict { + t.Errorf("%s: P1 %s, want %s", s.name, run.Probes[0].Verdict, s.verdict) + } + if run.Probes[1].Verdict != VerdictDeferred || !strings.Contains(run.Probes[1].Error, "overlay") { + t.Errorf("%s: a deferred probe answered %+v", s.name, run.Probes[1]) + } + if n := store.puts[FindingsState+"/bus.EVENTS.stream-missing"]; n != s.missingPs { + t.Errorf("%s: the unchanged finding was written %d times", s.name, n) + } + } + raw, _, _ := store.Get(FindingsState, "probe.P1.failed") + if raw != nil { + t.Errorf("the blind finding outlived the probe running again: %s", raw) + } + // Every run is kept, the last as `last`, and said as checked with the controller's heartbeat body. + if keys, _ := store.Keys(VerdictsState); len(keys) != len(steps) { + t.Errorf("verdicts kept %d runs, want %d", len(keys), len(steps)) + } + // Said on the seat's subject, where its consumers hear it, de-duplicated by the run's id. + if len(*said) != len(steps) || !strings.HasPrefix((*said)[0], "mesh.seat.mesh-self-check.event.checked doctor-") { + t.Fatalf("said %v", *said) + } + _, rest, _ := strings.Cut((*said)[0], " ") + _, rest, _ = strings.Cut(rest, " ") + var body map[string]any + _ = json.Unmarshal([]byte(rest), &body) + for _, field := range []string{"run", "at", "started", "took", "interval-seconds", "counts", "probes", "controller", "why"} { + if _, ok := body[field]; !ok { + t.Errorf("checked's body lacks %s, which doctor-heartbeat carries: %v", field, body) + } + } + last, _, _ := store.Get(VerdictState, LastKey) + var r Run + _ = json.Unmarshal(last, &r) + if r.Counts.Passed != 1 || r.Counts.Deferred != 1 { + t.Errorf("the last verdict counts %+v", r.Counts) + } + // A finding carries the operator's words of its kind, the probe that found it and who observed it. + found, fail = []Observation{missing}, nil + c.RunOnce(context.Background(), "the test") + raw, _, _ = store.Get(FindingsState, "bus.EVENTS.stream-missing") + f, err := health.DecodeFinding(raw) + if err != nil || f.Kind != "stream-wrong" || f.Source != "P1" || f.Observer != Module || f.Headline == "" || + f.Explanation == "" || f.Token != "stream-missing" || len(f.Evidence) != 1 || f.Run == "" { + t.Errorf("the written finding is %+v", f) + } + // Blind to its own bucket, a run writes and clears nothing. + store.keysErr = errors.New("the bus is away") + found = nil + c.RunOnce(context.Background(), "the test") + store.keysErr = nil + if got := store.findingKeys(); !slices.Equal(got, []string{"bus.EVENTS.stream-missing"}) { + t.Errorf("a run blind to its findings changed them: %v", got) + } +} + +// A probe that panics or overruns its bound could not run: never a pass. +func TestAProbeThatPanicsOrOverrunsCouldNotRun(t *testing.T) { + within := Within + Within = 50 * time.Millisecond + t.Cleanup(func() { Within = within }) + for name, run := range map[string]func(context.Context, *Checker) ([]Observation, error){ + "panic": func(context.Context, *Checker) ([]Observation, error) { panic("boom") }, + "overrun": func(ctx context.Context, _ *Checker) ([]Observation, error) { time.Sleep(time.Second); return nil, nil }, + } { + c, _ := checker(newStore(), Probe{ID: "P", Kind: "k", Run: run}) + c.Now = time.Now + r := c.RunOnce(context.Background(), "the test") + if r.Probes[0].Verdict != VerdictFailedToRun || r.Counts.Passed != 0 { + t.Errorf("%s: %+v", name, r.Probes[0]) + } + } +} + +// The verb answers the last run, one probe whole, a run now, and before any run says when one comes; the reads answer +// every state the module keeps (ADR 0564 §12). +func TestTheVerbsAndReads(t *testing.T) { + store := newStore() + many := make([]Observation, 25) + for i := range many { + many[i] = Observation{Severity: Warning, Subject: Subject{Scope: ScopeBus, ID: fmt.Sprintf("S%02d", i)}, Summary: "x"} + } + c, _ := checker(store, Probe{ID: "D7", Kind: "stream-wrong", + Run: func(context.Context, *Checker) ([]Observation, error) { return many, nil }}) + if _, err := c.Doctor(false, "", "x"); err == nil || !strings.Contains(err.Error(), "first run") { + t.Fatalf("before a run: %v", err) + } + if _, err := c.Doctor(true, "", "asked by person.jochen"); err != nil { + t.Fatal(err) + } + over, _ := c.Doctor(false, "", "") + if v := over["run"].(Run).Probes[0]; len(v.Found) != overviewFindings || v.More == "" { + t.Errorf("the overview carries %d findings, more %q", len(v.Found), v.More) + } + whole, _ := c.Doctor(false, "d7", "") + if v := whole["run"].(Run).Probes[0]; len(v.Found) != 25 { + t.Errorf("one probe whole carries %d findings", len(v.Found)) + } + if _, err := c.Doctor(false, "D99", ""); err == nil { + t.Error("an unknown probe answered") + } + // A fresh checker over the same bucket answers the last verdict from it. + fresh, _ := checker(store) + if _, err := fresh.Doctor(false, "", ""); err != nil { + t.Errorf("the last verdict is not read from its bucket: %v", err) + } + fs, err := c.Findings("") + if err != nil || fs.(map[string]any)["count"] != 25 { + t.Errorf("findings answered %v, %v", fs, err) + } + if one, err := c.Findings("bus.S03.stream-wrong"); err != nil || !strings.Contains(string(one.(json.RawMessage)), "S03") { + t.Errorf("one finding answered %v, %v", one, err) + } + vs, err := c.Verdicts("", 0) + if err != nil || len(vs.(map[string]any)["verdicts"].([]map[string]any)) != 1 { + t.Errorf("verdicts answered %v, %v", vs, err) + } + if _, err := c.Verdicts("doctor-1-1", 0); err == nil { + t.Error("an unknown run answered") + } + if p := c.Probes()["probes"].([]map[string]any); len(p) != 1 || p[0]["id"] != "D7" { + t.Errorf("probes answered %v", p) + } +} + +// Only the probes the setting `probes` lists run: one not listed runs nothing, has no verdict, clears what it raised before, +// and is reported by the verb as off by setting; the default list is D13 and D-fill (the operator, 2026-10-30). +func TestOnlyTheProbesTheSettingListsRun(t *testing.T) { + ran := map[string]int{} + finds := Observation{Token: "stream-missing", Severity: Urgent, Subject: Subject{Scope: ScopeBus, ID: "EVENTS"}, Summary: "gone"} + probe := func(id string) Probe { + return Probe{ID: id, Kind: "stream-wrong", Run: func(context.Context, *Checker) ([]Observation, error) { + ran[id]++ + return []Observation{finds}, nil + }} + } + store := newStore() + c, _ := checker(store, probe("D7"), probe("D13")) + c.Enabled = func() []string { return []string{"D7", "D13"} } + run := c.RunOnce(context.Background(), "test") + if ran["D7"] != 1 || ran["D13"] != 1 || len(run.Probes) != 2 || len(store.findingKeys()) != 1 { + t.Fatalf("both listed: ran %v, %d verdicts, findings %v", ran, len(run.Probes), store.findingKeys()) + } + c.Enabled = func() []string { return []string{"D13"} } + run = c.RunOnce(context.Background(), "test") + if ran["D7"] != 1 || ran["D13"] != 2 || len(run.Probes) != 1 || run.Probes[0].ID != "D13" { + t.Fatalf("D7 unlisted: ran %v, verdicts %+v", ran, run.Probes) + } + // What the unlisted probe raised stays only as long as another listed probe sees the same key (D13 raises the same + // key here); an unlisted probe alone clears its findings. + c.Enabled = func() []string { return nil } + c.RunOnce(context.Background(), "test") + if len(store.findingKeys()) != 0 { + t.Fatalf("with nothing listed, findings are %v", store.findingKeys()) + } + rows, _ := c.Probes()["probes"].([]map[string]any) + if len(rows) != 2 || rows[0]["off"] != "off by setting" || rows[1]["off"] != "off by setting" { + t.Fatalf("the verb says %v", rows) + } + c.Enabled = nil + if !c.On("anything") || !slices.Equal(DefaultProbes, []string{"D13", "D-fill"}) { + t.Fatal("no setting runs every probe, and the default list is D13 and D-fill") + } +} diff --git a/modules/mesh-self-check/cmd/mesh-self-check/tools.go b/modules/mesh-self-check/cmd/mesh-self-check/tools.go new file mode 100644 index 00000000..a4377b73 --- /dev/null +++ b/modules/mesh-self-check/cmd/mesh-self-check/tools.go @@ -0,0 +1,218 @@ +package main + +// The seat's verbs and the module's own tools (novox/hq ADR 0564 §6, §12): `doctor` answers the last verdict at once, +// runs now, or gives one probe's verdict whole; `probes` is the registry; and a read of each state it keeps — the +// findings, the last verdict and the week's verdicts — so a person sees all it holds and a recovery is a call. + +import ( + "context" + "encoding/json" + "fmt" + "sort" + "strings" + "time" + + stdio "git.novox.be/novox/mesh-sdk/go" +) + +// Verbs are the seat's verbs, as the manifest claims them. +var Verbs = []string{"doctor", "probes"} + +// Tool names: the seat's verbs, and the module's own reads. +const ( + DoctorTool = Seat + ".doctor" + ProbesTool = Seat + ".probes" + FindingsTool = "self_check_findings" + VerdictsTool = "self_check_verdicts" +) + +func obj(props map[string]any) map[string]any { + return map[string]any{"type": "object", "properties": props} +} + +// Tools are what the runtime serves. +func Tools(c *Checker) []stdio.Tool { + return []stdio.Tool{ + {Name: DoctorTool, + Description: "The mesh's self-check (novox/hq ADR 0564 §6): the last run's verdict at once, each probe passed, " + + "failed with what it found, could not run (never a pass) or deferred with the facts it lacks; run=true runs " + + "every probe now and answers that run; probe= gives one probe's verdict whole, every finding.", + Input: obj(map[string]any{ + "run": map[string]any{"type": "boolean", "description": "run every probe now"}, + "probe": map[string]any{"type": "string", "description": "one probe's verdict, whole, e.g. D6"}}), + Run: func(a map[string]any) (any, error) { + return c.Doctor(boolArg(a, "run"), strArg(a, "probe"), "asked by "+stdio.CallerOf(a)) + }}, + {Name: ProbesTool, + Description: "The self-check's registry: each probe's id, what it asserts, where it came from, the condition kinds " + + "it raises, and for one deferred the facts it lacks on the bus.", + Run: func(map[string]any) (any, error) { return c.Probes(), nil }}, + {Name: FindingsTool, + Description: "What the self-check finds wrong now, as the condition keeper reads it (its bucket findings): every " + + "finding's key, kind, severity, the probe that found it and its summary; or one finding whole by key.", + Input: obj(map[string]any{"key": map[string]any{"type": "string", "description": "one finding's key"}}), + Run: func(a map[string]any) (any, error) { return c.Findings(strArg(a, "key")) }}, + {Name: VerdictsTool, + Description: "The self-check's runs of the last week (its bucket verdicts, each kept a week) newest first, each " + + "with its counts; or one run whole by its id. The last run is the doctor verb's.", + Input: obj(map[string]any{ + "run": map[string]any{"type": "string", "description": "one run's id, e.g. doctor-1793284800-12"}, + "limit": map[string]any{"type": "number", "description": "at most this many, newest (default 50)"}}), + Run: func(a map[string]any) (any, error) { return c.Verdicts(strArg(a, "run"), intArg(a, "limit", 50)) }}, + } +} + +// Doctor is the verb: the last verdict, a run now, or one probe of either. +func (c *Checker) Doctor(run bool, probe, why string) (map[string]any, error) { + var r Run + switch { + case run: + r = c.RunOnce(context.Background(), why) + case c.LastRun() != nil: + r = *c.LastRun() + default: + raw, found, err := c.Store.Get(VerdictState, LastKey) + if err != nil { + return nil, fmt.Errorf("the last verdict cannot be read: %w", err) + } + if !found { + return nil, fmt.Errorf("the self-check has not finished its first run yet: it runs %s after it starts, then "+ + "every %s — run=true runs it now", FirstAfter, Every) + } + if err := json.Unmarshal(raw, &r); err != nil { + return nil, fmt.Errorf("the last verdict is not readable: %w", err) + } + } + if probe != "" { + one, ok := oneProbe(r, probe) + if !ok { + return nil, fmt.Errorf("the run %s has no probe %q — %s lists them", r.Run, probe, ProbesTool) + } + r = one + } else { + r = Overview(r) + } + return map[string]any{"run": r, "age": c.Now().Sub(r.At).Round(time.Second).String(), + "note": "a probe that could not run is never a pass; each finding is a condition the keeper raises until a run no " + + "longer sees it, and one a single look can be wrong about is written when two runs in a row see it"}, nil +} + +func oneProbe(r Run, id string) (Run, bool) { + for _, p := range r.Probes { + if strings.EqualFold(p.ID, id) { + r.Probes = []ProbeVerdict{p} + return r, true + } + } + return Run{}, false +} + +// Probes is the registry as the verb answers it. +func (c *Checker) Probes() map[string]any { + var out []map[string]any + for _, p := range c.Registry { + row := map[string]any{"id": p.ID, "asserts": p.Asserts, "from": p.From, "kind": p.Kind} + if len(p.Raises) > 0 { + row["raises"] = p.Raises + } + if !c.On(p.ID) { + row["off"] = "off by setting" + } else if p.Run == nil { + row["deferred"] = p.Deferred + } + out = append(out, row) + } + return map[string]any{"probes": out, "every": Every.String(), "each within": Within.String()} +} + +// Findings are the findings written now, or one whole. +func (c *Checker) Findings(key string) (any, error) { + if key != "" { + raw, found, err := c.Store.Get(FindingsState, key) + if err != nil { + return nil, err + } + if !found { + return nil, fmt.Errorf("no finding is written under %s now", key) + } + return json.RawMessage(raw), nil + } + written, err := c.written() + if err != nil { + return nil, fmt.Errorf("the findings cannot be read: %w", err) + } + rows := []map[string]any{} + for k, f := range written { + rows = append(rows, map[string]any{"key": k, "kind": f.Kind, "severity": f.Severity, "probe": f.Source, + "summary": f.Summary}) + } + sort.Slice(rows, func(i, j int) bool { return rows[i]["key"].(string) < rows[j]["key"].(string) }) + return map[string]any{"findings": rows, "count": len(rows)}, nil +} + +// Verdicts are the week's runs newest first, or one whole. +func (c *Checker) Verdicts(run string, limit int) (any, error) { + if run != "" { + raw, found, err := c.Store.Get(VerdictsState, run) + if err != nil { + return nil, err + } + if !found { + return nil, fmt.Errorf("no run %s is kept: runs are kept a week", run) + } + return json.RawMessage(raw), nil + } + keys, err := c.Store.Keys(VerdictsState) + if err != nil { + return nil, fmt.Errorf("the week's verdicts cannot be read: %w", err) + } + sort.Slice(keys, func(i, j int) bool { return runOrder(keys[i]) > runOrder(keys[j]) }) + total := len(keys) + if limit > 0 && len(keys) > limit { + keys = keys[:limit] + } + rows := []map[string]any{} + for _, k := range keys { + raw, found, err := c.Store.Get(VerdictsState, k) + if err != nil || !found { + continue + } + var r Run + if json.Unmarshal(raw, &r) == nil { + rows = append(rows, map[string]any{"run": r.Run, "at": r.At, "why": r.Why, "counts": r.Counts}) + } + } + return map[string]any{"verdicts": rows, "kept": total, "kept-for": "a week"}, nil +} + +// runOrder orders run ids `doctor--` by time, then number. +func runOrder(id string) string { + parts := strings.Split(id, "-") + if len(parts) != 3 { + return id + } + return fmt.Sprintf("%020s-%020s", parts[1], parts[2]) +} + +func strArg(a map[string]any, k string) string { + s, _ := a[k].(string) + return strings.TrimSpace(s) +} + +// boolArg reads a flag as a boolean or as the seat's "true". +func boolArg(a map[string]any, k string) bool { + switch v := a[k].(type) { + case bool: + return v + case string: + return strings.EqualFold(strings.TrimSpace(v), "true") + } + return false +} + +func intArg(a map[string]any, k string, otherwise int) int { + if n, ok := a[k].(float64); ok && n > 0 { + return int(n) + } + return otherwise +} diff --git a/modules/mesh-self-check/cmd/mesh-self-check/whole_integration_test.go b/modules/mesh-self-check/cmd/mesh-self-check/whole_integration_test.go new file mode 100644 index 00000000..54ae1679 --- /dev/null +++ b/modules/mesh-self-check/cmd/mesh-self-check/whole_integration_test.go @@ -0,0 +1,298 @@ +//go:build integration + +package main + +// The self-check whole against a real bus (novox/hq ADR 0564 §6, issue 1952 SC1): the built binary, played over its +// real stdin and stdout as the node's tool runtime plays it (tools/call; mesh/state.* answered from a real +// nats-server's key-value buckets, mesh/publish onto its subjects, and the bus module's nats_streams answered from +// that server's own streams). One run asked through the seat's verb finds the mesh's streams missing on a bare bus, +// writes them as findings into its bucket on the bus, keeps the verdict, and says checked. + +import ( + "bufio" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" + + "git.novox.be/novox/mesh-sdk/go/health" + "git.novox.be/novox/mesh-sdk/go/testbus" +) + +type wire struct { + ID json.RawMessage `json:"id,omitempty"` + Method string `json:"method,omitempty"` + Params json.RawMessage `json:"params,omitempty"` + Result json.RawMessage `json:"result,omitempty"` + Error *struct { + Message string `json:"message"` + } `json:"error,omitempty"` +} + +// player is the node's tool runtime, as far as the self-check needs it, over a real bus. +type player struct { + ctx context.Context + nc *nats.Conn + js jetstream.JetStream + mu sync.Mutex + in io.Writer + next int + pending map[string]chan wire +} + +func (p *player) send(m map[string]any) { + raw, _ := json.Marshal(m) + p.mu.Lock() + defer p.mu.Unlock() + _, _ = p.in.Write(append(raw, '\n')) +} + +func (p *player) answer(id json.RawMessage, result any, err error) { + if err != nil { + p.send(map[string]any{"id": id, "error": map[string]any{"code": -32000, "message": err.Error()}}) + return + } + p.send(map[string]any{"id": id, "result": result}) +} + +func (p *player) serve(out io.Reader) { + scanner := bufio.NewScanner(out) + scanner.Buffer(make([]byte, 1<<20), 1<<24) + for scanner.Scan() { + var m wire + if json.Unmarshal(scanner.Bytes(), &m) != nil { + continue + } + if m.Method == "" { + var key string + _ = json.Unmarshal(m.ID, &key) + p.mu.Lock() + ch := p.pending[key] + delete(p.pending, key) + p.mu.Unlock() + if ch != nil { + ch <- m + } + continue + } + go p.handle(m) + } +} + +func (p *player) kv(state string) (jetstream.KeyValue, error) { + return p.js.KeyValue(p.ctx, Module+"_"+state) +} + +func (p *player) handle(m wire) { + var a struct { + State string `json:"state"` + Key string `json:"key"` + Value json.RawMessage `json:"value"` + Body json.RawMessage `json:"body"` + } + _ = json.Unmarshal(m.Params, &a) + switch m.Method { + case "mesh/state.get": + kv, err := p.kv(a.State) + if err != nil { + p.answer(m.ID, nil, err) + return + } + e, err := kv.Get(p.ctx, a.Key) + if errors.Is(err, jetstream.ErrKeyNotFound) { + p.answer(m.ID, nil, nil) + return + } + if err != nil { + p.answer(m.ID, nil, err) + return + } + p.answer(m.ID, map[string]any{"key": e.Key(), "value": json.RawMessage(e.Value()), "revision": e.Revision()}, nil) + case "mesh/state.put": + kv, err := p.kv(a.State) + if err == nil { + var rev uint64 + if rev, err = kv.Put(p.ctx, a.Key, a.Value); err == nil { + p.answer(m.ID, map[string]any{"revision": rev}, nil) + return + } + } + p.answer(m.ID, nil, err) + case "mesh/state.keys": + kv, err := p.kv(a.State) + if err != nil { + p.answer(m.ID, nil, err) + return + } + keys, err := kv.Keys(p.ctx) + if errors.Is(err, jetstream.ErrNoKeysFound) { + keys, err = []string{}, nil + } + p.answer(m.ID, keys, err) + case "mesh/state.delete": + kv, err := p.kv(a.State) + if err == nil { + err = kv.Delete(p.ctx, a.Key) + } + p.answer(m.ID, map[string]any{}, err) + case "mesh/seat.publish": + var e struct { + Subject string `json:"subject"` + Body json.RawMessage `json:"body"` + } + _ = json.Unmarshal(m.Params, &e) + p.answer(m.ID, map[string]any{}, p.nc.Publish(e.Subject, e.Body)) + case "mesh/publish": + var e struct { + Key string `json:"key"` + Body json.RawMessage `json:"body"` + } + _ = json.Unmarshal(m.Params, &e) + p.answer(m.ID, map[string]any{}, p.nc.Publish("mesh.mod."+Module+".event."+e.Key, e.Body)) + case "mesh/ask": + var ask struct { + Key string `json:"key"` + } + _ = json.Unmarshal(m.Params, &ask) + if ask.Key != StreamsTool { + p.answer(m.ID, nil, errors.New("nats: no responders available for request")) + return + } + // The bus module's answer, from the server's own streams. + var detail []map[string]any + for name := range p.js.StreamNames(p.ctx).Name() { + detail = append(detail, map[string]any{"name": name}) + } + p.answer(m.ID, map[string]any{"detail": detail}, nil) + default: + p.answer(m.ID, nil, errors.New("nats: no responders available for request")) + } +} + +func (p *player) call(name string, args map[string]any) (json.RawMessage, error) { + p.mu.Lock() + p.next++ + id := fmt.Sprintf("c-%d", p.next) + ch := make(chan wire, 1) + p.pending[id] = ch + p.mu.Unlock() + p.send(map[string]any{"id": id, "method": "tools/call", "params": map[string]any{"name": name, "arguments": args, + "_meta": map[string]any{"mesh/caller": "person.jochen"}}}) + select { + case m := <-ch: + if m.Error != nil { + return nil, errors.New(m.Error.Message) + } + var r struct { + Content []struct { + Text string `json:"text"` + } `json:"content"` + IsError bool `json:"isError"` + } + if err := json.Unmarshal(m.Result, &r); err != nil || len(r.Content) != 1 { + return nil, fmt.Errorf("%s answered %s", name, m.Result) + } + if r.IsError { + return nil, errors.New(r.Content[0].Text) + } + return json.RawMessage(r.Content[0].Text), nil + case <-time.After(60 * time.Second): + return nil, fmt.Errorf("%s did not answer in 60s", name) + } +} + +// 1952/SC1: one run of the self-check whole ends with checked and a verdict, its findings on the bus. +func TestTheSelfCheckWholeRunsOnTheBus(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + dir := t.TempDir() + bin := filepath.Join(dir, Module) + build := exec.Command("go", "build", "-o", bin, ".") + build.Stderr = os.Stderr + if err := build.Run(); err != nil { + t.Fatalf("the self-check does not build: %v", err) + } + nc, err := nats.Connect(testbus.URL(t)) + if err != nil { + t.Fatal(err) + } + t.Cleanup(nc.Close) + js, err := jetstream.New(nc) + if err != nil { + t.Fatal(err) + } + for _, s := range []string{FindingsState, VerdictState, VerdictsState} { + if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: Module + "_" + s}); err != nil { + t.Fatal(err) + } + } + checked, err := nc.SubscribeSync("mesh.seat." + Seat + ".event." + CheckedEvent) + if err != nil { + t.Fatal(err) + } + cmd := exec.CommandContext(ctx, bin) + cmd.Env = append(os.Environ(), "MESH_NODE=ace", "MESH_MODULE="+Module) + stdin, _ := cmd.StdinPipe() + stdout, _ := cmd.StdoutPipe() + cmd.Stderr = os.Stderr + if err := cmd.Start(); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = stdin.Close(); cancel(); _ = cmd.Wait() }) + p := &player{ctx: ctx, nc: nc, js: js, in: stdin, pending: map[string]chan wire{}} + go p.serve(stdout) + + raw, err := p.call(DoctorTool, map[string]any{"run": true}) + if err != nil { + t.Fatal(err) + } + var answer struct { + Run Run `json:"run"` + } + if err := json.Unmarshal(raw, &answer); err != nil { + t.Fatal(err) + } + verdicts := map[string]string{} + for _, v := range answer.Run.Probes { + verdicts[v.ID] = v.Verdict + } + if verdicts["D7"] != VerdictFail || verdicts["DW"] != VerdictFail || verdicts["D15"] != VerdictFailedToRun || + verdicts["D2"] != VerdictDeferred || answer.Run.Controller != "ace" || !strings.Contains(answer.Run.Why, "person.jochen") { + t.Errorf("the run answered %+v", answer.Run) + } + msg, err := checked.NextMsg(10 * time.Second) + if err != nil { + t.Fatalf("checked was not said: %v", err) + } + var said Run + if err := json.Unmarshal(msg.Data, &said); err != nil || said.Run != answer.Run.Run { + t.Errorf("checked said %s", msg.Data) + } + findings, _ := js.KeyValue(ctx, Module+"_"+FindingsState) + e, err := findings.Get(ctx, "bus.EVENTS.stream-missing") + if err != nil { + t.Fatalf("the missing stream is not a finding on the bus: %v", err) + } + f, err := health.DecodeFinding(e.Value()) + if err != nil || f.Kind != "stream-wrong" || f.Severity != Urgent || f.Source != "D7" { + t.Errorf("the finding is %s", e.Value()) + } + verdict, _ := js.KeyValue(ctx, Module+"_"+VerdictState) + if _, err := verdict.Get(ctx, LastKey); err != nil { + t.Errorf("the last verdict is not kept: %v", err) + } + if raw, err := p.call(VerdictsTool, map[string]any{}); err != nil || !strings.Contains(string(raw), answer.Run.Run) { + t.Errorf("the week's verdicts answered %s, %v", raw, err) + } +} diff --git a/modules/mesh-self-check/go.mod b/modules/mesh-self-check/go.mod new file mode 100644 index 00000000..06acc2cf --- /dev/null +++ b/modules/mesh-self-check/go.mod @@ -0,0 +1,22 @@ +module mesh-self-check + +go 1.26.0 + +require ( + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 + github.com/nats-io/nats.go v1.54.0 +) + +require ( + github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op // indirect + github.com/google/go-tpm v0.9.8 // indirect + github.com/klauspost/compress v1.20.0 // indirect + github.com/minio/highwayhash v1.0.4 // indirect + github.com/nats-io/jwt/v2 v2.8.1 // indirect + github.com/nats-io/nats-server/v2 v2.11.17 // indirect + github.com/nats-io/nkeys v0.4.16 // indirect + github.com/nats-io/nuid v1.0.1 // indirect + golang.org/x/crypto v0.57.0 // indirect + golang.org/x/sys v0.48.0 // indirect + golang.org/x/time v0.15.0 // indirect +) diff --git a/modules/mesh-self-check/go.sum b/modules/mesh-self-check/go.sum new file mode 100644 index 00000000..a2b8e018 --- /dev/null +++ b/modules/mesh-self-check/go.sum @@ -0,0 +1,27 @@ +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 h1:lKU3HjNXGiVbLW2hUgFjU/TMTUYMlNnnmZxG5T7biQA= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= +github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= +github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= +github.com/google/go-tpm v0.9.8/go.mod h1:h9jEsEECg7gtLis0upRBQU+GhYVH6jMjrFxI8u6bVUY= +github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA= +github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI= +github.com/minio/highwayhash v1.0.4 h1:asJizugGgchQod2ja9NJlGOWq4s7KsAWr5XUc9Clgl4= +github.com/minio/highwayhash v1.0.4/go.mod h1:GGYsuwP/fPD6Y9hMiXuapVvlIUEhFhMTh0rxU3ik1LQ= +github.com/nats-io/jwt/v2 v2.8.1 h1:V0xpGuD/N8Mi+fQNDynXohVvp7ZztevW5io8CUWlPmU= +github.com/nats-io/jwt/v2 v2.8.1/go.mod h1:nWnOEEiVMiKHQpnAy4eXlizVEtSfzacZ1Q43LIRavZg= +github.com/nats-io/nats-server/v2 v2.11.17 h1:GKEghcFK6A+aFx11Yf1LjgLC3txAwvyhnYzhBIQZA8I= +github.com/nats-io/nats-server/v2 v2.11.17/go.mod h1:B1sFVz4StNosQ903ak4N1G01Fl/9f8e06mXpFIE2K24= +github.com/nats-io/nats.go v1.54.0 h1:vsXoOxjHp/GmPUN+EcI7uOf/uB+iAP+kEsAFNQN0yzA= +github.com/nats-io/nats.go v1.54.0/go.mod h1:y+DZoD1oBOYfZTU681eTUiUjI0vbqYGixNVFHcjHJ0k= +github.com/nats-io/nkeys v0.4.16 h1:rd5oAuLOb8mnAycB0xleuEBNS1pVVnN0fv/FF34Eypg= +github.com/nats-io/nkeys v0.4.16/go.mod h1:llLgWoI0o4z/Q57q2R1kHfmocyhGV6VG/U18Glg1Afs= +github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= +github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= +golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M= +golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA= +golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo= +golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= +golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= +golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= diff --git a/modules/mesh-self-check/module.json b/modules/mesh-self-check/module.json new file mode 100644 index 00000000..e4037e98 --- /dev/null +++ b/modules/mesh-self-check/module.json @@ -0,0 +1,87 @@ +{ + "module": "mesh-self-check", + "version": "1", + "runs-as": "mesh-self-check", + "claims": [ + { + "name": "mesh-self-check", + "scope": "mesh", + "serves": [ + "doctor", + "probes" + ] + } + ], + "invokes": [ + "nats.nats_streams", + "nats.nats_closed_connections", + "seat:mesh-watchdogs.signals", + "seat:node-intrusion-prevention.banned" + ], + "reads": [ + "mesh-delivery-assigner.placement", + "mesh-delivery-assigner.seat-holders", + "mesh-delivery-watchman.presence", + "mesh-host.health", + "disk-load.readings", + "restic.good", + "mesh-delivery-indexer.modules", + "mesh-delivery-indexer.manifests" + ], + "state": [ + "findings", + "verdict", + { + "name": "verdicts", + "ttl-seconds": 604800 + } + ], + "own-secrets": { + "broker": "${dir:state}/broker" + }, + "secrets-owner": "mesh-self-check", + "resources": [ + { + "id": "account", + "type": "user", + "name": "mesh-self-check", + "shell": "/usr/bin/nologin", + "home": "/var/lib/mesh-self-check" + }, + { + "id": "state", + "type": "directory", + "mode": "0700", + "owner": "mesh-self-check", + "place": ".", + "unknown": "program" + } + ], + "build": { + "artifacts": [ + { + "name": "tools", + "kind": "bundle", + "language": "go", + "system": "arch", + "from": "cmd/mesh-self-check", + "binary": "mesh-self-check", + "loads": [ + "mesh-self-check" + ] + } + ] + }, + "config": { + "probes": { + "type": "list", + "description": "The ids of the probes that run, each judging one thing of the mesh (mesh-self-check.probes lists them with what each asserts). A probe not listed runs nothing and raises nothing, and what it raised before is cleared. D8 and DA are in the code and off by default.", + "trusted": true, + "default": [ + "D13", + "D-fill" + ], + "runtime": true + } + } +} diff --git a/modules/mesh-vault/cmd/mesh-vault/accounts.go b/modules/mesh-vault/cmd/mesh-vault/accounts.go index e25aff28..c102337e 100644 --- a/modules/mesh-vault/cmd/mesh-vault/accounts.go +++ b/modules/mesh-vault/cmd/mesh-vault/accounts.go @@ -53,6 +53,8 @@ type AccountHash struct { Adopted bool `json:"adopted,omitempty"` From string `json:"from,omitempty"` Node string `json:"node,omitempty"` + // Rotating is a rotation under way over a second login (rotation.go, ADR 0561 §4): absent when none is. + Rotating *Rotating `json:"rotating,omitempty"` } // AccountStore is the vault's state of bus-account hashes: read and written by the vault's instances alone. @@ -61,6 +63,8 @@ type AccountStore interface { Put(a AccountHash) error // Users is every user with a key. Users() ([]string, error) + // Delete takes a user's key out: an account retired (retire.go). + Delete(user string) error } // busAccounts is AccountStore over the vault's own state through its runtime. @@ -80,6 +84,8 @@ func (busAccounts) Get(user string) (*AccountHash, error) { func (busAccounts) Users() ([]string, error) { return stdio.State(BusAccountsState).Keys() } +func (busAccounts) Delete(user string) error { return stdio.State(BusAccountsState).Delete(user) } + func (busAccounts) Put(a AccountHash) error { _, err := stdio.State(BusAccountsState).Put(a.User, a) return err @@ -181,6 +187,10 @@ func (v *Vault) HashAccount(secret string, force bool) (Hashed, error) { if err != nil { return Hashed{}, err } + if rec.Retired != nil { + v.hashes.paid(secret) + return Hashed{}, fmt.Errorf("%s is retired (%s): its hash is written again only when it is wanted again", secret, rec.Retired.Why) + } user, password, err := accountOf(secret, value) if err != nil { return Hashed{}, err @@ -259,7 +269,7 @@ func (v *Vault) HashHeld() (written []Hashed, failed map[string]string) { return nil, failed } for _, r := range records { - if !Covered(r.Name) { + if !Covered(r.Name) || r.Retired != nil { continue } h, err := v.HashAccount(r.Name, false) diff --git a/modules/mesh-vault/cmd/mesh-vault/adopt.go b/modules/mesh-vault/cmd/mesh-vault/adopt.go index 2e6e998b..5d005ca3 100644 --- a/modules/mesh-vault/cmd/mesh-vault/adopt.go +++ b/modules/mesh-vault/cmd/mesh-vault/adopt.go @@ -58,32 +58,6 @@ type adoptionKept struct { failed string // why the last look found nothing to adopt, or "" } -// busNode is the node the assignments name as holding the bus module: exactly one. -func (v *Vault) busNode() (string, error) { - if v.Assignments.Keys == nil { - return "", errors.New("the vault cannot read the assignments, so it cannot tell which node holds the bus") - } - keys, err := v.Assignments.Keys() - if err != nil { - return "", fmt.Errorf("the assignments' keys could not be read: %w", err) - } - var nodes []string - for _, k := range keys { - if node, module, ok := strings.Cut(k, "."); ok && module == BusModule { - nodes = append(nodes, node) - } - } - sort.Strings(nodes) - switch len(nodes) { - case 1: - return nodes[0], nil - case 0: - return "", fmt.Errorf("no node is assigned the bus module %s", BusModule) - } - return "", fmt.Errorf("%d nodes are assigned the bus module %s (%s): the vault adopts from one alone", len(nodes), - BusModule, strings.Join(nodes, ", ")) -} - // AdoptNow is the tool account_adopt: the bus node's have read now and adopted, though it says the same as before. func (v *Vault) AdoptNow(caller string) (Adoption, error) { if v.Haves == nil { @@ -143,6 +117,9 @@ func (v *Vault) adopt(node string, raw []byte, by string, force bool) (Adoption, if err != nil { return Adoption{}, fmt.Errorf("the have of %s on %s is not a have: %w", BusModule, node, err) } + if have.UserList != nil { + v.LoadedListHeard(have.UserList.Revision) + } if have.UserList == nil || have.UserList.Handed == nil { return Adoption{}, fmt.Errorf("the have of %s on %s hands no hashes yet (its node-engine hands them once it "+ "heard the vault's key in its answer to node-key, and writes the list)", BusModule, node) diff --git a/modules/mesh-vault/cmd/mesh-vault/askers.go b/modules/mesh-vault/cmd/mesh-vault/askers.go index e5267120..cad8a1de 100644 --- a/modules/mesh-vault/cmd/mesh-vault/askers.go +++ b/modules/mesh-vault/cmd/mesh-vault/askers.go @@ -1,6 +1,8 @@ package main import ( + "git.novox.be/novox/mesh-sdk/go/provisions" + "encoding/json" "fmt" "strings" @@ -72,8 +74,23 @@ func (l AllowList) May(caller, name string) (bool, string) { // internal/broker, KindProvisioner). const ProvisionerPrefix = "provisioner." -// AssignmentsRead is the controller's assignments mirror as this module reads it (its manifest's `reads`). -const AssignmentsRead = "mesh-controller.assignments" +// AssignmentsRead is where the vault reads each module's provisioning plan on each node (its manifest's `reads`): the +// seat mesh-provisioner's state `plans`, `.` (novox/hq ADR 0570 clause 2), no longer the controller's +// assignments. A plan is read as the assignment's value carried it (asAssignment), so the rules that read a plan are +// unchanged. +const AssignmentsRead = provisions.PlansRead + +// asAssignment is a plan's key in the form the vault's rules read: `{node, module, provisioning: plan}`, nil for none. +func asAssignment(key string, raw []byte, err error) ([]byte, error) { + if err != nil || raw == nil { + return raw, err + } + p, err := provisions.DecodePlan(raw) + if err != nil { + return nil, fmt.Errorf("the plan %s: %w", key, err) + } + return json.Marshal(map[string]any{"node": p.Node, "module": p.Module, "provisioning": p.Plan}) +} // AssignmentReader reads one value of the assignments mirror, `.`: nil when there is none. type AssignmentReader func(key string) ([]byte, error) diff --git a/modules/mesh-vault/cmd/mesh-vault/intake.go b/modules/mesh-vault/cmd/mesh-vault/intake.go index b1eb1811..b887916b 100644 --- a/modules/mesh-vault/cmd/mesh-vault/intake.go +++ b/modules/mesh-vault/cmd/mesh-vault/intake.go @@ -120,6 +120,15 @@ func fromTheController(caller, verb string) error { return nil } +// fromAGrantedCaller refuses a caller the runtime did not prove: `rotate` and `issued` answer every caller whose +// manifest invokes them, which the bus grants it (novox/hq ADR 0570, Consequences), no longer the controller alone. +func fromAGrantedCaller(caller, verb string) error { + if strings.TrimSpace(caller) == "" || caller == stdio.Unproven { + return fmt.Errorf("%s answers a caller the runtime proves, and it proved none", verb) + } + return nil +} + // openCall makes a key for one call and keeps its private half here until the call is taken or two minutes // have passed. func (v *Vault) openCall(verb, secret, by string, ask *AskRecord) (string, string, error) { @@ -177,6 +186,16 @@ func (v *Vault) takeCall(verb, id string) (*callKey, error) { return c, nil } +// openedBy is the caller that opened a secret-ask's call still open, as the ask's record names it; "" for none. +func (v *Vault) openedBy(id string) string { + v.callsMu.Lock() + defer v.callsMu.Unlock() + if c, ok := v.pending[id]; ok && c.verb == "secret-ask" && c.ask != nil { + return c.ask.Caller + } + return "" +} + // open opens what was sealed to the call's key, and forgets the key. func (c *callKey) open(sealed string) ([]byte, error) { defer func() { c.private = [32]byte{} }() @@ -305,15 +324,33 @@ func (v *Vault) AcceptTake(caller, call, sealed string) (Changed, error) { type AskOf struct { Node, Module, Name, Desk string // AskedBy is who asked the controller, as the bus named its caller; the controller's word, taken from the - // controller alone. + // controller alone. A caller asking directly is named by the runtime's proof, never by this (ADR 0563 §8). AskedBy string } +// askedByFor is who asked a secret-ask as the bus says it (novox/hq ADR 0563 §8): for a call relayed by the controller, +// the caller the bus named to the controller, which it takes from the controller alone, as before; for a caller the +// manifest names, that caller as the runtime proved it. Any other caller is refused: no caller names itself. +func (v *Vault) askedByFor(caller string, a AskOf) (string, error) { + switch { + case caller == ControllerCaller: + return askedByName(a.AskedBy), nil + case v.AskCallers.Allows(caller): + return askedByName(caller), nil + } + if caller == "" { + caller = stdio.Unproven + } + return "", fmt.Errorf("secret-ask answers the controller and the callers the vault's manifest names, and %s is "+ + "neither; nobody was asked to type anything", caller) +} + // AskOpen records an ask for a module's own secret before any prompt opens, holding its bounds — one open ask // per secret, three an hour (ADR 0277 §24, §25) — and answers the key the desk's prompt seals what is typed to, // and who asked as the record says it, which the prompt names (§26). func (v *Vault) AskOpen(caller string, a AskOf) (Opened, error) { - if err := fromTheController(caller, "secret-ask"); err != nil { + askedBy, err := v.askedByFor(caller, a) + if err != nil { return Opened{}, err } secret, err := OwnSecret(a.Node, a.Module, a.Name) @@ -323,7 +360,6 @@ func (v *Vault) AskOpen(caller string, a AskOf) (Opened, error) { if !namePart.MatchString(a.Desk) { return Opened{}, fmt.Errorf("nobody was asked to type anything: the desk %q is not a machine's name", a.Desk) } - askedBy := askedByName(a.AskedBy) v.Store.asksMu.Lock() defer v.Store.asksMu.Unlock() now := v.Now().UTC() @@ -378,9 +414,13 @@ var askOutcomes = map[string]bool{"dismissed": true, "timed-out": true, "empty": // AskEnd ends an ask: with what the desk sealed to its key, which the vault opens and keeps as typed, or with // the outcome the desk answered instead. How it ended is recorded whatever happens. func (v *Vault) AskEnd(caller, call, sealed, outcome string) (Changed, error) { - if err := fromTheController(caller, "secret-ask"); err != nil { + if _, err := v.askedByFor(caller, AskOf{}); err != nil { return Changed{}, err } + // An ask is ended by the caller that opened it, and by no other. + if opener := v.openedBy(call); opener != "" && opener != caller { + return Changed{}, fmt.Errorf("the ask %s was opened by %s, and %s may not end it", call, opener, caller) + } if sealed == "" && !askOutcomes[outcome] { return Changed{}, fmt.Errorf("an ask ends with what was sealed to its key, or with one of dismissed, " + "timed-out, empty, refused or failed") @@ -464,7 +504,7 @@ func askedByName(caller string) string { // 10, ADR 0305 with the vault in the controller's place): the controller asks the provider's tool, on the // provider's machine, with the call's key, and passes on what it answered. func (v *Vault) IssuedOpen(caller, provision, provider, module string) (Opened, error) { - if err := fromTheController(caller, "issued"); err != nil { + if err := fromAGrantedCaller(caller, "issued"); err != nil { return Opened{}, err } return v.openIssued(provision, provider, module) @@ -489,7 +529,7 @@ func (v *Vault) openIssued(provision, provider, module string) (Opened, error) { // IssuedTake opens what the provider's tool sealed to the call's key and keeps it as the provision's one // credential. The same value again writes nothing (ADR 0305 §5): every holder that has it is left as it is. func (v *Vault) IssuedTake(caller, call, sealed string) (Changed, error) { - if err := fromTheController(caller, "issued"); err != nil { + if err := fromAGrantedCaller(caller, "issued"); err != nil { return Changed{}, err } c, err := v.takeCall("issued", call) @@ -527,8 +567,8 @@ type Asked struct { Unchanged bool `json:"unchanged,omitempty"` } -// AskAtDesk is `secret-ask` (ADR 0349 clause 9, ADR 0277 §22–§27): asked by the controller alone, which names who -// asked it. The ask is recorded and its bounds held before any prompt opens; the vault asks the desk's prompt +// AskAtDesk is `secret-ask` (ADR 0349 clause 9, ADR 0277 §22–§27): asked by the controller, which names who asked it, +// or by a caller the vault's manifest names, named by the runtime's proof (ADR 0563 §8). The ask is recorded and its bounds held before any prompt opens; the vault asks the desk's prompt // itself, in its own name, naming the asker as recorded and giving a key made for this one call; what is typed // comes back sealed to that key and opens here. How the prompt ended is the answer and the record's. func (v *Vault) AskAtDesk(caller string, a AskOf) (Asked, error) { @@ -612,7 +652,7 @@ func (v *Vault) TakeIssued(caller, provision, provider, module, tool string) (Is // credential, as before) or one per holder (`holders`: node, module and sealed each, kept as that holder's own and // handed to it alone, perholder.go). Never both. func (v *Vault) TakeIssuedFor(caller, provision, provider, module, tool string, holders []Holder) (Issued, error) { - if err := fromTheController(caller, "issued"); err != nil { + if err := fromAGrantedCaller(caller, "issued"); err != nil { return Issued{}, err } return v.takeIssuedFor(caller, provision, provider, module, tool, holders) @@ -716,9 +756,11 @@ func (n Narrow) String() string { // value shared by every holder (ADR 0158) is one value, so it is never rotated for one consumer alone: refused, and // nothing changes. A module's own secret names no consumer. func (v *Vault) RotateFor(caller, secret string, one Narrow, by, why string) (Changed, error) { + if err := fromAGrantedCaller(caller, "rotate"); err != nil { + return Changed{}, err + } if caller != ControllerCaller { - return Changed{}, fmt.Errorf("the vault rotates for the controller alone, which knows whether the module's "+ - "secrets are filled from the vault yet; %s is not the controller: `secret rotate` at the controller", caller) + by = "" // only the controller relays who asked at its terminal; any other caller is who asked } if strings.TrimSpace(why) == "" { return Changed{}, errors.New("not rotated: a rotation says why it was asked (`why`)") diff --git a/modules/mesh-vault/cmd/mesh-vault/intake_test.go b/modules/mesh-vault/cmd/mesh-vault/intake_test.go index e795bad7..6d844cf0 100644 --- a/modules/mesh-vault/cmd/mesh-vault/intake_test.go +++ b/modules/mesh-vault/cmd/mesh-vault/intake_test.go @@ -13,6 +13,7 @@ import ( "encoding/base64" "encoding/json" "errors" + stdio "git.novox.be/novox/mesh-sdk/go" "os" "path/filepath" "strings" @@ -173,6 +174,29 @@ func TestAnAskIsBoundedToOneOpenPerSecretAndThreeAnHour(t *testing.T) { if _, err := w.AskOpen(ControllerCaller, telegramAsk("g14.claude-code")); err != nil { t.Errorf("an ask left open past its time still held the secret: %v", err) } + // 774 (ADR 0563 §8): a caller the manifest names asks directly, is named by its proven caller and never by the + // asked_by it gives, and is held to the same bounds; one it does not name is refused. + t.Run("a named caller asks itself, within the bounds", func(t *testing.T) { + x := newVault(t, nil) + x.AskCallers = stdio.Callers{"node-tools"} + if _, err := x.AskOpen("ace.claude-code", telegramAsk("x")); err == nil || !strings.Contains(err.Error(), "ace.claude-code") { + t.Errorf("a caller the manifest does not name opened an ask: %v", err) + } + opened, err := x.AskOpen("g14.node-tools", telegramAsk("person.operator")) + if err != nil || opened.AskedBy != "g14.node-tools" { + t.Fatalf("a named caller's ask: %+v, %v", opened, err) + } + if _, err := x.AskEnd(ControllerCaller, opened.Call, "", "dismissed"); err == nil { + t.Error("another caller ended a named caller's ask") + } + if _, err := x.AskOpen("ace.node-tools", telegramAsk("")); err == nil || !strings.Contains(err.Error(), "still open (asked by g14.node-tools)") { + t.Errorf("a second ask while a named caller's is open: %v", err) + } + recorded, _ := x.Store.Asks(10) + if len(recorded) == 0 || recorded[0].AskedBy != "g14.node-tools" || recorded[0].Caller != "g14.node-tools" { + t.Errorf("recorded as %+v", recorded) + } + }) } // Only the controller reaches a desk's prompt, so only it opens and ends an ask; a value not sealed to the @@ -283,9 +307,12 @@ func TestAnIssuingProviderAnswersOnlyTheVaultsCallSealedToTheVault(t *testing.T) t.Errorf("the provider's tool answered %q", caller) } } - // The vault takes the steps from the controller alone. - if _, err := v.IssuedOpen("anchor.node-tools", "plex-api", "anchor", "plex"); err == nil { - t.Error("the vault opened an issued call for another caller") + // The vault takes the steps from a caller the runtime proved, whose manifest the bus granted the verb (ADR 0570), + // and from no unproven one. + for _, caller := range []string{"unproven", ""} { + if _, err := v.IssuedOpen(caller, "plex-api", "anchor", "plex"); err == nil { + t.Errorf("the vault opened an issued call for %q", caller) + } } changed, err := issue(t, v, provider) @@ -413,17 +440,17 @@ func TestRotateKeepsWhoAskedAndWhoCarriedIt(t *testing.T) { } } -// Rotate is the controller's alone, and says why (the review of issue 675): only the controller knows whether a -// module is filled from the vault yet, so a rotation asked by any other caller — a proven one included — would -// drift from the value the controller holds; and a rotation without a reason is refused. Neither changes anything. -func TestRotateIsTheControllersAloneAndSaysWhy(t *testing.T) { +// Rotate answers a caller the runtime proved, whose manifest the bus granted it (novox/hq ADR 0570 Consequences), and +// says why: an unproven caller, and a rotation without a reason, are refused and change nothing; a proven caller +// other than the controller is who asked, whatever `by` it says. +func TestRotateIsAProvenCallersAndSaysWhy(t *testing.T) { v := newVault(t, nil) if _, err := v.Mint("own/g14/dunst/token", "the test"); err != nil { t.Fatal(err) } - for _, caller := range []string{"g14.claude-code", "g14.node-provisioner", "unproven", ""} { + for _, caller := range []string{"unproven", ""} { if _, err := v.Rotate(caller, "own/g14/dunst/token", "jochen", "leaked"); err == nil || - !strings.Contains(err.Error(), "controller alone") { + !strings.Contains(err.Error(), "proves") { t.Errorf("%q rotated: %v", caller, err) } } @@ -436,6 +463,12 @@ func TestRotateIsTheControllersAloneAndSaysWhy(t *testing.T) { if rec, _, _ := v.Store.Read("own/g14/dunst/token"); rec.Generation != 1 { t.Fatalf("a refused rotation changed the secret: generation %d", rec.Generation) } + if _, err := v.Rotate("g14.claude-code", "own/g14/dunst/token", "jochen", "leaked in a log"); err != nil { + t.Fatal(err) + } + if rec, _, _ := v.Store.Read("own/g14/dunst/token"); rec.Generation != 2 || rec.ChangedBy != "g14.claude-code" { + t.Errorf("a session's rotation is recorded by %q at generation %d", rec.ChangedBy, rec.Generation) + } } // The names the vault keeps them under are names, never paths. diff --git a/modules/mesh-vault/cmd/mesh-vault/issued_account_test.go b/modules/mesh-vault/cmd/mesh-vault/issued_account_test.go index 7625ec8b..41f600e5 100644 --- a/modules/mesh-vault/cmd/mesh-vault/issued_account_test.go +++ b/modules/mesh-vault/cmd/mesh-vault/issued_account_test.go @@ -48,6 +48,13 @@ func (f *fakeAccounts) Put(a AccountHash) error { return nil } +func (f *fakeAccounts) Delete(user string) error { + f.mu.Lock() + defer f.mu.Unlock() + delete(f.keys, user) + return nil +} + func (f *fakeAccounts) Users() ([]string, error) { f.mu.Lock() defer f.mu.Unlock() diff --git a/modules/mesh-vault/cmd/mesh-vault/lease.go b/modules/mesh-vault/cmd/mesh-vault/lease.go new file mode 100644 index 00000000..3a399d00 --- /dev/null +++ b/modules/mesh-vault/cmd/mesh-vault/lease.go @@ -0,0 +1,49 @@ +package main + +// The switch on the controller's lease (novox/hq ADR 0561 §3): minting is on only while the lease is handed to a holder +// that is no controller, and off for a controller's value, a key deleted or expired, or one that cannot be read. The +// write that hands it away turns it on and mints every wanted account the vault holds no value for; a lease handed back +// turns it off. No clock and no setting turns it. A key's age passing is heard by no watch, so the lease is read afresh +// before every mint (LeaseRead): a lease that expired since mints nothing. + +import "git.novox.be/novox/mesh-sdk/go/buscompose" + +// LeaseHeard is the watch of the controller's lease. When minting turns on or off, every wanted account is judged again: +// minted when on, held back with the new why when off. +func (v *Vault) LeaseHeard(raw []byte, found bool, readErr error) { + if !v.setLease(raw, found, readErr) { + return + } + on, why := v.mintingOn() + v.log("minting is %s: %s", map[bool]string{true: "ON", false: "off"}[on], why) + for _, user := range v.wantedUsers() { + if w, ok := v.wanted(user); ok { + if _, err := v.mintWanted(w, "the lease write"); err != nil { + v.log("%s is not minted: %v", user, err) + } + } + } +} + +// setLease records what the lease says, and answers whether minting turned on or off by it. +func (v *Vault) setLease(raw []byte, found bool, readErr error) bool { + on, why := buscompose.LeaseHandedAway(raw, found, readErr) + m := &v.minting + m.mu.Lock() + defer m.mu.Unlock() + flipped := on != m.on || m.why == "" + if flipped { + m.since = v.Now().UTC() + } + m.on, m.why = on, why + return flipped +} + +// leaseAfresh reads the lease again before a mint, where the vault can: a key that expired since is heard by no watch. +func (v *Vault) leaseAfresh() { + if v.LeaseRead == nil { + return + } + raw, found, err := v.LeaseRead() + v.setLease(raw, found, err) +} diff --git a/modules/mesh-vault/cmd/mesh-vault/main.go b/modules/mesh-vault/cmd/mesh-vault/main.go index 7370f0af..089c2abc 100644 --- a/modules/mesh-vault/cmd/mesh-vault/main.go +++ b/modules/mesh-vault/cmd/mesh-vault/main.go @@ -8,6 +8,10 @@ package main import ( + "git.novox.be/novox/mesh-sdk/go/provisions" + + "encoding/json" + "errors" "fmt" "os" @@ -50,11 +54,35 @@ func build() *Env { // Who may ask (askers.go): a node's provisioner client, for what its module's plan there names. // Where it sees a consumer bind a provider (pair.go): the same mirror, every key. // The assignments as the vault's watch keeps them (watchAssignments), read over the bus until it has caught up. - assigned = &WatchedAssignments{Fallback: readAssignment} - env.Vault = &Vault{Store: store, Own: own, Askers: Declared{Read: assigned.Read, Confirm: readAssignment, Store: store}, + assigned = &WatchedAssignments{Fallback: readPlanRaw} + watchedPlan := func(key string) ([]byte, error) { raw, err := assigned.Read(key); return asAssignment(key, raw, err) } + env.Vault = &Vault{Store: store, Own: own, Askers: Declared{Read: watchedPlan, Confirm: readAssignment, Store: store}, Assignments: Mirror{Keys: assignmentKeys, Read: readAssignment}, Emit: emit, Ask: stdio.Ask, - Now: time.Now, Log: say, Accounts: busAccounts{}, Haves: readHave, - HaveKeys: func() ([]string, error) { return stdio.State(HaveRead).Keys() }} + Now: time.Now, Log: say, Accounts: busAccounts{}, Haves: readHave, LeaseRead: readLease, WantRead: readWant, + HaveKeys: func() ([]string, error) { return stdio.State(HaveRead).Keys() }, Placement: placement, Signers: busSigners{}, + AskCallers: askCallers(os.Getenv("MESH_VAULT_SECRET_ASK_CALLERS")), + Bound: Bound{ + Get: func(key string) ([]byte, error) { + e, err := stdio.State(BoundState).Get(key) + if err != nil || e == nil { + return nil, err + } + return e.Value, nil + }, + Put: func(key string, value []byte) error { + _, err := stdio.State(BoundState).Put(key, json.RawMessage(value)) + return err + }, + Delete: func(key string) error { return stdio.State(BoundState).Delete(key) }, + Keys: func() ([]string, error) { return stdio.State(BoundState).Keys() }, + Provision: func(key string) ([]byte, error) { + e, err := stdio.State(provisions.ProvisionsRead).Get(key) + if err != nil || e == nil { + return nil, err + } + return e.Value, nil + }, + }} return env } @@ -120,6 +148,74 @@ func watchHandings(env *Env) bool { return true } +// watchWants mints from the grants part's wanted accounts as they change (mint.go, ADR 0561 §2): a watch, never a poll. +// It answers whether the watch began; one that did not is begun again at the next minute pass. +func watchWants(env *Env) bool { + if env.Vault == nil { + return true + } + err := stdio.State(WantedRead).Watch("", func(c stdio.StateChange) error { + env.Vault.WantHeard(c.Key, c.Value, c.Op == "delete" || c.Op == "purge" || len(c.Value) == 0) + return nil + }) + if err != nil { + say("the grants part's wanted accounts are not watched, so nothing is minted; begun again at the next pass: %v", err) + return false + } + // The watch returns once every current key was handed: what has no key now is no longer wanted. + env.Vault.WantsRead() + return true +} + +// placement reads the seat mesh-placement's placement through the runtime under the vault's own account (its manifest reads it). +var placement = PlacementReader{ + Keys: func() ([]string, error) { return stdio.State(PlacementRead).Keys() }, + Read: func(key string) ([]byte, error) { + e, err := stdio.State(PlacementRead).Get(key) + if err != nil || e == nil { + return nil, err + } + return e.Value, nil + }, +} + +// readWant is one wanted account of the grants part, read afresh (issue-account). +func readWant(user string) ([]byte, bool, error) { + e, err := stdio.State(WantedRead).Get(user) + if err != nil || e == nil { + return nil, false, err + } + return e.Value, true, nil +} + +// readLease is the controller's lease key, read through the runtime under the vault's own account (its manifest reads it). +func readLease() ([]byte, bool, error) { + e, err := stdio.State(LeaseRead).Get(LeaseKey) + if err != nil || e == nil { + return nil, false, err + } + return e.Value, true, nil +} + +// watchLease turns minting on and off as the controller's lease is written (lease.go, ADR 0561 §3): a watch, never a +// poll; and the lease is read afresh before every mint. It answers whether the watch began. +func watchLease(env *Env) bool { + if env.Vault == nil { + return true + } + env.Vault.LeaseHeard(readLease()) + err := stdio.State(LeaseRead).Watch(LeaseKey, func(c stdio.StateChange) error { + deleted := c.Op == "delete" || c.Op == "purge" || len(c.Value) == 0 + env.Vault.LeaseHeard(c.Value, !deleted, nil) + return nil + }) + if err != nil { + say("the controller's lease is not watched, so minting stays as last read; begun again at the next pass: %v", err) + return false + } + return true +} + // adoptOperatorKey takes the operator key from the export the mesh writes to the vault's root (what // `operator key set` recorded; the controller keeps only a public half there), and seals again any // operator copy not yet sealed to the current key. Run every minute, so a failure partway is finished. It @@ -153,6 +249,12 @@ func adoptOperatorKey(env *Env) error { // readAssignment is one value of the controller's assignments mirror, read through the runtime under the vault's // own account (its manifest `reads` it), or nil when there is none. func readAssignment(key string) ([]byte, error) { + raw, err := readPlanRaw(key) + return asAssignment(key, raw, err) +} + +// readPlanRaw is one key of the seat mesh-provisioner's plans as it is kept, nil when there is none. +func readPlanRaw(key string) ([]byte, error) { e, err := stdio.State(AssignmentsRead).Get(key) if err != nil || e == nil { return nil, err @@ -173,7 +275,11 @@ func main() { written, failed := env.Vault.HashHeld() say("hashed %d bus accounts at start; %d could not be, tried again every minute: %v", len(written), len(failed), failed) } + // The operator key first: nothing is kept without it, so a want heard before it would not be minted. + env.pass() watched, watching := watchHandings(env), watchAssignments() + lease := watchLease(env) + wants := watchWants(env) for { if !watched { watched = watchHandings(env) @@ -181,9 +287,23 @@ func main() { if !watching { watching = watchAssignments() } + if !lease { + lease = watchLease(env) + } + if !wants { + wants = watchWants(env) + } env.pass() if env.Vault != nil { + // The signing key and authority once the operator key is taken, and `signers` when it differs (ADR 0550 §2). + if err := env.Vault.SigningPass(); err != nil { + env.Vault.signingSaid(err) + } else { + env.Vault.signingSaid(nil) + } env.Vault.HashOwed() + env.Vault.MintOwed() + env.Vault.RetirePass() } env.pairPass() if env.Custody != nil { @@ -197,3 +317,15 @@ func main() { os.Exit(1) } } + +// askCallers are the callers the vault's manifest names for secret-ask besides the controller (novox/hq ADR 0563 §8): +// MESH_VAULT_SECRET_ASK_CALLERS, comma-separated, each a module (on any machine), . or person.. +func askCallers(list string) stdio.Callers { + var out stdio.Callers + for _, c := range strings.Split(list, ",") { + if c = strings.TrimSpace(c); c != "" { + out = append(out, c) + } + } + return out +} diff --git a/modules/mesh-vault/cmd/mesh-vault/mint.go b/modules/mesh-vault/cmd/mesh-vault/mint.go new file mode 100644 index 00000000..06bb80a7 --- /dev/null +++ b/modules/mesh-vault/cmd/mesh-vault/mint.go @@ -0,0 +1,376 @@ +package main + +// The vault mints every new bus account (novox/hq ADR 0561 §1 to §4, issue 774). +// +// The grants part says which accounts the mesh wants, in its state `accounts` that the vault alone reads (§2): one key +// per bus user, holding what the controller's credential carries beside the password. The vault watches it, and for a +// user it holds no value for and whose hash it did not adopt it makes a password, keeps the account whole (built by +// mesh-sdk's HeldCredential, byte for byte the controller's shape) as the module's own secret on that node +// (`own///|provisioner-broker>`), sealed, with the operator's copy; writes the +// bcrypt hash into `bus-accounts`; and says `account-issued`, then `secret-changed`. The node's engine asks `own` again +// on `secret-changed` and composes. +// +// **One issuer at every moment** (§3): minting is on only while the controller's lease is handed to a holder that is no +// controller (mesh-sdk LeaseHandedAway). Until then the controller mints, the vault keeps hashing what it is handed, and +// vault_status names every wanted account it holds back and why. **Nothing is minted anew** (§4): an account whose value +// the vault holds, or whose hash it adopted, is never minted; the node accounts and the controller's own are never wanted. +// **Never two values for one user** (§7): a user whose hash the vault already wrote from another of its secrets is held +// back, said, and never replaced but by a rotation. + +import ( + "crypto/rand" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "sort" + "strings" + "sync" + "time" + + "git.novox.be/novox/mesh-sdk/go/buscompose" +) + +// WantedRead is the grants part's state of wanted accounts, as the vault's manifest reads it. +const WantedRead = buscompose.GrantsAccounts + +// LeaseRead is the controller's lease, as the vault's manifest reads it, and LeaseKey its one key. +const ( + LeaseRead = buscompose.ControllerLease + LeaseKey = buscompose.ControllerLeaseKey +) + +// AccountIssuedEvent is what the vault says when it minted an account (mesh-sdk buscompose.AccountIssued). +const AccountIssuedEvent = buscompose.AccountIssuedEvent + +// minting is the switch and what the vault knows of the wanted accounts, kept in memory and read again from the bus at +// every start: the grants part's state and the lease are the record, never this. +type minting struct { + mu sync.Mutex + on bool + why string + since time.Time + // wants are the wanted accounts, by user, as last heard. + wants map[string]buscompose.WantedAccount + // heldBack are the wanted users the vault did not mint, with why (vault_status). + heldBack map[string]string + // owed are the wanted users whose mint failed, tried again at the pass (MintOwed). + owed map[string]bool + // minted are the accounts this vault minted since it started, by user (vault_status). + minted map[string]Minted + // loadedRevision is the highest revision of the user list the bus node said it loaded (rotation.go). + loadedRevision uint64 + // retiring is what the passes keep of the accounts no longer wanted (retire.go). + retiring retiring + // mintMu keeps one mint at a time: a want heard, the switch turning on, and issue-account. + mintMu sync.Mutex +} + +// Minted is what one mint did: the account and its secret, never its value or its hash. +type Minted struct { + User string `json:"user"` + Secret string `json:"secret"` + Generation int `json:"generation"` + At time.Time `json:"at"` + // Hashed says its hash is written; false leaves it owed, written at the next pass. + Hashed bool `json:"hashed"` +} + +// MintState is what vault_status says of minting. +type MintState struct { + On bool `json:"on"` + Why string `json:"why"` + Since time.Time `json:"since,omitzero"` + Wanted int `json:"wanted"` + HeldBack map[string]string `json:"held_back"` + Minted []Minted `json:"minted"` +} + +// MintState answers how minting stands. +func (v *Vault) MintState() MintState { + m := &v.minting + m.mu.Lock() + defer m.mu.Unlock() + out := MintState{On: m.on, Why: m.why, Since: m.since, Wanted: len(m.wants), HeldBack: map[string]string{}, Minted: []Minted{}} + if out.Why == "" { + out.Why = "the controller's lease has not been read yet, so nothing is minted" + } + for u, why := range m.heldBack { + out.HeldBack[u] = why + } + for _, x := range m.minted { + out.Minted = append(out.Minted, x) + } + sort.Slice(out.Minted, func(i, j int) bool { return out.Minted[i].User < out.Minted[j].User }) + return out +} + +// mintingOn says whether the vault mints now, and why. +func (v *Vault) mintingOn() (bool, string) { + v.minting.mu.Lock() + defer v.minting.mu.Unlock() + if v.minting.why == "" { + return false, "the controller's lease has not been read yet, so nothing is minted" + } + return v.minting.on, v.minting.why +} + +func (v *Vault) holdBack(user, why string) { + m := &v.minting + m.mu.Lock() + defer m.mu.Unlock() + if m.heldBack == nil { + m.heldBack = map[string]string{} + } + m.heldBack[user] = why +} + +func (v *Vault) noLongerHeld(user string) { + m := &v.minting + m.mu.Lock() + defer m.mu.Unlock() + delete(m.heldBack, user) +} + +// AccountSecret is the vault's name of the own secret a wanted account is held in. +func AccountSecret(w buscompose.WantedAccount) (string, error) { + return OwnSecret(w.Node, w.Module, w.OwnSecret()) +} + +// WantHeard is the watch of the grants part's wanted accounts: a key put is a want, minted when minting is on; a key +// deleted is a want gone (retire.go). +func (v *Vault) WantHeard(user string, raw []byte, deleted bool) { + if deleted { + v.wantGone(user) + return + } + w, err := buscompose.ReadWantedAccount(user, raw) + if err != nil { + v.log("the wanted account %s is not read, so nothing is minted for it: %v", user, err) + v.holdBack(user, "its wanted key is not readable: "+err.Error()) + return + } + v.minting.mu.Lock() + if v.minting.wants == nil { + v.minting.wants = map[string]buscompose.WantedAccount{} + } + v.minting.wants[user] = w + v.minting.mu.Unlock() + v.wantBack(user) + if _, err := v.mintWanted(w, "the grants part's want of "+user); err != nil { + v.log("%s is not minted: %v", user, err) + } +} + +// errHeldBack is a want the vault does not mint now, with why: not an error of the vault. +type errHeldBack struct{ why string } + +func (e errHeldBack) Error() string { return e.why } + +// mintWanted mints w when minting is on, the vault holds no value for it and adopted no hash of it; otherwise it says +// why not and, where that is a hold, keeps it for vault_status. +func (v *Vault) mintWanted(w buscompose.WantedAccount, by string) (Minted, error) { + m, err := v.mintOnce(w, by) + var held errHeldBack + v.owe(w.User, err != nil && !errors.As(err, &held) && w.Valid() == nil) + return m, err +} + +// mintOnce is one try of mintWanted. +func (v *Vault) mintOnce(w buscompose.WantedAccount, by string) (Minted, error) { + v.minting.mintMu.Lock() + defer v.minting.mintMu.Unlock() + if err := w.Valid(); err != nil { + v.holdBack(w.User, err.Error()) + return Minted{}, err + } + secret, err := AccountSecret(w) + if err != nil { + v.holdBack(w.User, err.Error()) + return Minted{}, err + } + if _, held, err := v.Store.Read(secret); err != nil { + return Minted{}, err + } else if held { + // Its value is held (the controller's, handed in, or one the vault minted): never minted anew (§4). + v.noLongerHeld(w.User) + return Minted{}, errHeldBack{secret + " is held already: an account the vault holds a value for is never minted anew (ADR 0561 §4)"} + } + if v.Accounts == nil { + v.holdBack(w.User, "the vault has no state of bus accounts to write the hash to") + return Minted{}, errors.New("the vault has no state of bus accounts to write the hash to") + } + was, err := v.Accounts.Get(w.User) + if err != nil { + v.holdBack(w.User, "its key in "+BusAccountsState+" cannot be read: "+err.Error()) + return Minted{}, err + } + if was != nil && was.Adopted { + // The hash the bus loaded is kept (§4): not minted, and not held back either. + v.noLongerHeld(w.User) + return Minted{}, errHeldBack{w.User + "'s hash was adopted from the list the bus loaded: it keeps it, and is never minted (ADR 0561 §4)"} + } + if was != nil && was.Of != "" && was.Of != secret { + why := fmt.Sprintf("%s holds the hash of %s for %s, not of %s: a user has one value, so nothing is minted until "+ + "a person sees why (ADR 0561 §7)", BusAccountsState, was.Of, w.User, secret) + v.holdBack(w.User, why) + return Minted{}, errHeldBack{why} + } + v.leaseAfresh() + if on, why := v.mintingOn(); !on { + v.holdBack(w.User, "held back: "+why) + return Minted{}, errHeldBack{why} + } + raw := make([]byte, 30) + if _, err := rand.Read(raw); err != nil { + return Minted{}, err + } + value, err := buscompose.HeldCredential(w, base64.RawURLEncoding.EncodeToString(raw)) + if err != nil { + v.holdBack(w.User, err.Error()) + return Minted{}, err + } + c, err := v.keepLocked(secret, string(value), "the vault", "", OriginMade, keepOptions{neverReplace: true, quiet: true}) + if err != nil { + v.holdBack(w.User, "its value could not be kept: "+err.Error()) + return Minted{}, err + } + out := Minted{User: w.User, Secret: secret, Generation: c.Generation, At: c.At} + // The hash first, then account-issued, then secret-changed: the grants part composes the user before the node asks. + if _, err := v.HashAccount(secret, false); err != nil { + v.hashes.owe(secret, err.Error()) + v.log("%s is minted and kept, and ITS HASH IS NOT WRITTEN, written at the next pass: %v", w.User, err) + } else { + out.Hashed = true + if v.Emit != nil { + e := buscompose.AccountIssued{User: w.User, Kind: w.Kind, Node: w.Node, Module: w.Module, Process: w.Process, + Generation: c.Generation, At: c.At} + body := map[string]any{} + b, _ := json.Marshal(e) + _ = json.Unmarshal(b, &body) + if err := v.Emit(AccountIssuedEvent, body); err != nil { + v.log("%s is minted and hashed, but %s could not be said: %v", w.User, AccountIssuedEvent, err) + } + } + } + v.announce(c) + v.minting.mu.Lock() + if v.minting.minted == nil { + v.minting.minted = map[string]Minted{} + } + v.minting.minted[w.User] = out + v.minting.mu.Unlock() + v.noLongerHeld(w.User) + v.log("minted the bus account %s as %s (generation %d), for %s", w.User, secret, c.Generation, by) + return out, nil +} + +// MintOwed tries again, at the vault's minute pass, every wanted account whose mint failed (its value could not be kept, +// its key read, its password made): a want heard is never lost to one refusal. A want held back is not owed: the switch +// or the want changing judges it again. +func (v *Vault) MintOwed() { + v.minting.mu.Lock() + var owed []string + for u := range v.minting.owed { + owed = append(owed, u) + } + v.minting.mu.Unlock() + sort.Strings(owed) + for _, u := range owed { + if w, ok := v.wanted(u); ok { + if _, err := v.mintWanted(w, "the pass, after a mint that failed"); err != nil { + v.log("%s is still not minted: %v", u, err) + } + } else { + v.owe(u, false) + } + } +} + +// owe marks user's mint as failed (tried again at the pass) or not. +func (v *Vault) owe(user string, owed bool) { + v.minting.mu.Lock() + defer v.minting.mu.Unlock() + if !owed { + delete(v.minting.owed, user) + return + } + if v.minting.owed == nil { + v.minting.owed = map[string]bool{} + } + v.minting.owed[user] = true +} + +// heldBackOrMinted is a mint's answer as a tool gives it. +func heldBackOrMinted(m Minted, err error) (any, error) { + var held errHeldBack + if errors.As(err, &held) { + return map[string]any{"minted": false, "why": held.why}, nil + } + if err != nil { + return nil, err + } + return map[string]any{"minted": true, "account": m}, nil +} + +// IssueAccount is the seat's verb issue-account (ADR 0561 §9): one wanted user minted now, its key read afresh from the +// grants part's state. The recovery when a want was not heard: it mints nothing a want would not, and is refused while +// minting is off, for a node's account and the controller's (never minted here, §4), and for a user nobody wants. +func (v *Vault) IssueAccount(caller, user string) (any, error) { + user = strings.TrimSpace(user) + if user == "" { + return nil, errors.New("issue-account names a bus user (`user`), e.g. ace.mesh-build-go") + } + if nodeOrController(user) { + return nil, fmt.Errorf("%s is never minted by the vault: a node's account and the controller's keep the hash they "+ + "have (ADR 0561 §4)", user) + } + v.leaseAfresh() + if on, why := v.mintingOn(); !on { + return nil, fmt.Errorf("not minted: %s", why) + } + raw, found, err := []byte(nil), false, error(nil) + if v.WantRead != nil { + raw, found, err = v.WantRead(user) + } else if w, ok := v.wanted(user); ok { + raw, err = json.Marshal(w) + found = err == nil + } + if err != nil { + return nil, fmt.Errorf("the wanted account %s could not be read: %w", user, err) + } + if !found { + return nil, fmt.Errorf("the grants part holds no wanted account %s: the vault mints only what placement composes "+ + "(grants_accounts lists them)", user) + } + w, err := buscompose.ReadWantedAccount(user, raw) + if err != nil { + return nil, err + } + v.log("issue-account %s, asked by %s", user, someoneOr("", caller)) + return heldBackOrMinted(v.mintWanted(w, "issue-account by "+someoneOr("", caller))) +} + +// wantedUsers is every wanted user heard, sorted. +func (v *Vault) wantedUsers() []string { + v.minting.mu.Lock() + defer v.minting.mu.Unlock() + out := make([]string, 0, len(v.minting.wants)) + for u := range v.minting.wants { + out = append(out, u) + } + sort.Strings(out) + return out +} + +func (v *Vault) wanted(user string) (buscompose.WantedAccount, bool) { + v.minting.mu.Lock() + defer v.minting.mu.Unlock() + w, ok := v.minting.wants[user] + return w, ok +} + +// nodeOrController says user is a node's account or the controller's: never minted here (ADR 0561 §4, §9). +func nodeOrController(user string) bool { + return user == ControllerCaller || strings.HasPrefix(user, "node.") +} diff --git a/modules/mesh-vault/cmd/mesh-vault/mint_integration_test.go b/modules/mesh-vault/cmd/mesh-vault/mint_integration_test.go new file mode 100644 index 00000000..133c3131 --- /dev/null +++ b/modules/mesh-vault/cmd/mesh-vault/mint_integration_test.go @@ -0,0 +1,422 @@ +//go:build integration + +package main + +// The vault whole against a real bus (novox/hq ADR 0561 §1 to §4, issue 774's acceptance 12): the built binary, played +// over its real stdin and stdout as the node's tool runtime plays it (tools/call with the caller in _meta; mesh/state.* +// and mesh/publish answered from a real nats-server's key-value buckets and subjects), its store in a directory of its +// own. A want written to the grants part's state is minted only once the controller's lease is handed away: the hash is +// in bus-accounts, own answers the value sealed to the node, the value is the controller's shape and its hash's +// password, account-issued is said; an adopted user is never minted, and a lease handed back mints nothing. + +import ( + "bufio" + "context" + "crypto/rand" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" + "golang.org/x/crypto/bcrypt" + "golang.org/x/crypto/nacl/box" + + "git.novox.be/novox/mesh-sdk/go/buscompose" + "git.novox.be/novox/mesh-sdk/go/seal" + "git.novox.be/novox/mesh-sdk/go/testbus" +) + +// wire is one line between the runtime and the bundle (the SDK's stdio message). +type wire struct { + ID json.RawMessage `json:"id,omitempty"` + Method string `json:"method,omitempty"` + Params json.RawMessage `json:"params,omitempty"` + Result json.RawMessage `json:"result,omitempty"` + Error *struct { + Code int `json:"code"` + Message string `json:"message"` + } `json:"error,omitempty"` +} + +// player is the node's tool runtime as far as the vault needs it, over a real bus. +type player struct { + t *testing.T + ctx context.Context + nc *nats.Conn + js jetstream.JetStream + + mu sync.Mutex + in io.Writer + next int + pending map[string]chan wire + said []string +} + +// bucketOf is the bucket a state names: the vault's own by its local name, another's as `.`. +func bucketOf(state string) string { + if owner, name, ok := strings.Cut(state, "."); ok { + return owner + "_" + name + } + return "mesh-vault_" + state +} + +func (p *player) send(m map[string]any) { + raw, _ := json.Marshal(m) + p.mu.Lock() + defer p.mu.Unlock() + _, _ = p.in.Write(append(raw, '\n')) +} + +func (p *player) answer(id json.RawMessage, result any, err error) { + if err != nil { + p.send(map[string]any{"id": id, "error": map[string]any{"code": -32000, "message": err.Error()}}) + return + } + p.send(map[string]any{"id": id, "result": result}) +} + +func (p *player) kv(state string) (jetstream.KeyValue, error) { + return p.js.KeyValue(p.ctx, bucketOf(state)) +} + +// serve answers what the bundle asks, and hands the answers to what the test asked it. +func (p *player) serve(out io.Reader) { + scanner := bufio.NewScanner(out) + scanner.Buffer(make([]byte, 1<<20), 1<<24) + for scanner.Scan() { + var m wire + if json.Unmarshal(scanner.Bytes(), &m) != nil { + continue + } + if m.Method == "" { + var key string + _ = json.Unmarshal(m.ID, &key) + p.mu.Lock() + ch := p.pending[key] + delete(p.pending, key) + p.mu.Unlock() + if ch != nil { + ch <- m + } + continue + } + go p.handle(m) + } +} + +func (p *player) handle(m wire) { + var a struct { + State string `json:"state"` + Key string `json:"key"` + Value json.RawMessage `json:"value"` + Body json.RawMessage `json:"body"` + } + _ = json.Unmarshal(m.Params, &a) + switch m.Method { + case "mesh/state.get": + kv, err := p.kv(a.State) + if err != nil { + p.answer(m.ID, nil, err) + return + } + e, err := kv.Get(p.ctx, a.Key) + if errors.Is(err, jetstream.ErrKeyNotFound) { + p.answer(m.ID, nil, nil) + return + } + if err != nil { + p.answer(m.ID, nil, err) + return + } + p.answer(m.ID, map[string]any{"key": e.Key(), "value": json.RawMessage(e.Value()), "revision": e.Revision()}, nil) + case "mesh/state.put": + kv, err := p.kv(a.State) + if err == nil { + var rev uint64 + rev, err = kv.Put(p.ctx, a.Key, a.Value) + if err == nil { + p.answer(m.ID, map[string]any{"revision": rev}, nil) + return + } + } + p.answer(m.ID, nil, err) + case "mesh/state.keys": + kv, err := p.kv(a.State) + if err != nil { + p.answer(m.ID, nil, err) + return + } + keys, err := kv.Keys(p.ctx) + if errors.Is(err, jetstream.ErrNoKeysFound) { + keys, err = []string{}, nil + } + p.answer(m.ID, keys, err) + case "mesh/state.delete": + kv, err := p.kv(a.State) + if err == nil { + err = kv.Delete(p.ctx, a.Key) + } + p.answer(m.ID, map[string]any{}, err) + case "mesh/state.watch": + kv, err := p.kv(a.State) + if err != nil { + p.answer(m.ID, nil, err) + return + } + key := strings.ReplaceAll(a.Key, "**", ">") + if key == "" { + key = ">" + } + w, err := kv.Watch(p.ctx, key) + if err != nil { + p.answer(m.ID, nil, err) + return + } + var watch string + _ = json.Unmarshal(m.ID, &watch) + current := true + for e := range w.Updates() { + if e == nil { + current = false + p.answer(m.ID, map[string]any{}, nil) + continue + } + op := "put" + if e.Operation() != jetstream.KeyValuePut { + op = "delete" + } + change := map[string]any{"state": a.State, "key": e.Key(), "op": op, "revision": e.Revision(), "current": current} + if op == "put" { + change["value"] = json.RawMessage(e.Value()) + } + p.mu.Lock() + p.next++ + id := fmt.Sprintf("w-%d", p.next) + p.mu.Unlock() + p.send(map[string]any{"id": id, "method": "mesh/state", "params": map[string]any{"watch": watch, "change": change}}) + } + case "mesh/publish": + var e struct { + Key string `json:"key"` + Body json.RawMessage `json:"body"` + } + _ = json.Unmarshal(m.Params, &e) + p.mu.Lock() + p.said = append(p.said, e.Key+" "+string(e.Body)) + p.mu.Unlock() + p.answer(m.ID, map[string]any{}, p.nc.Publish("mesh.mod.mesh-vault.event."+e.Key, e.Body)) + default: + // mesh/ask and the rest: nothing answers on this bus, as a call with no responder. + p.answer(m.ID, nil, errors.New("nats: no responders available for request")) + } +} + +// call is a tools/call of the vault as caller, answered as the tool's JSON. +func (p *player) call(name, caller string, args map[string]any) (json.RawMessage, error) { + p.mu.Lock() + p.next++ + id := fmt.Sprintf("c-%d", p.next) + ch := make(chan wire, 1) + p.pending[id] = ch + p.mu.Unlock() + p.send(map[string]any{"id": id, "method": "tools/call", "params": map[string]any{"name": name, "arguments": args, + "_meta": map[string]any{"mesh/caller": caller}}}) + select { + case m := <-ch: + var r struct { + Content []struct { + Text string `json:"text"` + } `json:"content"` + IsError bool `json:"isError"` + } + if m.Error != nil { + return nil, errors.New(m.Error.Message) + } + if err := json.Unmarshal(m.Result, &r); err != nil || len(r.Content) != 1 { + return nil, fmt.Errorf("%s answered %s", name, m.Result) + } + if r.IsError { + return nil, errors.New(r.Content[0].Text) + } + return json.RawMessage(r.Content[0].Text), nil + case <-time.After(30 * time.Second): + return nil, fmt.Errorf("%s did not answer in 30s", name) + } +} + +func (p *player) put(t *testing.T, state, key string, v any) { + t.Helper() + kv, err := p.kv(state) + if err != nil { + t.Fatal(err) + } + raw, _ := json.Marshal(v) + if _, err := kv.Put(p.ctx, key, raw); err != nil { + t.Fatal(err) + } +} + +func (p *player) hashOf(t *testing.T, user string) *AccountHash { + t.Helper() + kv, err := p.kv(BusAccountsState) + if err != nil { + t.Fatal(err) + } + e, err := kv.Get(p.ctx, user) + if errors.Is(err, jetstream.ErrKeyNotFound) { + return nil + } + if err != nil { + t.Fatal(err) + } + var a AccountHash + if err := json.Unmarshal(e.Value(), &a); err != nil { + t.Fatal(err) + } + return &a +} + +func eventually(t *testing.T, what string, ok func() bool) { + t.Helper() + for end := time.Now().Add(30 * time.Second); time.Now().Before(end); time.Sleep(50 * time.Millisecond) { + if ok() { + return + } + } + t.Fatalf("never: %s", what) +} + +// 774/AC12. +func TestTheVaultWholeMintsOnTheBusOnlyFromTheLeaseWrite(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + dir := t.TempDir() + bin := filepath.Join(dir, "mesh-vault") + build := exec.Command("go", "build", "-o", bin, ".") + build.Stderr = os.Stderr + if err := build.Run(); err != nil { + t.Fatalf("the vault does not build: %v", err) + } + nc, err := nats.Connect(testbus.URL(t)) + if err != nil { + t.Fatal(err) + } + t.Cleanup(nc.Close) + js, err := jetstream.New(nc) + if err != nil { + t.Fatal(err) + } + for _, b := range []string{"mesh-vault_bus-accounts", "mesh-grants_accounts", "mesh-controller_lease", + "mesh-provisioner_plans", "node-engine_have", "mesh-delivery-assigner_placement"} { + if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: b, History: 2}); err != nil { + t.Fatal(err) + } + } + // The operator key, as the mesh exports it to the vault's root. + opPub, _, err := box.GenerateKey(rand.Reader) + if err != nil { + t.Fatal(err) + } + root := filepath.Join(dir, "root") + _ = os.MkdirAll(root, 0o700) + export, _ := json.Marshal(map[string]string{"operator-key": base64.StdEncoding.EncodeToString(opPub[:])}) + if err := os.WriteFile(filepath.Join(root, "export.json"), export, 0o600); err != nil { + t.Fatal(err) + } + cmd := exec.CommandContext(ctx, bin) + cmd.Env = append(os.Environ(), "MESH_VAULT_STORE="+filepath.Join(dir, "store"), "MESH_VAULT_STATE="+filepath.Join(dir, "state"), + "MESH_VAULT_ROOT="+root, "MESH_VAULT_LEDGER="+filepath.Join(dir, "ledger"), "MESH_RECEIVES="+filepath.Join(dir, "mesh.json"), + "MESH_NODE=novox", "MESH_MODULE=mesh-vault") + stdin, _ := cmd.StdinPipe() + stdout, _ := cmd.StdoutPipe() + cmd.Stderr = os.Stderr + if err := cmd.Start(); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = stdin.Close(); cancel(); _ = cmd.Wait() }) + p := &player{t: t, ctx: ctx, nc: nc, js: js, in: stdin, pending: map[string]chan wire{}} + go p.serve(stdout) + + status := func() MintState { + raw, err := p.call("vault_status", "person.jochen", nil) + if err != nil { + return MintState{} + } + var s struct { + Minting MintState `json:"minting"` + } + _ = json.Unmarshal(raw, &s) + return s.Minting + } + // The controller holds the lease: a want is held back, nothing is minted. + p.put(t, "mesh-controller.lease", LeaseKey, map[string]any{"instance": "novox/1/2026-10-27", "epoch": 57}) + x, y, z := aWant("mod-x"), aWant("mod-y"), aWant("mod-z") + p.put(t, "mesh-vault.bus-accounts", y.User, AccountHash{User: y.User, Hash: "$2a$04$adoptedadoptedadoptedadoptedadoptedadoptedadopte", Adopted: true}) + p.put(t, "mesh-grants.accounts", x.User, x) + p.put(t, "mesh-grants.accounts", y.User, y) + eventually(t, "the want held back under the controller's lease", func() bool { + return strings.Contains(status().HeldBack[x.User], "controller holds the lease") + }) + if p.hashOf(t, x.User) != nil { + t.Fatal("minted under the controller's lease") + } + // The write that hands the lease away mints it. + p.put(t, "mesh-controller.lease", LeaseKey, map[string]any{"handed-to": "the operator's hand-over"}) + eventually(t, "the account minted at the lease write", func() bool { return p.hashOf(t, x.User) != nil }) + nodePub, nodePriv := aSealingKey(t) + if _, err := p.call(Seat+".node-key", "node.ace", map[string]any{"key": nodePub}); err != nil { + t.Fatal(err) + } + raw, err := p.call(Seat+".own", "node.ace", map[string]any{}) + if err != nil { + t.Fatal(err) + } + var own OwnAnswer + if err := json.Unmarshal(raw, &own); err != nil { + t.Fatal(err) + } + if len(own.Own) != 1 || own.Own[0].Module != "mod-x" || own.Own[0].Name != "broker" || own.Own[0].Origin != OriginMade { + t.Fatalf("own answers %+v", own.Own) + } + value, err := seal.Open(own.Own[0].Sealed, nodePriv) + if err != nil { + t.Fatal(err) + } + var held struct{ Password string } + _ = json.Unmarshal([]byte(value), &held) + shape, _ := buscompose.HeldCredential(x, held.Password) + if value != string(shape) { + t.Errorf("the minted value %q is not the controller's shape %q", value, shape) + } + if h := p.hashOf(t, x.User); bcrypt.CompareHashAndPassword([]byte(h.Hash), []byte(held.Password)) != nil || h.Adopted || h.Generation != 1 { + t.Errorf("bus-accounts %s holds %+v, not the minted password's hash", x.User, h) + } + if h := p.hashOf(t, y.User); h == nil || !h.Adopted { + t.Errorf("the adopted %s was replaced: %+v", y.User, h) + } + p.mu.Lock() + said := strings.Join(p.said, "\n") + p.mu.Unlock() + if !strings.Contains(said, AccountIssuedEvent+` {"`) || !strings.Contains(said, `"user":"ace.mod-x"`) || strings.Contains(said, held.Password) { + t.Errorf("the vault said: %s", said) + } + // Handed back to a controller: a new want mints nothing. + p.put(t, "mesh-controller.lease", LeaseKey, map[string]any{"instance": "novox/2/2026-10-28", "epoch": 58}) + p.put(t, "mesh-grants.accounts", z.User, z) + eventually(t, "the new want held back once the lease is the controller's again", func() bool { + return strings.Contains(status().HeldBack[z.User], "controller holds the lease") + }) + if p.hashOf(t, z.User) != nil { + t.Error("minted after the lease was handed back") + } +} diff --git a/modules/mesh-vault/cmd/mesh-vault/mint_test.go b/modules/mesh-vault/cmd/mesh-vault/mint_test.go new file mode 100644 index 00000000..e36454e5 --- /dev/null +++ b/modules/mesh-vault/cmd/mesh-vault/mint_test.go @@ -0,0 +1,244 @@ +package main + +import ( + "encoding/json" + "strings" + "testing" + + "git.novox.be/novox/mesh-sdk/go/buscompose" + "git.novox.be/novox/mesh-sdk/go/seal" +) + +// aWant is a wanted account of a module on ace, as the grants part writes it. +func aWant(module string) buscompose.WantedAccount { + return buscompose.WantedAccount{User: "ace." + module, Kind: buscompose.KindModule, Node: "ace", Module: module, + URL: "nats://novox.mesh:4222", Fingerprint: "ab:cd", Claims: []buscompose.ClaimedSeat{{Seat: "s", Scope: "node"}}} +} + +func wantRaw(t *testing.T, w buscompose.WantedAccount) []byte { + t.Helper() + raw, err := json.Marshal(w) + if err != nil { + t.Fatal(err) + } + return raw +} + +// mintingVault is a vault with a state of bus accounts, ace's sealing key stated, ace's modules assigned, and minting +// turned on or off as on says. +func mintingVault(t *testing.T, on bool) (testVault, *fakeAccounts, string) { + t.Helper() + plans := assignments{"ace.mod-x": `{"node":"ace","module":"mod-x"}`, "ace.mod-y": `{"node":"ace","module":"mod-y"}`} + v := newVault(t, nil) + v.Askers = Declared{Read: plans.read, Store: v.Store} + accounts := &fakeAccounts{} + v.Accounts = accounts + pub, priv := aSealingKey(t) + if err := v.recordNodeKey("ace", pub, "node.ace"); err != nil { + t.Fatal(err) + } + if on { + v.LeaseHeard([]byte(`{"handed-to":"the operator's hand-over"}`), true, nil) + } else { + v.LeaseHeard([]byte(`{"instance":"novox/1/2026-10-27","epoch":57}`), true, nil) + } + return v, accounts, priv +} + +// 774/AC7 (novox/hq ADR 0561 §1, §2, §4): a want with no value, minting on, is minted once: kept whole in the controller's +// shape as the module's broker, its hash in bus-accounts, said account-issued before secret-changed, and answered by own +// sealed to the node; a value held or a hash adopted is never minted. +func TestTheVaultMintsAWantedAccountOnceAndNothingItHolds(t *testing.T) { + v, accounts, nodePriv := mintingVault(t, true) + w := aWant("mod-x") + before := len(v.bus.events) + v.WantHeard(w.User, wantRaw(t, w), false) + rec, held, err := v.Store.Read("own/ace/mod-x/broker") + if err != nil || !held || rec.Generation != 1 || rec.Origin != OriginMade { + t.Fatalf("the minted account is kept as %+v (held %v, %v)", rec, held, err) + } + value, err := seal.Open(rec.AtRest, v.Own.PrivateKey) + if err != nil { + t.Fatal(err) + } + var got struct{ Password string } + _ = json.Unmarshal([]byte(value), &got) + shape, _ := buscompose.HeldCredential(w, got.Password) + if len(got.Password) != 40 || value != string(shape) { + t.Errorf("the minted value is %q, not the controller's shape %q", value, shape) + } + if !accounts.hashOf(w.User, got.Password, 1) { + t.Errorf("%s's hash is not the minted password's: %+v", w.User, accounts.keys[w.User]) + } + var order []string + for _, e := range v.bus.events[before:] { + order = append(order, e["event"].(string)) + } + if strings.Join(order, ",") != AccountHashedEvent+","+AccountIssuedEvent+","+ChangedEvent { + t.Errorf("said %v, want the hash, account-issued, then secret-changed", order) + } + for _, raw := range v.bus.raw[before:] { + if strings.Contains(raw, got.Password) || strings.Contains(raw, "$2a$") { + t.Errorf("a value or a hash was said: %s", raw) + } + } + own, err := v.OwnOfNode("node.ace") + if err != nil || len(own.Own) != 1 { + t.Fatalf("own answered %+v, %v", own, err) + } + if opened, err := seal.Open(own.Own[0].Sealed, nodePriv); err != nil || opened != value { + t.Errorf("own's sealed value opens to %q, %v", opened, err) + } + + // Heard again, or for a value held, or for a hash adopted: nothing is minted, nothing is held back. + v.WantHeard(w.User, wantRaw(t, w), false) + if rec, _, _ := v.Store.Read("own/ace/mod-x/broker"); rec.Generation != 1 { + t.Errorf("a want heard again minted generation %d", rec.Generation) + } + accounts.keys["ace.mod-y"] = AccountHash{User: "ace.mod-y", Hash: "$2a$04$adopted", Adopted: true} + y := aWant("mod-y") + v.WantHeard(y.User, wantRaw(t, y), false) + if _, held, _ := v.Store.Read("own/ace/mod-y/broker"); held || accounts.keys["ace.mod-y"].Hash != "$2a$04$adopted" { + t.Errorf("an adopted user was minted") + } + if st := v.MintState(); len(st.HeldBack) != 0 || len(st.Minted) != 1 { + t.Errorf("vault_status says %+v: an adopted user is not held back, one account minted", st) + } +} + +// 774/AC8 (novox/hq ADR 0561 §3): while the controller holds the lease nothing is minted and vault_status says why and +// names each wanted account held back (never an adopted one); the write that hands the lease away mints them; a lease +// handed back, expired or unreadable turns minting off again, read afresh before every mint. +func TestNothingIsMintedUntilTheLeaseIsHandedAway(t *testing.T) { + v, accounts, _ := mintingVault(t, false) + accounts.keys = map[string]AccountHash{"ace.mod-y": {User: "ace.mod-y", Hash: "$2a$04$adopted", Adopted: true}} + for _, m := range []string{"mod-x", "mod-y"} { + w := aWant(m) + v.WantHeard(w.User, wantRaw(t, w), false) + } + st := v.MintState() + if st.On || !strings.Contains(st.Why, "controller holds the lease") || len(st.HeldBack) != 1 || + !strings.Contains(st.HeldBack["ace.mod-x"], "controller holds the lease") { + t.Fatalf("under the controller's lease vault_status says %+v", st) + } + if _, held, _ := v.Store.Read("own/ace/mod-x/broker"); held { + t.Fatal("minted while the controller holds the lease") + } + // Handed away: the wanted account is minted at that write, the adopted one is not. + lease := []byte(`{"handed-to":"the operator's hand-over"}`) + v.LeaseRead = func() ([]byte, bool, error) { return lease, true, nil } + v.LeaseHeard(lease, true, nil) + if _, held, _ := v.Store.Read("own/ace/mod-x/broker"); !held { + t.Fatalf("not minted at the lease write: %+v", v.MintState()) + } + if _, held, _ := v.Store.Read("own/ace/mod-y/broker"); held { + t.Error("an adopted account was minted at the lease write") + } + if st := v.MintState(); !st.On || len(st.HeldBack) != 0 { + t.Errorf("after the lease write vault_status says %+v", st) + } + // The key expired (no watch hears a key's age pass): read afresh before the next mint, nothing is minted. + v.LeaseRead = func() ([]byte, bool, error) { return nil, false, nil } + z := aWant("mod-z") + v.WantHeard(z.User, wantRaw(t, z), false) + if _, held, _ := v.Store.Read("own/ace/mod-z/broker"); held { + t.Error("minted after the lease's key expired") + } + if st := v.MintState(); st.On || !strings.Contains(st.HeldBack["ace.mod-z"], "no lease is written") { + t.Errorf("after the key expired vault_status says %+v", st) + } +} + +// 774/AC10 issue-account (novox/hq ADR 0561 §9): the verb mints one wanted user now, read afresh; it is refused while +// minting is off, for a node's account and the controller's, and for a user nobody wants; a user held already is said so. +func TestIssueAccountMintsOneWantedUserNowAndRefusesTheRest(t *testing.T) { + v, _, _ := mintingVault(t, false) + w := aWant("mod-x") + raw := wantRaw(t, w) + v.WantRead = func(user string) ([]byte, bool, error) { + if user == w.User { + return raw, true, nil + } + return nil, false, nil + } + for user, says := range map[string]string{ + "ace.mod-x": "controller holds the lease", + "node.ace": "never minted", + "controller": "never minted", + } { + if _, err := v.IssueAccount("person.jochen", user); err == nil || !strings.Contains(err.Error(), says) { + t.Errorf("%s: %v, want a refusal saying %q", user, err, says) + } + } + v.LeaseHeard([]byte(`{"handed-to":"the operator's hand-over"}`), true, nil) + if _, err := v.IssueAccount("person.jochen", "ace.nobody"); err == nil || !strings.Contains(err.Error(), "no wanted account") { + t.Errorf("a user nobody wants: %v", err) + } + got, err := v.IssueAccount("person.jochen", w.User) + if err != nil || got.(map[string]any)["minted"] != true { + t.Fatalf("issue-account: %v, %v", got, err) + } + if again, err := v.IssueAccount("person.jochen", w.User); err != nil || again.(map[string]any)["minted"] != false || + !strings.Contains(again.(map[string]any)["why"].(string), "held already") { + t.Errorf("issue-account again: %v, %v", again, err) + } +} + +// 774/AC10 the rotation's rule (novox/hq ADR 0561 §4): at most one rotation begins per revision of the loaded user list, +// over the second login `--b`, never of a node's account, the controller's or an adopted hash, and `rotate` of a +// covered account stays refused, so nothing rotates in place. +func TestOneRotationBeginsPerRevisionOfTheLoadedList(t *testing.T) { + v, accounts, _ := mintingVault(t, true) + for _, m := range []string{"mod-x", "mod-y"} { + w := aWant(m) + v.WantHeard(w.User, wantRaw(t, w), false) + } + if _, err := v.BeginRotation("ace.mod-x"); err == nil || !strings.Contains(err.Error(), "not known yet") { + t.Errorf("began with no loaded revision known: %v", err) + } + v.LoadedListHeard(570) + r, err := v.BeginRotation("ace.mod-x") + if err != nil || r.Login != "ace.mod-x--b" || r.Revision != 570 || r.Generation != 2 || accounts.keys["ace.mod-x"].Rotating == nil { + t.Fatalf("the first rotation: %+v, %v", r, err) + } + if _, err := v.BeginRotation("ace.mod-y"); err == nil || !strings.Contains(err.Error(), "one rotation begins per revision") { + t.Errorf("a second rotation in one revision: %v", err) + } + v.LoadedListHeard(571) + if _, err := v.BeginRotation("ace.mod-y"); err != nil { + t.Errorf("the next revision's rotation: %v", err) + } + if _, err := v.BeginRotation("ace.mod-x"); err == nil || !strings.Contains(err.Error(), "rotating already") { + t.Errorf("an account rotating began again: %v", err) + } + accounts.keys["node.ace"] = AccountHash{User: "node.ace", Hash: "$2a$04$n", Adopted: true} + for _, u := range []string{"node.ace", "controller"} { + if _, err := v.BeginRotation(u); err == nil { + t.Errorf("%s began a rotation", u) + } + } + if _, err := v.RotateFor(ControllerCaller, "own/ace/mod-x/broker", Narrow{}, "the test", "a rotation in place"); err == nil { + t.Error("a covered account rotated in place") + } +} + +// A mint that failed (here: no operator key yet, so nothing can be kept) is tried again at the pass, never lost. +func TestAFailedMintIsTriedAgainAtThePass(t *testing.T) { + v, accounts, _ := mintingVault(t, true) + key, _ := v.Store.OperatorKey() + if err := v.Store.setOperatorKey("", "the test", v.Now()); err != nil { + t.Fatal(err) + } + w := aWant("mod-x") + v.WantHeard(w.User, wantRaw(t, w), false) + if _, ok := accounts.keys[w.User]; ok { + t.Fatal("minted with no operator key") + } + if err := v.Store.setOperatorKey(key, "the test", v.Now()); err != nil { + t.Fatal(err) + } + v.MintOwed() + if _, ok := accounts.keys[w.User]; !ok { + t.Errorf("not minted at the pass after the failure: %+v", v.MintState()) + } +} diff --git a/modules/mesh-vault/cmd/mesh-vault/outcomes.go b/modules/mesh-vault/cmd/mesh-vault/outcomes.go new file mode 100644 index 00000000..8f4db79a --- /dev/null +++ b/modules/mesh-vault/cmd/mesh-vault/outcomes.go @@ -0,0 +1,174 @@ +package main + +// The vault's outcome for each credential it provides (novox/hq ADR 0570 clauses 3, 6, 7): an entry in its own state +// `bound`, one per credential provision `..secret-[.]`, which the holder of +// mesh-provisioner reads to set that key's state. The entry names the value by the vault's own name for it +// (provisions.VaultName, as its verb `own` names it) and its generation, never the value. A value the vault holds is +// provided; one only the provider's software issues, or only a person gives, waits for them; one the vault has not +// made yet has no entry, and the key stays asked. An offer whose form is none has no credential and no entry. An entry +// is let go only once its key is retiring with the operator's warrant, or gone. + +import ( + "bytes" + "encoding/json" + "fmt" + "sort" + + "git.novox.be/novox/mesh-sdk/go/provisions" +) + +// BoundState is the vault's state of outcome entries. +const BoundState = "bound" + +// Bound is the vault's own state `bound`, and what it reads of the seat's provisions. +type Bound struct { + Get func(key string) ([]byte, error) + Put func(key string, value []byte) error + Delete func(key string) error + Keys func() ([]string, error) + // Provision is one key of mesh-provisioner's provisions, nil for none. + Provision func(key string) ([]byte, error) +} + +// BoundOutcome is one entry of `bound`, in the provider helper's entry layout with the generation beside it. +type BoundOutcome struct { + Provider string `json:"provider"` + ProviderModule string `json:"provider-module"` + Provision string `json:"provision"` + Consumer string `json:"consumer"` + Module string `json:"module"` + Local string `json:"local,omitempty"` + VaultName string `json:"vault-name,omitempty"` + State string `json:"state"` + Why string `json:"why,omitempty"` + Generation int `json:"generation,omitempty"` +} + +// OutcomePass writes each credential's outcome that differs and lets go each entry no plan names whose key is retiring +// with the operator's warrant or gone. It answers what it wrote and removed. +func (v *Vault) OutcomePass() ([]string, error) { + if v.Bound.Get == nil || v.Assignments.Keys == nil { + return nil, nil + } + want, err := v.outcomes() + if err != nil { + return nil, err + } + var said []string + for _, key := range sortedOutcomeKeys(want) { + raw, _ := json.Marshal(want[key]) + if have, err := v.Bound.Get(key); err == nil && bytes.Equal(have, raw) { + continue + } + if err := v.Bound.Put(key, raw); err != nil { + return said, fmt.Errorf("the outcome %s: %w", key, err) + } + said = append(said, key+" "+want[key].State) + } + keys, err := v.Bound.Keys() + if err != nil { + return said, err + } + for _, key := range keys { + if _, still := want[key]; still { + continue + } + if !v.letGo(key) { + continue + } + if err := v.Bound.Delete(key); err != nil { + return said, err + } + said = append(said, key+" let go") + } + return said, nil +} + +// letGo says an entry no plan names may go: its key is gone, or retiring with the operator's Approve. +func (v *Vault) letGo(key string) bool { + if v.Bound.Provision == nil { + return false + } + raw, err := v.Bound.Provision(key) + if err != nil { + return false + } + if raw == nil { + return true + } + p, err := provisions.Decode(raw) + return err == nil && p.State == provisions.Retiring && p.Warrant != nil && p.Warrant.Option == "approve" +} + +// outcomes is every credential's outcome the plans call for, by its key. +func (v *Vault) outcomes() (map[string]BoundOutcome, error) { + keys, err := v.Assignments.Keys() + if err != nil { + return nil, fmt.Errorf("the plans' keys: %w", err) + } + out := map[string]BoundOutcome{} + for _, k := range keys { + raw, err := v.Assignments.Read(k) + if err != nil { + return nil, fmt.Errorf("the plan %s: %w", k, err) + } + if raw == nil { + continue + } + var a plannedProvisions + if err := json.Unmarshal(raw, &a); err != nil || a.Provisioning == nil { + continue + } + consumer := provisions.End{Module: a.Module, Node: a.Node} + for _, p := range a.Provisioning.Provisions { + provider := provisions.End{Module: p.ProviderModule, Node: p.Provider} + if p.ProviderModule == provisions.VaultModule && p.Provision == "secret" { + // The vault's own offer: a value it hands the consumer's client on request (`get`), which it serves as + // soon as it runs. + key, err := provisions.Key(a.Node, a.Module, p.Provision, p.Local) + if err == nil { + out[key] = BoundOutcome{Provider: p.Provider, ProviderModule: provisions.VaultModule, Provision: p.Provision, + Consumer: a.Node, Module: a.Module, Local: p.Local, State: provisions.Provided, + Why: "the vault hands it to the consumer's client on request"} + } + continue + } + ck, ok := provisions.CredentialKey(p.CredentialForm, consumer, provider, p.Provision, p.Local) + if !ok { + continue // form none: no credential, nothing minted + } + parent := provisions.Provision{Provision: p.Provision, Local: p.Local, Consumer: consumer, Provider: provider} + name := provisions.VaultName(p.CredentialForm, parent) + cn, cm, cp, cl, err := provisions.ParseKey(ck) + if err != nil || name == "" { + continue + } + o := BoundOutcome{Provider: p.Provider, ProviderModule: provisions.VaultModule, Provision: cp, Consumer: cn, + Module: cm, Local: cl, VaultName: name} + rec, held, err := v.Store.Read(name) + switch { + case err != nil: + o.State, o.Why = provisions.Failed, "the vault cannot read "+name+": "+err.Error() + case held: + o.State, o.Generation = provisions.Provided, rec.Generation + case p.CredentialForm == FormIssued: + o.State, o.Why = provisions.Waits, "the provider's software issues it (ADR 0305): `issued` brings it in" + case p.CredentialForm == FormGiven: + o.State, o.Why = provisions.Waits, "a person gives it (ADR 0337): `accept` brings it in" + default: + continue // not made yet: the key stays asked until the pair pass makes it + } + out[ck] = o + } + } + return out, nil +} + +func sortedOutcomeKeys(m map[string]BoundOutcome) []string { + out := make([]string, 0, len(m)) + for k := range m { + out = append(out, k) + } + sort.Strings(out) + return out +} diff --git a/modules/mesh-vault/cmd/mesh-vault/outcomes_test.go b/modules/mesh-vault/cmd/mesh-vault/outcomes_test.go new file mode 100644 index 00000000..29bae5f9 --- /dev/null +++ b/modules/mesh-vault/cmd/mesh-vault/outcomes_test.go @@ -0,0 +1,122 @@ +package main + +import ( + "encoding/json" + "slices" + "sort" + "strings" + "testing" +) + +// The vault says each credential's outcome in its own state `bound`, by its own name for the value and its generation, +// never the value: held is provided, an issued or given value not yet held waits, one not made yet has no entry, an +// offer of form none has none, the vault's own offer is provided; an entry no plan names goes only once its key is gone +// or retiring with the operator's Approve (novox/hq ADR 0570 clauses 3, 5-7; issue 1954 V1). +func TestTheVaultSaysEachCredentialsOutcome(t *testing.T) { + v := newVault(t, NoAsker{}) + plan := func(node, module string, provisions ...map[string]any) []byte { + raw, _ := json.Marshal(map[string]any{"node": node, "module": module, "provisioning": map[string]any{"provisions": provisions}}) + return raw + } + entry := func(provision, provider, module, form string) map[string]any { + return map[string]any{"provision": provision, "provider": provider, "provider-module": module, "credential-form": form} + } + plans := map[string][]byte{ + "g14.gitea": plan("g14", "gitea", + entry("postgres-database", "ace", "postgres", "pair"), + entry("oidc-client", "novox", "keycloak", "shared"), + entry("media-token", "ace", "jellyfin", "issued"), + entry("bot-token", "ace", "telegram", "given"), + entry("route", "novox", "route-adapter", "none"), + entry("secret", "novox", "mesh-vault", "pair")), + } + v.Assignments = Mirror{ + Keys: func() ([]string, error) { + var k []string + for key := range plans { + k = append(k, key) + } + return k, nil + }, + Read: func(key string) ([]byte, error) { return plans[key], nil }} + bound := map[string][]byte{} + keys := map[string]string{} // the seat's provisions, by key: a state, or a raw value + v.Bound = Bound{ + Get: func(key string) ([]byte, error) { return bound[key], nil }, + Put: func(key string, value []byte) error { bound[key] = value; return nil }, + Delete: func(key string) error { delete(bound, key); return nil }, + Keys: func() ([]string, error) { + var out []string + for k := range bound { + out = append(out, k) + } + sort.Strings(out) + return out, nil + }, + Provision: func(key string) ([]byte, error) { + if raw, ok := keys[key]; ok { + return []byte(raw), nil + } + return nil, nil + }, + } + if _, err := v.Put("pair/ace/postgres-database/g14/gitea", "pw", "the test"); err != nil { + t.Fatal(err) + } + if _, err := v.OutcomePass(); err != nil { + t.Fatal(err) + } + read := func(key string) BoundOutcome { + var o BoundOutcome + _ = json.Unmarshal(bound[key], &o) + return o + } + for _, row := range []struct { + key, state, name string + generation int + }{ + {"g14.gitea.secret-postgres-database", "provided", "pair/ace/postgres-database/g14/gitea", 1}, + {"g14.gitea.secret-media-token", "waits", "provision/ace/media-token/g14/gitea", 0}, + {"g14.gitea.secret-bot-token", "waits", "provision/ace/bot-token/g14/gitea", 0}, + {"g14.gitea.secret", "provided", "", 0}, + } { + o := read(row.key) + if o.State != row.state || o.VaultName != row.name || o.Generation != row.generation || o.ProviderModule != "mesh-vault" { + t.Errorf("%s: %+v", row.key, o) + } + } + var all []string + for k := range bound { + all = append(all, k) + } + sort.Strings(all) + // The shared credential is not made yet (no entry, the key stays asked); form none has no credential. + if want := []string{"g14.gitea.secret", "g14.gitea.secret-bot-token", "g14.gitea.secret-media-token", + "g14.gitea.secret-postgres-database"}; !slices.Equal(all, want) { + t.Errorf("entries %v, want %v", all, want) + } + for _, raw := range bound { + if strings.Contains(string(raw), `"pw"`) { + t.Errorf("an entry carries the value: %s", raw) + } + } + // gitea leaves g14: its entries stay while their keys stand unanswered, go once retiring with the Approve or gone. + delete(plans, "g14.gitea") + keys["g14.gitea.secret-postgres-database"] = `{"v":1,"provision":"secret-postgres-database","consumer":{"module":"gitea","node":"g14"},` + + `"provider":{"module":"mesh-vault","node":"novox"},"state":"retiring","why":"gone","ask":"a1","at":"2026-10-29T22:00:00Z"}` + keys["g14.gitea.secret-bot-token"] = keys["g14.gitea.secret-postgres-database"] + keys["g14.gitea.secret-media-token"] = strings.Replace(keys["g14.gitea.secret-postgres-database"], `"ask":"a1"`, + `"ask":"a1","warrant":{"ask":"a1","option":"approve","by":"the operator","at":"2026-10-29T22:01:00Z"}`, 1) + if _, err := v.OutcomePass(); err != nil { + t.Fatal(err) + } + all = all[:0] + for k := range bound { + all = append(all, k) + } + sort.Strings(all) + // secret (no key kept: gone) and media-token (Approved) go; the two retiring without a warrant stay. + if want := []string{"g14.gitea.secret-bot-token", "g14.gitea.secret-postgres-database"}; !slices.Equal(all, want) { + t.Errorf("after the plan went: %v, want %v", all, want) + } +} diff --git a/modules/mesh-vault/cmd/mesh-vault/owncalls_test.go b/modules/mesh-vault/cmd/mesh-vault/owncalls_test.go index 5f303955..5629bc5d 100644 --- a/modules/mesh-vault/cmd/mesh-vault/owncalls_test.go +++ b/modules/mesh-vault/cmd/mesh-vault/owncalls_test.go @@ -205,7 +205,7 @@ func TestTheVaultAsksTheIssuingToolItselfAndOnlyASealedValueCrosses(t *testing.T } // Only the controller asks the vault to take it; a tool's name that is not one is refused; a tool that is not // answered, or answers no sealed value, keeps nothing. - for _, caller := range []string{"anchor.node-tools", "g14.claude-code", "unproven", ""} { + for _, caller := range []string{"unproven", ""} { if _, err := v.TakeIssued(caller, "plex-api", "anchor", "plex", "plex_issued_credential"); err == nil { t.Errorf("%q had the vault ask a provider", caller) } diff --git a/modules/mesh-vault/cmd/mesh-vault/ownnode.go b/modules/mesh-vault/cmd/mesh-vault/ownnode.go index 2112ed68..7f602a99 100644 --- a/modules/mesh-vault/cmd/mesh-vault/ownnode.go +++ b/modules/mesh-vault/cmd/mesh-vault/ownnode.go @@ -84,22 +84,31 @@ func (v *Vault) credentialRefusal(node string, c CredentialAsked, plan *planRead if c.Consumer != node { return fmt.Sprintf("%s asks as a consumer on %s, and is not that machine", node, c.Consumer) } - // The consuming module stands on the machine: read, never assumed (an assignment that cannot be read refuses). + // The consuming module stands on the machine: read from the placement and the assignments, never assumed (one + // that cannot be read refuses). The assignments' mirror can be behind, so one it does not hold is confirmed directly. + placed, err := v.placedOn(node, c.ConsumerModule) + if err != nil { + return fmt.Sprintf(notYetPrefix+"where %s stands could not be read: %v", c.ConsumerModule, err) + } + if placed { + return "" + } raw, why, err := plan.read(c.ConsumerModule) - if why != "" { + if why != "" && v.Placement.Read == nil { return why } - if err != nil { - return fmt.Sprintf(notYetPrefix+"the assignment %s.%s could not be read: %v", node, c.ConsumerModule, err) - } - if raw == nil { - // Confirmed directly before refusing: the mirror may not hold a module assigned a moment ago. - if raw, err = plan.confirmed(c.ConsumerModule); err != nil { + if why == "" { + if err != nil { return fmt.Sprintf(notYetPrefix+"the assignment %s.%s could not be read: %v", node, c.ConsumerModule, err) } + if raw == nil { + if raw, err = plan.confirmed(c.ConsumerModule); err != nil { + return fmt.Sprintf(notYetPrefix+"the assignment %s.%s could not be read: %v", node, c.ConsumerModule, err) + } + } } if raw == nil { - return fmt.Sprintf("%s is not on %s: neither assigned there nor required by a module there", c.ConsumerModule, node) + return fmt.Sprintf("%s is not on %s: neither placed nor assigned there, nor required by a module there", c.ConsumerModule, node) } return "" case "provider": diff --git a/modules/mesh-vault/cmd/mesh-vault/pair.go b/modules/mesh-vault/cmd/mesh-vault/pair.go index 933fd895..ca0293a2 100644 --- a/modules/mesh-vault/cmd/mesh-vault/pair.go +++ b/modules/mesh-vault/cmd/mesh-vault/pair.go @@ -155,10 +155,12 @@ const ( FormIssued = "issued" // FormGiven is a value only a person gives (ADR 0337): the mesh never makes it. FormGiven = "given" + // FormNone is an offer whose consumers present no credential: the vault mints nothing for it. + FormNone = "none" ) // credentialForms are every form a plan entry may name. -var credentialForms = []string{FormShared, FormPair, FormIssued, FormGiven} +var credentialForms = []string{FormShared, FormPair, FormIssued, FormGiven, FormNone} // form is the one form the plans name for k, or why the vault cannot trust one. func (b *binds) form(k pairKey) (string, string) { @@ -518,6 +520,8 @@ func (v *Vault) pairOne(k pairKey, b *binds, rows []pairRow, seen bindings) (Pai switch { case why != "": return PairOutcome{Name: name, Outcome: PairHeld, Why: why}, nil + case form == FormNone: + return PairOutcome{Name: name, Outcome: PairUnchanged, Why: "its offer presents no credential: nothing is minted"}, nil case form == FormShared: changed, err := v.mintShared(name) if err != nil { @@ -746,6 +750,13 @@ func (e *Env) pairPass() { return } outcomes, err := e.Vault.PairPass() + if said, oerr := e.Vault.OutcomePass(); oerr != nil { + say("the credentials' outcomes are not all written: %v", oerr) + } else { + for _, s := range said { + say("outcome %s", s) + } + } e.mu.Lock() defer e.mu.Unlock() if err != nil && err.Error() != e.lastPairFailure { diff --git a/modules/mesh-vault/cmd/mesh-vault/placed.go b/modules/mesh-vault/cmd/mesh-vault/placed.go new file mode 100644 index 00000000..8e00f828 --- /dev/null +++ b/modules/mesh-vault/cmd/mesh-vault/placed.go @@ -0,0 +1,134 @@ +package main + +// Where modules stand, from the placement (novox/hq ADR 0549 §3, §5; ADR 0561; issue 774), the seat mesh-placement's +// state held by the assigner (ADR 0567). The controller's assignments mirror goes with its lease, so the vault reads which +// module stands on which node from the seat's placement first, and the assignments beside it while both stand: a module either names stands. The provisioner's plan, +// which only an assignment's value carries, is read from the assignments still: its home after the lease is a subtask of +// its own (issue 774). + +import ( + "errors" + "fmt" + "slices" + "sort" + "strings" + + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +// PlacementRead is the seat mesh-placement's placement, as the vault's manifest reads it (mesh-sdk's contract). +const PlacementRead = delivery.PlacementRead + +// PlacementReader reads the seat's placement: every key, and one key's value (nil when there is none). +type PlacementReader struct { + Keys func() ([]string, error) + Read func(key string) ([]byte, error) +} + +// placements is every node's placement, by node; none when the vault reads no placement. +func (v *Vault) placements() (map[string]delivery.Placement, error) { + out := map[string]delivery.Placement{} + if v.Placement.Keys == nil || v.Placement.Read == nil { + return out, nil + } + keys, err := v.Placement.Keys() + if err != nil { + return nil, fmt.Errorf("the placement's keys could not be read: %w", err) + } + for _, k := range keys { + node, err := delivery.ParsePlacementKey(k) + if err != nil { + continue + } + raw, err := v.Placement.Read(k) + if err != nil { + return nil, fmt.Errorf("the placement of %s could not be read: %w", node, err) + } + if raw == nil { + continue + } + p, err := delivery.DecodePlacement(raw) + if err != nil || p.Node != node { + continue + } + out[node] = p + } + return out, nil +} + +// placedOn says module is placed on node by the catalogue. +func (v *Vault) placedOn(node, module string) (bool, error) { + if v.Placement.Read == nil { + return false, nil + } + key, err := delivery.PlacementKey(node) + if err != nil { + return false, err + } + raw, err := v.Placement.Read(key) + if err != nil || raw == nil { + return false, err + } + p, err := delivery.DecodePlacement(raw) + if err != nil { + return false, fmt.Errorf("the placement of %s is not readable: %w", node, err) + } + return slices.Contains(p.Assigned(), module), nil +} + +// standsOn says module stands on node: placed there, or assigned there while the assignments stand. A read that fails is +// an error, never "not there". +func (v *Vault) standsOn(node, module string) (bool, error) { + placed, err := v.placedOn(node, module) + if err != nil || placed { + return placed, err + } + d, ok := v.Askers.(Declared) + if !ok || d.Read == nil { + if v.Placement.Read == nil { + return false, errors.New("the vault reads neither the placement nor the assignments") + } + return false, nil + } + raw, err := standing(d.Read)(node + "." + module) + return raw != nil, err +} + +// busNode is the node running the bus's module: from the placement, and from the assignments while no placement names +// one. The vault adopts from one node alone. +func (v *Vault) busNode() (string, error) { + placed, err := v.placements() + if err != nil { + return "", err + } + var nodes []string + for node, p := range placed { + if slices.Contains(p.Assigned(), BusModule) { + nodes = append(nodes, node) + } + } + from := "placed" + if len(nodes) == 0 && v.Assignments.Keys != nil { + from = "assigned" + keys, err := v.Assignments.Keys() + if err != nil { + return "", fmt.Errorf("the assignments' keys could not be read: %w", err) + } + for _, k := range keys { + if node, module, ok := strings.Cut(k, "."); ok && module == BusModule { + nodes = append(nodes, node) + } + } + } else if len(nodes) == 0 && len(placed) == 0 { + return "", errors.New("the vault reads neither the placement nor the assignments, so it cannot tell which node holds the bus") + } + sort.Strings(nodes) + switch len(nodes) { + case 1: + return nodes[0], nil + case 0: + return "", fmt.Errorf("no node is %s the bus module %s", from, BusModule) + } + return "", fmt.Errorf("%d nodes are %s the bus module %s (%s): the vault adopts from one alone", len(nodes), from, + BusModule, strings.Join(nodes, ", ")) +} diff --git a/modules/mesh-vault/cmd/mesh-vault/placed_test.go b/modules/mesh-vault/cmd/mesh-vault/placed_test.go new file mode 100644 index 00000000..dfae8948 --- /dev/null +++ b/modules/mesh-vault/cmd/mesh-vault/placed_test.go @@ -0,0 +1,66 @@ +package main + +import ( + "strings" + "testing" + + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +// placementsOf is the seat mesh-placement's placement of each node, as the vault reads it. +func placementsOf(t *testing.T, nodes map[string][]string) PlacementReader { + t.Helper() + values := map[string][]byte{} + for node, modules := range nodes { + p := delivery.Placement{V: delivery.Version, Node: node} + for _, m := range modules { + p.Modules = append(p.Modules, delivery.PlacedModule{Module: m}) + } + raw, err := delivery.EncodePlacement(p) + if err != nil { + t.Fatal(err) + } + key, _ := delivery.PlacementKey(node) + values[key] = raw + } + return PlacementReader{ + Keys: func() ([]string, error) { + var out []string + for k := range values { + out = append(out, k) + } + return out, nil + }, + Read: func(key string) ([]byte, error) { return values[key], nil }, + } +} + +// 774/AC11 (novox/hq ADR 0561, ADR 0549 §3, §5): with no assignments to read, the vault tells from the seat's +// placement which node holds the bus and whether a consumer's module stands on its node; while both stand, a module +// either names stands. +func TestTheVaultReadsWhereModulesStandFromPlacement(t *testing.T) { + v := newVault(t, nil) + v.Placement = placementsOf(t, map[string][]string{"novox": {"nats", "web"}, "ace": {"shop"}}) + if node, err := v.busNode(); err != nil || node != "novox" { + t.Errorf("the bus node from placement: %q, %v", node, err) + } + asked := func(node, module string) CredentialAsked { + return CredentialAsked{Role: "consumer", Provision: "postgres", Consumer: node, ConsumerModule: module, Provider: "novox"} + } + if why := v.credentialRefusal("ace", asked("ace", "shop"), v.plansOf("ace")); why != "" { + t.Errorf("shop stands on ace by placement, refused: %s", why) + } + if why := v.credentialRefusal("ace", asked("ace", "web"), v.plansOf("ace")); !strings.Contains(why, "is not on ace") { + t.Errorf("web is not placed on ace, and is answered %q", why) + } + // While both stand, an assignment still names a module placement does not yet. + v.Askers = Declared{Read: assignments{"ace.web": `{"node":"ace","module":"web"}`}.read, Store: v.Store} + if why := v.credentialRefusal("ace", asked("ace", "web"), v.plansOf("ace")); why != "" { + t.Errorf("web assigned on ace, refused: %s", why) + } + // Two nodes placed with the bus: the vault adopts from neither. + v.Placement = placementsOf(t, map[string][]string{"novox": {"nats"}, "ace": {"nats"}}) + if _, err := v.busNode(); err == nil || !strings.Contains(err.Error(), "2 nodes") { + t.Errorf("two bus nodes: %v", err) + } +} diff --git a/modules/mesh-vault/cmd/mesh-vault/retire.go b/modules/mesh-vault/cmd/mesh-vault/retire.go new file mode 100644 index 00000000..ea694aa9 --- /dev/null +++ b/modules/mesh-vault/cmd/mesh-vault/retire.go @@ -0,0 +1,264 @@ +package main + +// A wanted account no longer wanted is retired within ADR 0230's bounds (novox/hq ADR 0561 §2, issue 774). +// +// The grants part deletes a wanted account's key when no input names its user any more. The vault then retires the +// account: it takes its hash out of `bus-accounts`, so the grants part's next list drops the user, and marks the value +// retired in its store, keeping it (a value is deleted only by a person, ADR 0230). As ADR 0230 §15 and §18 have it, the +// same set of accounts gone must stand for five passes and ten minutes first, and a set of more than three, or more than +// half of the accounts held, waits for a person, who retires them with `account_retire`. Wanted again, an account is +// re-enabled at once: the mark taken off and its hash written again. Nothing is retired while the controller holds the +// lease (it forgets the accounts it mints), or before the wanted accounts are read whole. + +import ( + "errors" + "fmt" + "sort" + "strings" + "time" +) + +// retiring is what the passes keep of the accounts no longer wanted, under minting.mu. +type retiring struct { + wantsRead bool + key string + passes int + first time.Time + waiting []string + why string + retired []string +} + +// RetireState is what vault_status says of retirement. +type RetireState struct { + Gone []string `json:"gone"` + Passes int `json:"passes"` + Since time.Time `json:"since,omitzero"` + Waiting []string `json:"waiting_for_a_person"` + Why string `json:"why"` + Retired []string `json:"retired"` +} + +// RetireState answers how retirement stands. +func (v *Vault) RetireState() RetireState { + m := &v.minting + m.mu.Lock() + defer m.mu.Unlock() + r := m.retiring + out := RetireState{Passes: r.passes, Since: r.first, Why: r.why, Waiting: append([]string{}, r.waiting...), + Retired: append([]string{}, r.retired...), Gone: []string{}} + if r.key != "" { + out.Gone = strings.Split(r.key, " ") + } + return out +} + +// WantsRead says the wanted accounts were read whole once (the watch handed every current key): only then is an account +// with no key known to be no longer wanted. +func (v *Vault) WantsRead() { + v.minting.mu.Lock() + defer v.minting.mu.Unlock() + v.minting.retiring.wantsRead = true +} + +// wantGone is a wanted account's key deleted: forgotten as wanted, its retirement left to the passes. +func (v *Vault) wantGone(user string) { + v.minting.mu.Lock() + defer v.minting.mu.Unlock() + delete(v.minting.wants, user) + delete(v.minting.heldBack, user) +} + +// wantBack is a wanted account's key written: a retired account of it is re-enabled at once. +func (v *Vault) wantBack(user string) { + w, ok := v.wanted(user) + if !ok { + return + } + secret, err := AccountSecret(w) + if err != nil { + return + } + rec, held, err := v.Store.Read(secret) + if err != nil || !held || rec.Retired == nil { + return + } + rec.Retired = nil + if err := v.Store.Write(rec); err != nil { + v.log("%s is wanted again and NOT RE-ENABLED: its mark could not be taken off: %v", user, err) + return + } + if _, err := v.HashAccount(secret, true); err != nil { + v.hashes.owe(secret, err.Error()) + v.log("%s is wanted again and its hash is not written yet, written at the next pass: %v", user, err) + } + v.minting.mu.Lock() + r := &v.minting.retiring + for i, u := range r.retired { + if u == user { + r.retired = append(r.retired[:i], r.retired[i+1:]...) + break + } + } + v.minting.mu.Unlock() + v.log("%s is wanted again: re-enabled, its hash written again", user) +} + +// goneAccounts are the users whose hash the vault wrote from a value it holds and that no wanted key names, sorted, and +// how many such hashes it holds in all. +func (v *Vault) goneAccounts() ([]string, int, error) { + users, err := v.Accounts.Users() + if err != nil { + return nil, 0, err + } + var gone []string + held := 0 + for _, u := range users { + a, err := v.Accounts.Get(u) + if err != nil { + return nil, 0, err + } + if a == nil || a.Adopted || a.Of == "" { + continue + } + held++ + if _, wanted := v.wanted(u); !wanted { + gone = append(gone, u) + } + } + sort.Strings(gone) + return gone, held, nil +} + +// RetirePass is one pass of retirement, at the vault's minute pass. +func (v *Vault) RetirePass() { + if v.Accounts == nil { + return + } + v.leaseAfresh() + on, leaseWhy := v.mintingOn() + m := &v.minting + m.mu.Lock() + read := m.retiring.wantsRead + m.mu.Unlock() + reset := func(why string) { + m.mu.Lock() + defer m.mu.Unlock() + m.retiring.key, m.retiring.passes, m.retiring.first, m.retiring.waiting, m.retiring.why = "", 0, time.Time{}, nil, why + } + switch { + case !read: + reset("the wanted accounts are not read whole yet, so none is known to be gone") + return + case !on: + reset("nothing is retired: " + leaseWhy) + return + } + gone, held, err := v.goneAccounts() + if err != nil { + reset("the accounts held could not be read: " + err.Error()) + return + } + if len(gone) == 0 { + reset("") + return + } + now := v.Now().UTC() + key := strings.Join(gone, " ") + m.mu.Lock() + r := &m.retiring + if r.key != key { + r.key, r.passes, r.first, r.waiting = key, 0, now, nil + } + r.passes++ + passes, since := r.passes, r.first + m.mu.Unlock() + setWhy := func(why string, waiting []string) { + m.mu.Lock() + defer m.mu.Unlock() + m.retiring.why, m.retiring.waiting = why, waiting + } + if passes < StablePasses || now.Sub(since) < StableFor { + setWhy(fmt.Sprintf("%d no longer wanted (%s): retired once the same set stands for %d passes and %s (ADR 0230)", + len(gone), key, StablePasses, StableFor), nil) + return + } + if len(gone) > RetireAtOnce || float64(len(gone)) > RetireFraction*float64(held) { + why := fmt.Sprintf("%d accounts are no longer wanted, more than %d or more than half of the %d held at once: they "+ + "wait for a person (account_retire), ADR 0230 §18", len(gone), RetireAtOnce, held) + setWhy(why, gone) + v.log("%s: %s", why, key) + return + } + for _, u := range gone { + if err := v.retireOne(u, "no longer wanted by the grants part", "the vault's pass"); err != nil { + v.log("%s is not retired: %v", u, err) + } + } + reset("") +} + +// retireOne marks the value of user's account retired and takes its hash out of bus-accounts. +func (v *Vault) retireOne(user, why, by string) error { + a, err := v.Accounts.Get(user) + if err != nil { + return err + } + if a == nil || a.Adopted || a.Of == "" { + return fmt.Errorf("the vault holds no account of %s it made or was handed: an adopted hash is never retired here", user) + } + rec, held, err := v.Store.Read(a.Of) + if err != nil { + return err + } + if held { + rec.Retired = &RetiredMark{At: v.Now().UTC().Format(time.RFC3339), Why: why + ", by " + by} + if err := v.Store.Write(rec); err != nil { + return fmt.Errorf("its value could not be marked retired: %w", err) + } + } + if err := v.Accounts.Delete(user); err != nil { + return fmt.Errorf("its hash could not be taken out of %s: %w", BusAccountsState, err) + } + v.minting.mu.Lock() + v.minting.retiring.retired = append(v.minting.retiring.retired, user) + v.minting.mu.Unlock() + v.log("retired the bus account %s (%s): its hash is out of %s and its value kept, marked; %s", user, a.Of, + BusAccountsState, why) + return nil +} + +// RetireAccounts is the tool account_retire: a person retires accounts no longer wanted, past ADR 0230's bound, with why. +func (v *Vault) RetireAccounts(caller string, users []string, why string) (any, error) { + if strings.TrimSpace(why) == "" { + return nil, errors.New("account_retire says why (`why`)") + } + if len(users) == 0 { + return nil, errors.New("account_retire names the accounts (`users`), as vault_status lists them waiting") + } + // As the passes: never while the controller holds the lease (it forgets the accounts it mints), nor before the wanted + // accounts are read whole (an account not read yet is not known to be gone). + v.leaseAfresh() + if on, why := v.mintingOn(); !on { + return nil, fmt.Errorf("nothing was retired: %s", why) + } + v.minting.mu.Lock() + read := v.minting.retiring.wantsRead + v.minting.mu.Unlock() + if !read { + return nil, errors.New("nothing was retired: the wanted accounts are not read whole yet, so none is known to be gone") + } + for _, u := range users { + if _, wanted := v.wanted(u); wanted { + return nil, fmt.Errorf("%s is still wanted by the grants part: only an account no longer wanted is retired; nothing was retired", u) + } + } + var done []string + for _, u := range users { + if err := v.retireOne(u, strings.TrimSpace(why), someoneOr("", caller)); err != nil { + return map[string]any{"retired": done}, fmt.Errorf("%s: %w", u, err) + } + done = append(done, u) + } + return map[string]any{"retired": done}, nil +} diff --git a/modules/mesh-vault/cmd/mesh-vault/retire_test.go b/modules/mesh-vault/cmd/mesh-vault/retire_test.go new file mode 100644 index 00000000..2c4fe6ae --- /dev/null +++ b/modules/mesh-vault/cmd/mesh-vault/retire_test.go @@ -0,0 +1,89 @@ +package main + +import ( + "strings" + "testing" + "time" +) + +// 774/AC9 (novox/hq ADR 0561 §2, ADR 0230 §15, §18): a wanted account whose key is deleted is retired (its hash taken +// out of bus-accounts, its value kept, marked) only once the same set has stood for five passes and ten minutes, and never +// more than three, or more than half of those held, at once: past that it waits for a person (account_retire). Wanted +// again, it is re-enabled at once; and a retired account is not hashed again at a start. +func TestAWantGoneRetiresItsAccountOnlyWithinTheBounds(t *testing.T) { + v, accounts, _ := mintingVault(t, true) + clock := time.Date(2026, 10, 28, 9, 0, 0, 0, time.UTC) + v.Now = func() time.Time { return clock } + modules := []string{"m1", "m2", "m3", "m4", "m5", "m6", "m7", "m8"} + for _, m := range modules { + w := aWant(m) + v.WantHeard(w.User, wantRaw(t, w), false) + } + v.WantsRead() + if len(accounts.keys) != 8 { + t.Fatalf("minted %d", len(accounts.keys)) + } + pass := func(n int, every time.Duration) { + for i := 0; i < n; i++ { + clock = clock.Add(every) + v.RetirePass() + } + } + v.WantHeard("ace.m1", nil, true) + pass(5, time.Second) + if _, ok := accounts.keys["ace.m1"]; !ok { + t.Fatal("retired after five passes in five seconds: ADR 0230 waits ten minutes too") + } + pass(1, 10*time.Minute) + if _, ok := accounts.keys["ace.m1"]; ok { + t.Fatalf("not retired after five passes and ten minutes: %+v", v.RetireState()) + } + if rec, held, _ := v.Store.Read("own/ace/m1/broker"); !held || rec.Retired == nil { + t.Fatalf("the retired account's value is %+v (held %v): kept, and marked", rec, held) + } + if written, _ := v.HashHeld(); len(written) != 0 { + t.Errorf("a start hashed a retired account again: %+v", written) + } + // Wanted again: re-enabled at once. + w := aWant("m1") + v.WantHeard(w.User, wantRaw(t, w), false) + if rec, _, _ := v.Store.Read("own/ace/m1/broker"); rec.Retired != nil || accounts.keys["ace.m1"].Hash == "" { + t.Fatalf("wanted again, ace.m1 is %+v with hash %v", rec.Retired, accounts.keys["ace.m1"]) + } + // Four gone at once is over the bound: it waits for a person, who retires them by hand. + for _, m := range modules[4:] { + v.WantHeard("ace."+m, nil, true) + } + pass(6, 3*time.Minute) + st := v.RetireState() + if len(accounts.keys) != 8 || len(st.Waiting) != 4 || !strings.Contains(st.Why, "more than 3") { + t.Fatalf("over the bound: %d keys, %+v", len(accounts.keys), st) + } + if _, err := v.RetireAccounts("person.jochen", []string{"ace.m5"}, ""); err == nil { + t.Error("retired by hand with no why") + } + if _, err := v.RetireAccounts("person.jochen", []string{"ace.m2"}, "test"); err == nil { + t.Error("retired by hand an account still wanted") + } + // Never while the controller holds the lease, nor before the wants are read whole. + held := v.LeaseRead + v.LeaseRead = func() ([]byte, bool, error) { return []byte(`{"instance":"novox/1","epoch":57}`), true, nil } + if _, err := v.RetireAccounts("person.jochen", []string{"ace.m5"}, "the modules left"); err == nil || !strings.Contains(err.Error(), "controller holds the lease") { + t.Errorf("retired by hand under the controller's lease: %v", err) + } + v.LeaseRead = held + v.LeaseHeard([]byte(`{"handed-to":"the operator's hand-over"}`), true, nil) + v.minting.mu.Lock() + v.minting.retiring.wantsRead = false + v.minting.mu.Unlock() + if _, err := v.RetireAccounts("person.jochen", []string{"ace.m5"}, "the modules left"); err == nil || !strings.Contains(err.Error(), "not read whole") { + t.Errorf("retired by hand before the wants were read: %v", err) + } + v.WantsRead() + if _, err := v.RetireAccounts("person.jochen", []string{"ace.m5", "ace.m6"}, "the modules left"); err != nil { + t.Fatal(err) + } + if _, ok := accounts.keys["ace.m5"]; ok || len(accounts.keys) != 6 { + t.Errorf("after a person's retirement %d keys remain", len(accounts.keys)) + } +} diff --git a/modules/mesh-vault/cmd/mesh-vault/rotation.go b/modules/mesh-vault/cmd/mesh-vault/rotation.go new file mode 100644 index 00000000..31244f97 --- /dev/null +++ b/modules/mesh-vault/cmd/mesh-vault/rotation.go @@ -0,0 +1,94 @@ +package main + +// The record of a bus account's rotation, and its one rule (novox/hq ADR 0561 §4; issue 774, the lead's decision of +// 2026-10-27). A module's, a process's or a provisioner client's account rotates over two credentials (ADR 0114 §10): +// a second login, `--b`, carrying the first login's permissions verbatim, which the bus composer composes only while +// the account's key in `bus-accounts` names it in `rotating`. The vault starts at most one rotation per revision of the +// user list the bus loaded. What composes and delivers the second login is a subtask of its own, before the lease write; +// until it is built nothing begins a rotation, and `rotate` of a covered account stays refused, so nothing rotates in +// place. + +import ( + "fmt" + "strings" + "time" +) + +// SecondLoginSuffix makes an account's second login: reserved, so no module or process may be named to end in it. +const SecondLoginSuffix = "--b" + +// Rotating is a rotation under way, as the account's key in bus-accounts says it: the second login, the generation of +// the value it carries, since when, and the revision of the loaded user list it began at. +type Rotating struct { + Login string `json:"login"` + Generation int `json:"generation"` + Since time.Time `json:"since"` + Revision uint64 `json:"revision"` +} + +// LoadedListHeard keeps the revision of the user list the bus node says it loaded (its have of the bus module). +func (v *Vault) LoadedListHeard(revision uint64) { + v.minting.mu.Lock() + defer v.minting.mu.Unlock() + if revision > v.minting.loadedRevision { + v.minting.loadedRevision = revision + } +} + +// BeginRotation records a rotation of user's account at the loaded list's revision, under the one rule: none for a +// node's account or the controller's, none while minting is off, none of an account the vault did not make or was not +// handed, none twice for one account, and none when another began at this revision. +func (v *Vault) BeginRotation(user string) (Rotating, error) { + if nodeOrController(user) { + return Rotating{}, fmt.Errorf("%s does not rotate here: a node's account and the controller's keep the hash they have (ADR 0561 §4)", user) + } + if on, why := v.mintingOn(); !on { + return Rotating{}, fmt.Errorf("no rotation begins: %s", why) + } + v.minting.mu.Lock() + revision := v.minting.loadedRevision + v.minting.mu.Unlock() + if revision == 0 { + return Rotating{}, fmt.Errorf("no rotation begins: the revision of the user list the bus loaded is not known yet") + } + users, err := v.Accounts.Users() + if err != nil { + return Rotating{}, err + } + var mine *AccountHash + began := "" + for _, u := range users { + a, err := v.Accounts.Get(u) + if err != nil { + return Rotating{}, err + } + if a == nil { + continue + } + if a.Rotating != nil && a.Rotating.Revision == revision { + began = u + } + if u == user { + mine = a + } + } + switch { + case mine == nil || mine.Adopted || mine.Of == "": + return Rotating{}, fmt.Errorf("the vault holds no account of %s it made or was handed, so it rotates none", user) + case mine.Rotating != nil: + return Rotating{}, fmt.Errorf("%s is rotating already, since %s (second login %s)", user, mine.Rotating.Since.Format(time.RFC3339), mine.Rotating.Login) + case strings.HasSuffix(user, SecondLoginSuffix): + return Rotating{}, fmt.Errorf("%s is a second login, which never rotates of its own", user) + case began != "": + return Rotating{}, fmt.Errorf("no rotation of %s begins: %s's began at the loaded list's revision %d, and one "+ + "rotation begins per revision (ADR 0561 §4)", user, began, revision) + } + r := Rotating{Login: user + SecondLoginSuffix, Generation: mine.Generation + 1, Since: v.Now().UTC(), Revision: revision} + next := *mine + next.Rotating = &r + if err := v.Accounts.Put(next); err != nil { + return Rotating{}, err + } + v.log("a rotation of %s begins at the loaded list's revision %d: second login %s", user, revision, r.Login) + return r, nil +} diff --git a/modules/mesh-vault/cmd/mesh-vault/signing.go b/modules/mesh-vault/cmd/mesh-vault/signing.go new file mode 100644 index 00000000..7e05eb14 --- /dev/null +++ b/modules/mesh-vault/cmd/mesh-vault/signing.go @@ -0,0 +1,463 @@ +package main + +// The vault signs (novox/hq ADR 0550 §2; issue 774, design 47's L3 item d). It makes an ed25519 signing key and an x509 +// authority once, keeps each as its own secret with its operator copy (ADR 0113 §22), publishes their public halves in its +// state `signers`, and signs with them on its verb `sign`: a node's certificate for that node's node-engine alone, built +// from bus state (the name from the node's placement, the serving key from its record), never from bytes the caller chose. +// Each signature is said as `signed` and kept in the record of hand-outs, never a key. The controller's key is never +// copied here. + +import ( + "crypto/ed25519" + "crypto/rand" + "crypto/x509" + "encoding/base64" + "encoding/json" + "encoding/pem" + "errors" + "fmt" + "strings" + "time" + + stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/overlay" +) + +// The vault's own signing secrets: each an ed25519 seed in base64 (never PEM, which the vault takes as a private key +// someone sent it and refuses). +const ( + SigningKeySecret = "signing/key" + AuthorityKeySecret = "signing/authority-key" + // signedOutcome marks a signature in the record of hand-outs. + signedOutcome = "signed" +) + +// signingSecret says name is one of the vault's signing secrets, which no asker is ever handed. +func signingSecret(name string) bool { return name == SigningKeySecret || name == AuthorityKeySecret } + +// SignersStore is the vault's state `signers` (delivery.SignersBucket): a key's value, or nil; and a write. +type SignersStore interface { + Get(key string) ([]byte, error) + Put(key string, value []byte) error +} + +// busSigners is SignersStore over the vault's own state through its runtime. +type busSigners struct{} + +func (busSigners) Get(key string) ([]byte, error) { + e, err := stdio.State(delivery.SignersState).Get(key) + if err != nil || e == nil { + return nil, err + } + return e.Value, nil +} + +func (busSigners) Put(key string, value []byte) error { + _, err := stdio.State(delivery.SignersState).Put(key, json.RawMessage(value)) + return err +} + +// signingKeys are the vault's two keys, opened, and the authority's certificate. +type signingKeys struct { + key ed25519.PrivateKey + keyMade time.Time + authority ed25519.PrivateKey + authorityMade time.Time + certificate string + parsed *x509.Certificate +} + +// EnsureSigning makes the signing key and the authority once, when the store holds neither, and answers whether it made +// them. **Never made again over what is held** (as OwnKey refuses a new key beside secrets): one held without the other, +// or none held while `signers` already publishes a key, is a lost key, refused loudly with nothing made, because every +// node trusts what was published and a new key would be one nobody trusts. +func (v *Vault) EnsureSigning() (bool, error) { + _, keyHeld, err := v.Store.Read(SigningKeySecret) + if err != nil { + return false, err + } + _, authHeld, err := v.Store.Read(AuthorityKeySecret) + if err != nil { + return false, err + } + switch { + case keyHeld && authHeld: + return false, nil + case keyHeld != authHeld: + missing := AuthorityKeySecret + if authHeld { + missing = SigningKeySecret + } + return false, fmt.Errorf("THE VAULT'S %s IS MISSING BESIDE THE OTHER SIGNING SECRET: nothing is made, since a new one "+ + "would be trusted by no node; put it back from its operator copy (novox/hq ADR 0113 §21, ADR 0550 §2)", missing) + } + if v.Signers != nil { + if raw, err := v.Signers.Get(delivery.SignersKeyKey); err != nil { + return false, fmt.Errorf("the vault's state %s could not be read, so no signing key is made: %w", delivery.SignersRead, err) + } else if raw != nil { + return false, fmt.Errorf("THE VAULT'S SIGNING KEYS ARE MISSING WHILE %s PUBLISHES ONE: nothing is made, since every "+ + "node trusts the published key; put them back from their operator copies (novox/hq ADR 0113 §21)", delivery.SignersRead) + } + } + for _, name := range []string{SigningKeySecret, AuthorityKeySecret} { + _, priv, err := ed25519.GenerateKey(rand.Reader) + if err != nil { + return false, err + } + if _, err := v.keepWith(name, base64.StdEncoding.EncodeToString(priv.Seed()), "the vault", "", OriginMade, + keepOptions{neverReplace: true, quiet: true}); err != nil { + return false, fmt.Errorf("the vault's %s is not made: %w", name, err) + } + } + v.log("made the vault's signing key and authority, each kept with its operator copy (ADR 0550 §2)") + return true, nil +} + +// openSeed is a signing secret opened: its private key and when it was made. +func (v *Vault) openSeed(name string) (ed25519.PrivateKey, time.Time, error) { + r, held, err := v.Store.Read(name) + if err != nil { + return nil, time.Time{}, err + } + if !held { + return nil, time.Time{}, fmt.Errorf("the vault holds no %s yet: it is made once the operator key is taken", name) + } + value, err := openAtRest(r, v) + if err != nil { + return nil, time.Time{}, fmt.Errorf("%s does not open with the vault's key: %w", name, err) + } + seed, err := base64.StdEncoding.DecodeString(value) + if err != nil || len(seed) != ed25519.SeedSize { + return nil, time.Time{}, fmt.Errorf("%s is not an ed25519 seed", name) + } + return ed25519.NewKeyFromSeed(seed), r.MadeAt, nil +} + +// keys opens both signing secrets and derives the authority's certificate from its key and when it was made, so it is the +// same certificate at every start. +func (v *Vault) keys() (signingKeys, error) { + key, keyMade, err := v.openSeed(SigningKeySecret) + if err != nil { + return signingKeys{}, err + } + auth, authMade, err := v.openSeed(AuthorityKeySecret) + if err != nil { + return signingKeys{}, err + } + pub := auth.Public().(ed25519.PublicKey) + tmpl := delivery.AuthorityTemplate(pub, authMade) + der, err := x509.CreateCertificate(rand.Reader, tmpl, tmpl, pub, auth) + if err != nil { + return signingKeys{}, fmt.Errorf("the vault's authority certificate could not be made: %w", err) + } + parsed, err := x509.ParseCertificate(der) + if err != nil { + return signingKeys{}, err + } + return signingKeys{key: key, keyMade: keyMade, authority: auth, authorityMade: authMade, parsed: parsed, + certificate: string(pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}))}, nil +} + +// SignersValues are the two keys of `signers` as the vault holds them now. +func (v *Vault) SignersValues() (delivery.SignerKey, delivery.SignerAuthority, error) { + k, err := v.keys() + if err != nil { + return delivery.SignerKey{}, delivery.SignerAuthority{}, err + } + fp, err := delivery.AuthorityFingerprint(k.certificate) + if err != nil { + return delivery.SignerKey{}, delivery.SignerAuthority{}, err + } + return delivery.NewSignerKey(k.key.Public().(ed25519.PublicKey), k.keyMade), + delivery.SignerAuthority{Certificate: k.certificate, Fingerprint: fp, Made: k.authorityMade.UTC()}, nil +} + +// PublishSigners writes `signers` from the keys the vault holds, each key only when its value differs (or always with +// force, signers_reassert), and answers which keys it wrote. +func (v *Vault) PublishSigners(force bool) ([]string, error) { + if v.Signers == nil { + return nil, errors.New("the vault has no state " + delivery.SignersRead + " to write") + } + key, auth, err := v.SignersValues() + if err != nil { + return nil, err + } + var written []string + for name, value := range map[string]any{delivery.SignersKeyKey: key, delivery.SignersAuthorityKey: auth} { + raw, err := json.Marshal(value) + if err != nil { + return written, err + } + if !force { + held, err := v.Signers.Get(name) + if err != nil { + return written, fmt.Errorf("%s %s could not be read: %w", delivery.SignersRead, name, err) + } + if same(held, raw) { + continue + } + } + if err := v.Signers.Put(name, raw); err != nil { + return written, fmt.Errorf("%s %s could not be written: %w", delivery.SignersRead, name, err) + } + written = append(written, name) + } + if len(written) > 0 { + v.log("wrote %s %s (ADR 0550 §2)", delivery.SignersRead, strings.Join(written, " and ")) + } + return written, nil +} + +// same says two JSON values are the same value. +func same(a, b []byte) bool { + if a == nil || b == nil { + return false + } + var x, y any + if json.Unmarshal(a, &x) != nil || json.Unmarshal(b, &y) != nil { + return false + } + ra, _ := json.Marshal(x) + rb, _ := json.Marshal(y) + return string(ra) == string(rb) +} + +// SigningPass makes the keys once and publishes `signers` when it differs: each minute pass, after the operator key. +func (v *Vault) SigningPass() error { + if _, err := v.EnsureSigning(); err != nil { + return err + } + if v.Signers == nil { + return nil + } + _, err := v.PublishSigners(false) + return err +} + +// SignArgs are `sign`'s arguments: the kind, and the kind's fields. +type SignArgs struct { + Kind string + Node string + Reply string +} + +// Sign answers caller's `sign`. The caller is the one the runtime proved; who may ask which kind is the vault's rule, +// never an argument: a node certificate for a machine's node-engine alone, about its own machine; an enrolment for no +// caller until adoption's record names one (ADR 0561 §5); the authority for the vault itself, never asked. +func (v *Vault) Sign(caller string, a SignArgs) (delivery.SignAnswer, error) { + if caller == "" || caller == stdio.Unproven || !busUser.MatchString(caller) { + return delivery.SignAnswer{}, errors.New("the vault signs only for a caller the runtime proved") + } + switch a.Kind { + case delivery.KindNodeCertificate: + return v.signNodeCertificate(caller, a.Node) + case delivery.KindEnrolment: + return delivery.SignAnswer{}, fmt.Errorf("%s may not ask for an enrolment's signature: no caller is named for it until "+ + "adoption's record names the seat that adopts (novox/hq ADR 0561 §5)", caller) + case delivery.KindAuthority: + return delivery.SignAnswer{}, fmt.Errorf("the authority is the vault's own, signed for itself; %s may not ask it", caller) + } + return delivery.SignAnswer{}, fmt.Errorf("%q is not a kind the vault signs (%s)", a.Kind, strings.Join(delivery.SignKinds, ", ")) +} + +// signNodeCertificate signs the certificate of the calling machine's internal name for the serving key its record states. +func (v *Vault) signNodeCertificate(caller, asked string) (delivery.SignAnswer, error) { + node, ok := nodeEngineOf(caller) + if !ok { + return delivery.SignAnswer{}, fmt.Errorf("a node certificate is signed for a machine's node-engine alone, about its own "+ + "machine; %s is not a node-engine", caller) + } + if asked != "" && asked != node { + return delivery.SignAnswer{}, fmt.Errorf("%s asks for the certificate of %s, and is %s's node-engine: a machine asks for its own alone", caller, asked, node) + } + name, err := v.internalName(node) + if err != nil { + return delivery.SignAnswer{}, err + } + serving, err := v.servingKey(node) + if err != nil { + return delivery.SignAnswer{}, err + } + k, err := v.keys() + if err != nil { + return delivery.SignAnswer{}, err + } + der, err := x509.CreateCertificate(rand.Reader, delivery.NodeCertificateTemplate(name, serving, k.authorityMade), + k.parsed, serving, k.authority) + if err != nil { + return delivery.SignAnswer{}, fmt.Errorf("the certificate of %s could not be signed: %w", name, err) + } + fp, _ := delivery.AuthorityFingerprint(k.certificate) + answer := delivery.SignAnswer{Kind: delivery.KindNodeCertificate, Node: node, Authority: k.certificate, + Certificate: string(pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der})), Signer: fp, Digest: delivery.DigestOf(der)} + if err := v.saySigned(caller, AuthorityKeySecret, answer); err != nil { + return delivery.SignAnswer{}, err + } + return answer, nil +} + +// internalName is node's name inside the mesh: only a node with an address on the private network in its placement has +// one (a certificate for a name nothing resolves is one nothing can check). +func (v *Vault) internalName(node string) (string, error) { + if v.Placement.Read == nil { + return "", errors.New("the vault reads no placement, so it cannot tell a node's internal name") + } + key, err := delivery.PlacementKey(node) + if err != nil { + return "", err + } + raw, err := v.Placement.Read(key) + if err != nil { + return "", fmt.Errorf(notYetPrefix+"the placement of %s could not be read: %w", node, err) + } + if raw == nil { + return "", fmt.Errorf("%s has no placement of the seat mesh-placement, so it has no name inside the mesh", node) + } + p, err := delivery.DecodePlacement(raw) + if err != nil { + return "", fmt.Errorf("the placement of %s is not readable: %w", node, err) + } + if p.Decided.Address == "" { + return "", fmt.Errorf("%s is not on the private network (its placement names no address), so it has no name inside the mesh to be certified for", node) + } + return overlay.InternalName(node), nil +} + +// servingKey is the serving key node's record states, `.node` of the have bucket, which that node alone writes. +func (v *Vault) servingKey(node string) (ed25519.PublicKey, error) { + if v.Haves == nil { + return nil, errors.New("the vault cannot read the have bucket (" + HaveRead + "), so it cannot read a node's record") + } + key, err := delivery.HaveNodeKey(node) + if err != nil { + return nil, err + } + raw, err := v.Haves(key) + if err != nil { + return nil, fmt.Errorf(notYetPrefix+"%s %s could not be read: %w", HaveRead, key, err) + } + if raw == nil { + return nil, fmt.Errorf("%s has written no record (%s %s), so it states no serving key", node, HaveRead, key) + } + h, err := delivery.DecodeHaveNode(raw) + if err != nil { + return nil, fmt.Errorf("the record of %s is not readable: %w", node, err) + } + if h.Record == nil || h.Record.ServingKey == "" { + return nil, fmt.Errorf("%s's record states no serving key, so there is no key to certify", node) + } + pub, err := base64.StdEncoding.DecodeString(h.Record.ServingKey) + if err != nil || len(pub) != ed25519.PublicKeySize { + return nil, fmt.Errorf("%s's serving key is not an ed25519 public key in base64", node) + } + return ed25519.PublicKey(pub), nil +} + +// saySigned keeps the signature in the record of hand-outs and says `signed`: who, which kind, which node, the digest and +// the signer's fingerprint. Never a key. A signature whose record cannot be written is not answered. +func (v *Vault) saySigned(caller, with string, a delivery.SignAnswer) error { + at := v.Now().UTC() + if err := v.recordHanding(Handing{To: caller, Name: with, Outcome: signedOutcome, + Why: a.Kind + " " + a.Node + " " + a.Digest}); err != nil { + return fmt.Errorf("the signature is not answered: its record could not be written: %w", err) + } + if v.Emit == nil { + return nil + } + raw, _ := json.Marshal(delivery.Signed{Caller: caller, Kind: a.Kind, Node: a.Node, Digest: a.Digest, Signer: a.Signer, At: at}) + var body map[string]any + _ = json.Unmarshal(raw, &body) + if err := v.Emit(delivery.SignedEvent, body); err != nil { + v.log("signed %s for %s, but %s could not be said: %v", a.Kind, caller, delivery.SignedEvent, err) + } + return nil +} + +// SignedRow is one signature as vault_signed lists it. +type SignedRow struct { + At time.Time `json:"at"` + Caller string `json:"caller"` + With string `json:"with"` + Kind string `json:"kind"` + Node string `json:"node,omitempty"` + Digest string `json:"digest"` +} + +// Signatures are the last signatures the vault made, newest first, at most limit. +func (v *Vault) Signatures(limit int) ([]SignedRow, error) { + all, err := v.Store.Handed(10000) + if err != nil { + return nil, err + } + out := []SignedRow{} + for _, h := range all { + if h.Outcome != signedOutcome || len(out) >= limit { + continue + } + f := strings.Fields(h.Why) + row := SignedRow{At: h.At, Caller: h.To, With: h.Name} + if len(f) == 3 { + row.Kind, row.Node, row.Digest = f[0], f[1], f[2] + } + out = append(out, row) + } + return out, nil +} + +// SigningState is what vault_status and vault_signers say of the signing secrets: whether each is held, its operator +// copy, and the fingerprints held against those published. +func (v *Vault) SigningState() map[string]any { + out := map[string]any{} + for _, name := range []string{SigningKeySecret, AuthorityKeySecret} { + r, held, err := v.Store.Read(name) + row := map[string]any{"held": held} + if err != nil { + row["error"] = err.Error() + } + if held { + row["generation"], row["made"], row["operator_copy"] = r.Generation, r.MadeAt, r.Operator.Sealed != "" + } + out[name] = row + } + key, auth, err := v.SignersValues() + if err != nil { + out["error"] = err.Error() + return out + } + out["key_fingerprint"], out["authority_fingerprint"] = key.Fingerprint, auth.Fingerprint + if v.Signers == nil { + return out + } + published := map[string]any{} + match := true + for name, want := range map[string]string{delivery.SignersKeyKey: key.Fingerprint, delivery.SignersAuthorityKey: auth.Fingerprint} { + raw, err := v.Signers.Get(name) + var got struct { + Fingerprint string `json:"fingerprint"` + } + if err != nil || raw == nil || json.Unmarshal(raw, &got) != nil { + published[name], match = nil, false + continue + } + published[name] = got.Fingerprint + match = match && got.Fingerprint == want + } + out["published"], out["published_matches"] = published, match + return out +} + +// signingSaid says why the signing pass did not finish, once, and again when it changes. +func (v *Vault) signingSaid(err error) { + line := "" + if err != nil { + line = err.Error() + } + v.mu.Lock() + changed := line != v.signingWhy + v.signingWhy = line + v.mu.Unlock() + if changed && line != "" { + v.log("the vault signs nothing yet: %s", line) + } +} diff --git a/modules/mesh-vault/cmd/mesh-vault/signing_test.go b/modules/mesh-vault/cmd/mesh-vault/signing_test.go new file mode 100644 index 00000000..68ef9f6d --- /dev/null +++ b/modules/mesh-vault/cmd/mesh-vault/signing_test.go @@ -0,0 +1,196 @@ +package main + +import ( + "crypto/ed25519" + "crypto/rand" + "encoding/base64" + "encoding/json" + "strings" + "testing" + + "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/overlay" + "git.novox.be/novox/mesh-sdk/go/seal" +) + +// signersState is the vault's state `signers` in memory, counting writes. +type signersState struct { + keys map[string][]byte + writes int +} + +func (s *signersState) Get(key string) ([]byte, error) { return s.keys[key], nil } +func (s *signersState) Put(key string, value []byte) error { + s.keys[key] = append([]byte(nil), value...) + s.writes++ + return nil +} + +// L3 s2 to s5 (novox/hq ADR 0550 §2): the vault makes its signing key and authority once, with their operator copies and +// no other holder; publishes their public halves only when they differ; signs a node's certificate for that node's +// node-engine alone, from its placement's name and its record's serving key; and no answer, event or record carries a +// key. +func TestTheVaultSigns(t *testing.T) { + v := newVault(t, nil) + state := &signersState{keys: map[string][]byte{}} + v.Signers = state + servingPub, _, _ := ed25519.GenerateKey(rand.Reader) + placements := map[string]delivery.Placement{ + "ace": {Node: "ace", Decided: delivery.NodeDecided{Address: "10.77.0.3"}}, + "g14": {Node: "g14", Decided: delivery.NodeDecided{Address: "10.77.0.4"}}, + "plain": {Node: "plain"}, + } + v.Placement = PlacementReader{Read: func(key string) ([]byte, error) { + node, err := delivery.ParsePlacementKey(key) + if err != nil { + return nil, err + } + p, ok := placements[node] + if !ok { + return nil, nil + } + return delivery.EncodePlacement(p) + }} + records := map[string]string{"ace": base64.StdEncoding.EncodeToString(servingPub), "plain": base64.StdEncoding.EncodeToString(servingPub)} + v.Haves = func(key string) ([]byte, error) { + node, _, _ := strings.Cut(key, ".") + serving, ok := records[node] + if !ok { + return nil, nil + } + return delivery.EncodeHaveNode(delivery.HaveNode{Record: &delivery.NodeRecord{Node: node, ServingKey: serving}}) + } + + // s2: made once, kept with their operator copies; a second pass makes nothing new and writes nothing again (s5). + if err := v.SigningPass(); err != nil { + t.Fatal(err) + } + var seeds []string + for _, name := range []string{SigningKeySecret, AuthorityKeySecret} { + r, held, err := v.Store.Read(name) + if err != nil || !held || r.Generation != 1 { + t.Fatalf("%s: held %v at generation %d, %v", name, held, r.Generation, err) + } + value, _ := seal.Open(r.AtRest, v.Own.PrivateKey) + if got := openOperator(t, v.operatorPublic, v.operatorPrivate, r.Operator.Sealed); got != value { + t.Errorf("%s: the operator copy does not open to the value held", name) + } + seeds = append(seeds, value) + } + key1, auth1, err := v.SignersValues() + if err != nil { + t.Fatal(err) + } + writes := state.writes + if writes != 2 { + t.Errorf("signers written %d times at the first pass, want its two keys", writes) + } + if err := v.SigningPass(); err != nil { + t.Fatal(err) + } + key2, auth2, _ := v.SignersValues() + if key2 != key1 || auth2 != auth1 || state.writes != writes { + t.Errorf("a second pass changed the keys or wrote signers again (%d writes)", state.writes) + } + if _, err := delivery.DecodeSignerKey(state.keys[delivery.SignersKeyKey]); err != nil { + t.Errorf("signers' key: %v", err) + } + authority, err := delivery.DecodeSignerAuthority(state.keys[delivery.SignersAuthorityKey]) + if err != nil { + t.Fatalf("signers' authority: %v", err) + } + // No other holder: `get` hands neither, even to an asker a rule names for it. + asker, _ := seal.GenerateKeyPair() + v.Askers = AllowList{"node.ace": {SigningKeySecret, AuthorityKeySecret}} + for _, name := range []string{SigningKeySecret, AuthorityKeySecret} { + if _, err := v.Get("node.ace", name, asker.PublicKey); err == nil { + t.Errorf("get handed %s", name) + } + } + v.Askers = nil + if forced, err := v.PublishSigners(true); err != nil || len(forced) != 2 { + t.Errorf("signers_reassert wrote %v, %v", forced, err) + } + + // s3, s4: who may ask which kind, and the certificate built from bus state. + for _, row := range []struct { + name, caller, kind, node, refused string + }{ + {"a module is not a node-engine", "ace.mesh-grants", delivery.KindNodeCertificate, "", "not a node-engine"}, + {"another node's certificate", "node.g14", delivery.KindNodeCertificate, "ace", "its own alone"}, + {"a node not on the private network", "node.plain", delivery.KindNodeCertificate, "", "not on the private network"}, + {"a node that states no serving key", "node.g14", delivery.KindNodeCertificate, "", "no record"}, + {"an enrolment has no caller yet", "node.ace", delivery.KindEnrolment, "", "ADR 0561 §5"}, + {"the authority is the vault's own", "node.ace", delivery.KindAuthority, "", "the vault's own"}, + {"an unproven caller", "", delivery.KindNodeCertificate, "", "proved"}, + {"the node's own certificate", "node.ace", delivery.KindNodeCertificate, "ace", ""}, + } { + v.bus.raw = nil + got, err := v.Sign(row.caller, SignArgs{Kind: row.kind, Node: row.node}) + if row.refused != "" { + if err == nil || !strings.Contains(err.Error(), row.refused) { + t.Errorf("%s: %v, want refused saying %q", row.name, err, row.refused) + } + if len(v.bus.raw) != 0 { + t.Errorf("%s: refused, and said %v", row.name, v.bus.raw) + } + continue + } + if err != nil { + t.Fatalf("%s: %v", row.name, err) + } + if err := delivery.CheckNodeCertificate(got.Certificate, []string{authority.Certificate}, overlay.InternalName("ace"), + servingPub, authority.Made.Add(1)); err != nil { + t.Errorf("%s: the certificate does not check against signers' authority: %v", row.name, err) + } + if got.Signer != authority.Fingerprint || got.Digest == "" { + t.Errorf("%s: signer %s, digest %q", row.name, got.Signer, got.Digest) + } + // Said once as signed, kept in the record, and no key in any of it. + answer, _ := json.Marshal(got) + said := strings.Join(v.bus.raw, "\n") + if !strings.Contains(said, `"event":"signed"`) || !strings.Contains(said, got.Digest) { + t.Errorf("%s: signed not said: %s", row.name, said) + } + rows, _ := v.Signatures(10) + if len(rows) != 1 || rows[0].Caller != "node.ace" || rows[0].Kind != delivery.KindNodeCertificate || rows[0].Digest != got.Digest { + t.Errorf("%s: vault_signed lists %+v", row.name, rows) + } + status, _ := json.Marshal(v.SigningState()) + for _, seed := range seeds { + raw, _ := base64.StdEncoding.DecodeString(seed) + priv := ed25519.NewKeyFromSeed(raw) + for _, text := range []string{string(answer), said, string(status), string(state.keys[delivery.SignersKeyKey]), + string(state.keys[delivery.SignersAuthorityKey])} { + if strings.Contains(text, seed) || strings.Contains(text, base64.StdEncoding.EncodeToString(priv)) { + t.Errorf("%s: a signing key's private half is in what the vault says: %s", row.name, text) + } + } + } + } + // A serving key changed in the record is the one certified next, never the one before. + other, _, _ := ed25519.GenerateKey(rand.Reader) + records["ace"] = base64.StdEncoding.EncodeToString(other) + got, err := v.Sign("node.ace", SignArgs{Kind: delivery.KindNodeCertificate}) + if err != nil || delivery.CheckNodeCertificate(got.Certificate, []string{authority.Certificate}, overlay.InternalName("ace"), + servingPub, authority.Made.Add(1)) == nil { + t.Errorf("a certificate for a serving key the record no longer states: %v", err) + } + + // s2: one secret lost beside the other, or both lost while signers publishes a key, makes nothing. + lost := newVault(t, nil) + lost.Signers = &signersState{keys: map[string][]byte{delivery.SignersKeyKey: state.keys[delivery.SignersKeyKey]}} + if made, err := lost.EnsureSigning(); made || err == nil || !strings.Contains(err.Error(), "MISSING") { + t.Errorf("keys made while signers publishes one: %v, %v", made, err) + } + half := newVault(t, nil) + if _, err := half.Put(SigningKeySecret, seeds[0], "the test"); err != nil { + t.Fatal(err) + } + if made, err := half.EnsureSigning(); made || err == nil || !strings.Contains(err.Error(), "MISSING") { + t.Errorf("an authority made beside a held signing key: %v, %v", made, err) + } + if _, held, _ := half.Store.Read(AuthorityKeySecret); held { + t.Error("the authority was made all the same") + } +} diff --git a/modules/mesh-vault/cmd/mesh-vault/store.go b/modules/mesh-vault/cmd/mesh-vault/store.go index 430f2ea2..8ede49d2 100644 --- a/modules/mesh-vault/cmd/mesh-vault/store.go +++ b/modules/mesh-vault/cmd/mesh-vault/store.go @@ -52,6 +52,8 @@ type Record struct { // (move-in with replace_running, owntake.go): kept sealed as it was, so the replacement can be read and undone. Any later // change of the secret drops it. Previous *PreviousValue `json:"previous,omitempty"` + // Retired marks a bus account no longer wanted (retire.go, ADR 0561 §2): its value kept, its hash taken out. + Retired *RetiredMark `json:"retired,omitempty"` } // PreviousValue is a replaced value as the store kept it: its generation, both its seals, and who changed it when. diff --git a/modules/mesh-vault/cmd/mesh-vault/tools.go b/modules/mesh-vault/cmd/mesh-vault/tools.go index 62709e5b..416e63cf 100644 --- a/modules/mesh-vault/cmd/mesh-vault/tools.go +++ b/modules/mesh-vault/cmd/mesh-vault/tools.go @@ -122,7 +122,8 @@ func definitions(env *Env) []toolDef { }}, {Name: Seat + ".secret-ask", Description: "Ask the operator for a module's own secret at a desk (ADR 0349 clause 9, ADR 0277 §22–§27), " + - "asked by the controller alone: node, module, name, desk and asked_by. The ask is recorded before any " + + "asked by the controller (naming asked_by) or by a caller the vault's manifest names, itself named by the " + + "bus (ADR 0563 §8): node, module, name and desk. The ask is recorded before any " + "prompt opens — one open ask per secret and three an hour — and the vault asks the desk's prompt itself, " + "in its own name, with a key made for the call; what is typed opens in the vault alone. Answers how the " + "prompt ended (given, dismissed, timed-out, empty, refused or failed) and who asked as recorded; never " + @@ -130,7 +131,7 @@ func definitions(env *Env) []toolDef { Input: map[string]any{"type": "object", "properties": map[string]any{ "node": str("the machine the secret is for"), "module": str("the module"), "name": str("the secret's name in the module"), "desk": str("the machine whose prompt asks"), - "asked_by": str("who asked the controller, as the bus named its caller")}}, + "asked_by": str("from the controller alone: who asked it, as the bus named its caller")}}, run: func(caller string, a map[string]any) (any, error) { v, err := env.ready() if err != nil { @@ -170,7 +171,7 @@ func definitions(env *Env) []toolDef { Description: "Make a secret anew (ADR 0114, ADR 0228): node, module and name for a module's own, or provider " + "and provision for a provision's credential. Refused for a value a provider's software issued, which " + "the software rotates, and for an accepted credential, which the vault does not remake. Never answers " + - "the value. From the controller alone, and with a why.", + "the value. For a caller the runtime proves, its manifest granted the verb, and with a why.", Input: map[string]any{"type": "object", "properties": map[string]any{ "node": str("the machine, for a module's own secret"), "module": str("the module"), "name": str("the secret's name in the module"), "provider": str("the provider's machine"), @@ -303,6 +304,20 @@ func definitions(env *Env) []toolDef { // clause 9, adopt.go): a public half, never the private one. return map[string]any{"recorded": true, "vault_key": v.Own.PublicKey}, nil }}, + {Name: Seat + ".issue-account", + Description: "Mint one bus account the grants part wants, now (novox/hq ADR 0561 §9): the recovery when the vault did not " + + "hear a want. Reads the wanted key afresh from mesh-grants.accounts and mints it as a want would: only while " + + "the controller's lease is handed away, never for a node's account or the controller's, never a user nobody " + + "wants, and never anew for one whose value the vault holds or whose hash it adopted (it says so). Answers " + + "whether it minted, the secret and generation, never the value or the hash.", + Input: map[string]any{"user": str("the bus user, e.g. ace.mesh-build-go")}, + run: func(caller string, a map[string]any) (any, error) { + v, err := env.ready() + if err != nil { + return nil, err + } + return v.IssueAccount(caller, arg(a, "user")) + }}, {Name: Seat + ".own", Description: "The own secrets of this machine's modules, each sealed to the machine's sealing key (novox/hq ADR 0547 §2, " + "ADR 0550 §5, issue 1126): from the node-engine of the machine alone, about its own machine (the bus proves which), " + @@ -476,6 +491,28 @@ func definitions(env *Env) []toolDef { } return env.Vault.HashUser(arg(a, "user"), caller) }}, + {Name: "account_retire", + Description: "Retire bus accounts the grants part no longer wants, by a person's act with why (novox/hq ADR 0561 §2, " + + "ADR 0230 §18): the vault takes each one's hash out of its state `bus-accounts`, so the grants part's next user " + + "list drops the user, and keeps its value marked retired. For the set vault_status lists as waiting for a " + + "person (more than three, or more than half, gone at once); refused for an account still wanted, an adopted " + + "hash, and with no why. Wanted again, an account is re-enabled at once.", + Input: map[string]any{"users": map[string]any{"type": "array", "items": str("a bus user")}, + "why": str("why they are retired")}, + run: func(caller string, a map[string]any) (any, error) { + if env.Vault == nil { + return nil, errors.New("the vault is not ready") + } + var users []string + if list, ok := a["users"].([]any); ok { + for _, u := range list { + if s, ok := u.(string); ok { + users = append(users, strings.TrimSpace(s)) + } + } + } + return env.Vault.RetireAccounts(caller, users, arg(a, "why")) + }}, {Name: "secret_vault_export", Description: "Every secret the vault itself holds, as its copy sealed to the operator key, in the shape of " + "`secret export`: saved to a file, `nox secret recover vault mesh-vault --key " + @@ -537,6 +574,57 @@ func definitions(env *Env) []toolDef { } return map[string]any{"deleted": consumer, "freed_bytes": -1}, nil }}, + {Name: Seat + ".sign", + Description: "Sign with the vault's key or authority (novox/hq ADR 0550 §2). Kind node-certificate: the certificate " + + "of the calling machine's internal name for the serving key its record states, built from bus state alone and " + + "asked by that machine's node-engine only (the bus proves which); kind enrolment: no caller is named until " + + "adoption's record (ADR 0561 §5). Answers the certificate, the authority, the signer's fingerprint and the " + + "digest, and says signed. Never a key.", + Input: map[string]any{"kind": str("node-certificate or enrolment"), + "node": str("the machine, for a node certificate: the caller's own (left out, the caller's)"), + "reply": str("the enrolment reply's bytes in base64, for an enrolment")}, + run: func(caller string, a map[string]any) (any, error) { + v, err := env.ready() + if err != nil { + return nil, err + } + return v.Sign(caller, SignArgs{Kind: arg(a, "kind"), Node: arg(a, "node"), Reply: arg(a, "reply")}) + }}, + {Name: "vault_signers", + Description: "The vault's signing key and authority (novox/hq ADR 0550 §2): whether each is held with its operator " + + "copy, the fingerprints the vault holds, what its state signers publishes, and whether they match. Never a key.", + run: func(string, map[string]any) (any, error) { + v, err := env.ready() + if err != nil { + return nil, err + } + return v.SigningState(), nil + }}, + {Name: "vault_signed", + Description: "The last signatures the vault made, newest first: when, the caller the bus proved, the kind, the node " + + "and the digest of what was signed. Never a key.", + Input: map[string]any{"limit": map[string]any{"type": "number", "description": "how many (default 50)"}}, + run: func(_ string, a map[string]any) (any, error) { + v, err := env.ready() + if err != nil { + return nil, err + } + return v.Signatures(limitOf(a)) + }}, + {Name: "signers_reassert", + Description: "Write the vault's state signers again from the keys it holds, whether or not it differs (novox/hq " + + "ADR 0550 §2): the by-hand path for a state lost or doubted. Makes no key.", + run: func(string, map[string]any) (any, error) { + v, err := env.ready() + if err != nil { + return nil, err + } + written, err := v.PublishSigners(true) + if err != nil { + return nil, err + } + return map[string]any{"written": written}, nil + }}, {Name: "vault_health", Description: "The vault's runtime health, as the node-engine asks it (novox/hq ADR 0366): unhealthy once " + "re-sealing the operator copies has failed on three passes in a row, saying what failed and why; " + @@ -544,7 +632,9 @@ func definitions(env *Env) []toolDef { run: func(string, map[string]any) (any, error) { return env.health(), nil }}, {Name: "vault_status", Description: "What needs a person's eye in the vault: operator copies still sealed to an earlier key, the operator " + - "key the mesh's export holds if the vault refused it, and how often the record of hand-outs was rolled.", + "key the mesh's export holds if the vault refused it, how often the record of hand-outs was rolled, and " + + "minting (novox/hq ADR 0561 §3, §9): whether the vault mints bus accounts now and why (only while the " + + "controller's lease is handed away), every wanted account it holds back with why, and those it minted.", run: func(string, map[string]any) (any, error) { if env.Vault == nil { return map[string]any{"ready": false, "error": fmt.Sprint(env.Broken)}, nil @@ -556,7 +646,7 @@ func definitions(env *Env) []toolDef { key, _ := env.Vault.Store.OperatorKey() return map[string]any{"ready": true, "operator_key": key, "copies_on_an_earlier_key": left, "operator_key_refused": env.keyRefused(), "handed_rolled": env.Vault.Store.Rolled(), - "ok": left == 0 && env.keyRefused() == "" && key != ""}, nil + "minting": env.Vault.MintState(), "retiring": env.Vault.RetireState(), "signing": env.Vault.SigningState(), "ok": left == 0 && env.keyRefused() == "" && key != ""}, nil }}, } } diff --git a/modules/mesh-vault/cmd/mesh-vault/vault.go b/modules/mesh-vault/cmd/mesh-vault/vault.go index 1d93f4bb..9b5634ad 100644 --- a/modules/mesh-vault/cmd/mesh-vault/vault.go +++ b/modules/mesh-vault/cmd/mesh-vault/vault.go @@ -35,13 +35,20 @@ type Vault struct { // Assignments is the controller's assignments mirror, where the vault sees a consumer bind a provider and // unbind (pair.go). Empty sees nothing, and makes no provision's credential. Assignments Mirror - Emit func(event string, body map[string]any) error + // Bound is the vault's own state of outcome entries, one per credential it provides, and its read of the seat's + // provisions (outcomes.go, novox/hq ADR 0570 clauses 3, 6). + Bound Bound + Emit func(event string, body map[string]any) error // Ask calls a tool on the mesh as the vault, through the runtime that launched it, under the vault's own bus // account (ADR 0259 §8's runs-as, issue 723): the desks' secret prompt and each issuing provider's tool, which // the bus lets the vault's account alone call in its own name. Nil asks nobody. Ask func(key string, body any) (json.RawMessage, error) - Now func() time.Time - Log func(format string, args ...any) + // AskCallers are the callers besides the controller that may ask secret-ask themselves, as the vault's manifest names + // them (MESH_VAULT_SECRET_ASK_CALLERS; novox/hq ADR 0563 §8): each is named by the runtime's proof. Empty: the + // controller alone. + AskCallers stdio.Callers + Now func() time.Time + Log func(format string, args ...any) // Accounts is the vault's state of bus-account hashes (accounts.go, ADR 0547 clause 9): nil writes none. Accounts AccountStore // HashCost is the bcrypt cost of those hashes: zero is the controller's, bcrypt's default. @@ -58,6 +65,19 @@ type Vault struct { hashes hashing // adoption is the last adoption (adopt.go), under mu. adoption adoptionKept + // WantRead reads one wanted account afresh (issue-account): nil reads the wants as the watch handed them. + WantRead func(user string) ([]byte, bool, error) + // Placement reads the seat's placement (placed.go): where modules stand once the assignments are gone. + Placement PlacementReader + // LeaseRead reads the controller's lease afresh before a mint (lease.go): nil reads it only as the watch hands it. + LeaseRead func() ([]byte, bool, error) + // minting is the switch on the lease and what the vault knows of the wanted bus accounts (mint.go). + minting minting + // Signers is the vault's state `signers`, where it publishes its signing halves (signing.go, ADR 0550 §2): nil writes + // none. + Signers SignersStore + // signingWhy is why the last signing pass did not finish, under mu (signing.go). + signingWhy string // callsMu guards pending: the keys of the calls open now (intake.go), never written anywhere. callsMu sync.Mutex pending map[string]*callKey @@ -106,8 +126,10 @@ func (v *Vault) keep(name, value, by, via, origin string) (Changed, error) { // was first made, and where it was moved from. type keepOptions struct { neverReplace bool - madeAt time.Time - movedFrom *MovedFrom + // quiet keeps secret-changed for the caller to say once the rest is done (a minted account: mint.go). + quiet bool + madeAt time.Time + movedFrom *MovedFrom // replaceOnly, keepPrevious and refuse are what move-in with replace_running keeps (owntake.go): only a value held is replaced, // keeping its origin and the value it replaces, and refuse, under the store's lock, says why a held value is not. replaceOnly bool @@ -208,7 +230,9 @@ func (v *Vault) keepLocked(name, value, by, via, origin string, opt keepOptions) return Changed{}, err } c := Changed{Name: name, Generation: rec.Generation, At: now, By: by} - v.announce(c) + if !opt.quiet { + v.announce(c) + } return c, nil } @@ -283,6 +307,10 @@ func (v *Vault) Get(caller, name, publicKey string) (Answer, error) { if err := validName(name); err != nil { return Answer{}, err } + // The vault's signing secrets are handed to nobody: it signs with them itself (ADR 0550 §2). + if signingSecret(name) { + return Answer{}, fmt.Errorf("%s is the vault's own signing secret and is handed to nobody: ask the verb sign", name) + } // A value issued for one holder is that holder's alone, whatever else would allow it (issue 749). if why := notTheHolder(caller, name); why != "" { v.log("refused %s to %s: %s", name, caller, why) diff --git a/modules/mesh-vault/cmd/mesh-vault/vault_test.go b/modules/mesh-vault/cmd/mesh-vault/vault_test.go index 1a3b002c..c3f22821 100644 --- a/modules/mesh-vault/cmd/mesh-vault/vault_test.go +++ b/modules/mesh-vault/cmd/mesh-vault/vault_test.go @@ -131,9 +131,11 @@ func filesUnder(t *testing.T, dir string) map[string]string { // The manifest declares one bucket, and no secret is in it: the vault's only place for a secret is its own store on its // own disk (ADR 0349 §7, ADR 0201 §13). The one bucket is bus-accounts, the bcrypt hashes of the bus accounts it holds -// (ADR 0547 clause 9), opened in accounts.go alone. It reads two buckets of others', and only reads them: the -// controller's assignments, for what each node's plan names (askers.go, design 57 V4), and the node-engines' haves, where -// the bus node hands the loaded list's hashes sealed to the vault (adopt.go, ADR 0547 clause 9), in main.go alone. +// (ADR 0547 clause 9), opened in accounts.go alone. It reads five buckets of others', and only reads them: the +// seat mesh-provisioner's plans, for what each node's plan names (ADR 0570 clause 2), (askers.go, design 57 V4), the node-engines' haves, where +// the bus node hands the loaded list's hashes sealed to the vault (adopt.go, ADR 0547 clause 9), the grants part's wanted +// accounts and the controller's lease (mint.go, lease.go, ADR 0561 §2, §3), and the seat's placement (placed.go), in +// main.go alone. func TestNoBucketHoldsASecret(t *testing.T) { raw, err := os.ReadFile("../../module.json") if err != nil { @@ -143,12 +145,14 @@ func TestNoBucketHoldsASecret(t *testing.T) { if err := json.Unmarshal(raw, &manifest); err != nil { t.Fatal(err) } - if s, _ := json.Marshal(manifest["state"]); string(s) != `[{"history":2,"name":"bus-accounts"}]` { - t.Errorf("the manifest declares buckets %s: a secret is in no bucket, sealed or not (ADR 0201 §13), and the one "+ - "bucket is the accounts' hashes (ADR 0547 clause 9)", s) + if s, _ := json.Marshal(manifest["state"]); string(s) != `[{"history":2,"name":"bus-accounts"},"signers","bound"]` { + t.Errorf("the manifest declares buckets %s: a secret is in no bucket, sealed or not (ADR 0201 §13), and the "+ + "buckets are the accounts' hashes (ADR 0547 clause 9), the signing keys' public halves (ADR 0550 §2) and the "+ + "credentials' outcomes, names and generations alone (ADR 0570 clauses 3, 6)", s) } - if r, _ := json.Marshal(manifest["reads"]); string(r) != `["mesh-controller.assignments","node-engine.have"]` { - t.Errorf("the manifest reads %s: the vault reads the controller's assignments and the haves alone", r) + if r, _ := json.Marshal(manifest["reads"]); string(r) != `["node-engine.have","mesh-grants.accounts","mesh-controller.lease","mesh-delivery-assigner.placement","mesh-provisioner.plans","mesh-provisioner.provisions"]` { + t.Errorf("the manifest reads %s: the vault reads the seat mesh-provisioner's plans and provisions, the haves, the "+ + "grants part's wanted accounts, the lease and the placement alone", r) } sources, _ := filepath.Glob("*.go") bucket := regexp.MustCompile(`stdio\.State\(|nats-io|jetstream|KeyValue`) @@ -166,14 +170,35 @@ func TestNoBucketHoldsASecret(t *testing.T) { code = []byte(strings.Replace(string(code), "stdio.State(BusAccountsState).Get(", "", 1)) code = []byte(strings.Replace(string(code), "stdio.State(BusAccountsState).Put(", "", 1)) code = []byte(strings.Replace(string(code), "stdio.State(BusAccountsState).Keys(", "", 1)) + code = []byte(strings.Replace(string(code), "stdio.State(BusAccountsState).Delete(", "", 1)) } - // The haves, in main.go alone: one read, one watch and their keys (own_drift), never a write; and one watch of - // the assignments, which the vault's rules read from memory once it caught up (issue 1126). + // The signing keys' public halves, in signing.go alone: one read and one write of its own bucket (ADR 0550 §2). + if f == "signing.go" { + code = []byte(strings.Replace(string(code), "stdio.State(delivery.SignersState).Get(", "", 1)) + code = []byte(strings.Replace(string(code), "stdio.State(delivery.SignersState).Put(", "", 1)) + } + // The haves, in main.go alone: one read, one watch and their keys (own_drift), never a write; one watch of the + // assignments, which the vault's rules read from memory once it caught up (issue 1126); and one watch of the wanted + // accounts, the grants part's, which hold no secret (ADR 0561 §2). if f == "main.go" { code = []byte(strings.Replace(string(code), "stdio.State(AssignmentsRead).Watch(", "", 1)) code = []byte(strings.Replace(string(code), "stdio.State(HaveRead).Get(", "", 1)) code = []byte(strings.Replace(string(code), "stdio.State(HaveRead).Watch(", "", 1)) code = []byte(strings.Replace(string(code), "stdio.State(HaveRead).Keys(", "", 1)) + code = []byte(strings.Replace(string(code), "stdio.State(WantedRead).Watch(", "", 1)) + code = []byte(strings.Replace(string(code), "stdio.State(WantedRead).Get(", "", 1)) + // And the controller's lease, one read and one watch (ADR 0561 §3). + code = []byte(strings.Replace(string(code), "stdio.State(LeaseRead).Get(", "", 1)) + code = []byte(strings.Replace(string(code), "stdio.State(LeaseRead).Watch(", "", 1)) + // And the seat's placement, where modules stand (placed.go): its keys and one key, read. + code = []byte(strings.Replace(string(code), "stdio.State(PlacementRead).Keys(", "", 1)) + code = []byte(strings.Replace(string(code), "stdio.State(PlacementRead).Get(", "", 1)) + // And its outcome entries (novox/hq ADR 0570 clauses 3, 6): a value's name and generation, never a value; and + // one read of the seat's provisions, which hold none. + for _, call := range []string{"Get(", "Put(", "Delete(", "Keys("} { + code = []byte(strings.Replace(string(code), "stdio.State(BoundState)."+call, "", 1)) + } + code = []byte(strings.Replace(string(code), "stdio.State(provisions.ProvisionsRead).Get(", "", 1)) } if m := bucket.Find(code); m != nil { t.Errorf("%s opens a bucket (%s): the vault keeps every secret in its own store only", f, m) diff --git a/modules/mesh-vault/go.mod b/modules/mesh-vault/go.mod index 34a8552a..cfda18d1 100644 --- a/modules/mesh-vault/go.mod +++ b/modules/mesh-vault/go.mod @@ -3,7 +3,7 @@ module mesh-vault go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261028174713-38dbe1d3a388 + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 golang.org/x/crypto v0.57.0 diff --git a/modules/mesh-vault/go.sum b/modules/mesh-vault/go.sum index 6245202d..a2b8e018 100644 --- a/modules/mesh-vault/go.sum +++ b/modules/mesh-vault/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261028174713-38dbe1d3a388 h1:ycu/sNieTdLpD3bzhZOCXdzcoqXUOxPRmqWV019REpo= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261028174713-38dbe1d3a388/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 h1:lKU3HjNXGiVbLW2hUgFjU/TMTUYMlNnnmZxG5T7biQA= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/mesh-vault/module.json b/modules/mesh-vault/module.json index 7b2e866f..efea0cb1 100644 --- a/modules/mesh-vault/module.json +++ b/modules/mesh-vault/module.json @@ -5,6 +5,9 @@ "provides": [ { "name": "secret", + "credential": { + "form": "pair" + }, "scope": "mesh", "identity": { "in": "a vault entry" @@ -16,17 +19,25 @@ "rotated", "secret-changed", "account-hashed", - "own-replaced" + "own-replaced", + "account-issued", + "signed" ], "reads": [ - "mesh-controller.assignments", - "node-engine.have" + "node-engine.have", + "mesh-grants.accounts", + "mesh-controller.lease", + "mesh-delivery-assigner.placement", + "mesh-provisioner.plans", + "mesh-provisioner.provisions" ], "state": [ { "name": "bus-accounts", "history": 2 - } + }, + "signers", + "bound" ], "receives": { "secret": "${dir:grants}/mesh.json" @@ -144,7 +155,8 @@ "MESH_VAULT_LEDGER": "${dir:ledger}", "MESH_VAULT_ROOT": "${dir:root}", "MESH_VAULT_STORE": "${dir:store}", - "MESH_VAULT_STATE": "${dir:state}" + "MESH_VAULT_STATE": "${dir:state}", + "MESH_VAULT_SECRET_ASK_CALLERS": "node-tools" } } ] @@ -157,6 +169,7 @@ "secret_own", "account_hash", "account_adopt", + "account_retire", "secret_asks", "secret_vault_export", "provisioner_retirement", @@ -166,7 +179,10 @@ "vault_status", "vault_health", "own_drift", - "own_take_switch" + "own_take_switch", + "vault_signers", + "vault_signed", + "signers_reassert" ], "claims": [ { @@ -181,7 +197,9 @@ "move-in", "node-key", "move-asks", - "own" + "own", + "issue-account", + "sign" ] } ] diff --git a/modules/mesh-watcher/cmd/mesh-watcher/main.go b/modules/mesh-watcher/cmd/mesh-watcher/main.go index 8a64c157..a4fb0b55 100644 --- a/modules/mesh-watcher/cmd/mesh-watcher/main.go +++ b/modules/mesh-watcher/cmd/mesh-watcher/main.go @@ -15,6 +15,8 @@ import ( "time" stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/health" ) func logf(format string, a ...any) { fmt.Fprintf(os.Stderr, format+"\n", a...) } @@ -98,8 +100,10 @@ func run(w *Watcher, l *listening, node string) { time.Sleep(10 * time.Minute) } }() + follow := newTakeover() + go followTakeover(follow) handle := func(e stdio.Envelope) error { - if !isHeartbeat(e.Key) { + if !follow.heard(e.Key) { return nil } b, err := DecodeBeat(e.Body) @@ -113,6 +117,9 @@ func run(w *Watcher, l *listening, node string) { time.Sleep(500 * time.Millisecond) for wait := 2 * time.Second; ; wait = min(wait*2, time.Minute) { err := stdio.Subscribe(ControllerSeat+".*", handle) + if err == nil { + err = stdio.Subscribe(health.SelfCheckModule+".*", handle) + } if err == nil { l.set("listening") logf("[mesh-watcher] listening for the self-check's heartbeat; watching from %s", node) @@ -144,60 +151,46 @@ func roundTrip(node string) error { } } -// controlNodes reads which machines hold the controller and the bus. +// controlNodes reads which machines hold the controller and the bus, from the placement seat's state of who holds +// which seat (novox/hq ADR 0571 clause 4). func controlNodes() ([]string, error) { - raw, err := stdio.Ask("seat:mesh-controller.seats", map[string]any{}) + st := stdio.State(delivery.SeatHoldersRead) + names, err := st.Keys() if err != nil { return nil, err } - return holdersOf(raw, "mesh-controller", "mesh-broker") + keys := map[string][]byte{} + for _, k := range names { + e, err := st.Get(k) + if err != nil { + return nil, err + } + if e != nil { + keys[k] = e.Value + } + } + return holderNodes(keys, "mesh-controller", "mesh-broker") } -// holdersOf reads the seats verb's answer, bare or inside a tool reply, for the named seats' machines. -func holdersOf(raw json.RawMessage, seats ...string) ([]string, error) { - var answer struct { - Seats []struct { - Seat string `json:"seat"` - Holders []struct { - Node string `json:"node"` - } `json:"holders"` - } `json:"seats"` - Output string `json:"output"` - Content []struct { - Text string `json:"text"` - } `json:"content"` - } - if err := json.Unmarshal(raw, &answer); err != nil { - var s string - if json.Unmarshal(raw, &s) == nil { - return holdersOf(json.RawMessage(s), seats...) - } - return nil, fmt.Errorf("the seats answer is not JSON") - } - if len(answer.Seats) == 0 { - switch { - case answer.Output != "": - return holdersOf(json.RawMessage(answer.Output), seats...) - case len(answer.Content) > 0: - return holdersOf(json.RawMessage(answer.Content[0].Text), seats...) - } - return nil, fmt.Errorf("the seats answer lists no seat") - } +// holderNodes is the machines holding any of the seats, in the order the seats are named, each once. +func holderNodes(keys map[string][]byte, seats ...string) ([]string, error) { var out []string seen := map[string]bool{} - for _, s := range answer.Seats { - for _, want := range seats { - if s.Seat != want { - continue - } - for _, h := range s.Holders { - if !seen[h.Node] { - seen[h.Node] = true - out = append(out, h.Node) - } + for _, seat := range seats { + holders, err := delivery.HoldersOf(keys, seat, "") + if err != nil { + return nil, err + } + for _, h := range holders { + if !seen[h.Node] { + seen[h.Node] = true + out = append(out, h.Node) } } } + if len(out) == 0 { + return nil, fmt.Errorf("the placement seat's state names no holder of %s", strings.Join(seats, " or ")) + } return out, nil } diff --git a/modules/mesh-watcher/cmd/mesh-watcher/manifest_test.go b/modules/mesh-watcher/cmd/mesh-watcher/manifest_test.go index 37052121..ddefa504 100644 --- a/modules/mesh-watcher/cmd/mesh-watcher/manifest_test.go +++ b/modules/mesh-watcher/cmd/mesh-watcher/manifest_test.go @@ -34,10 +34,10 @@ func TestTheManifestSaysWhatTheCodeDoes(t *testing.T) { if err := json.Unmarshal(raw, &m); err != nil { t.Fatal(err) } - if !reflect.DeepEqual(m.Consumes, []string{ControllerSeat + "." + HeartbeatEvent}) { + if !reflect.DeepEqual(m.Consumes, []string{ControllerSeat + "." + HeartbeatEvent, "mesh-self-check.checked"}) { t.Fatalf("consumes %v", m.Consumes) } - if !reflect.DeepEqual(m.Invokes, []string{m.Module + ".watcher_ping", "seat:mesh-controller.seats"}) { + if !reflect.DeepEqual(m.Invokes, []string{m.Module + ".watcher_ping"}) { t.Fatalf("invokes %v", m.Invokes) } if len(m.Claims) != 0 { diff --git a/modules/mesh-watcher/cmd/mesh-watcher/takeover.go b/modules/mesh-watcher/cmd/mesh-watcher/takeover.go new file mode 100644 index 00000000..540d70c3 --- /dev/null +++ b/modules/mesh-watcher/cmd/mesh-watcher/takeover.go @@ -0,0 +1,59 @@ +package main + +import ( + "sync" + "time" + + stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/health" +) + +// takeover is which self-check run the watcher hears: the controller's `doctor-heartbeat` until the keeper's +// take-over key names the keeper, then mesh-self-check's `checked`, whose body is the same run (novox/hq ADR 0564 +// clause 9, issue 1952 W1). An absent key follows the controller. +type takeover struct { + mu sync.Mutex + t health.Takeover +} + +func newTakeover() *takeover { + return &takeover{t: health.Takeover{V: health.TakeoverVersion, Follow: health.FollowController}} +} + +// heard says whether key is the followed self-check event. +func (f *takeover) heard(key string) bool { + f.mu.Lock() + defer f.mu.Unlock() + seat, event := f.t.SelfCheckEvent() + return key == seat+"."+event +} + +func (f *takeover) set(value []byte, found bool) error { + t, err := health.DecodeTakeover(value, found) + if err != nil { + return err + } + f.mu.Lock() + f.t = t + f.mu.Unlock() + return nil +} + +// followTakeover watches the key for the life of the process; one it cannot read leaves the event it heard. +func followTakeover(f *takeover) { + for wait := 2 * time.Second; ; wait = min(wait*2, time.Minute) { + err := stdio.State(health.KeeperModule+"."+health.TakeoverState).Watch(health.TakeoverKey, func(c stdio.StateChange) error { + if err := f.set(c.Value, c.Op != "delete" && len(c.Value) > 0); err != nil { + logf("[mesh-watcher] the take-over key is unreadable (%v); no change in what it hears", err) + } else { + logf("[mesh-watcher] the take-over key now follows %s", f.t.Follow) + } + return nil + }) + if err == nil { + return + } + logf("[mesh-watcher] not watching the take-over key yet (%v)", err) + time.Sleep(wait) + } +} diff --git a/modules/mesh-watcher/cmd/mesh-watcher/watcher_test.go b/modules/mesh-watcher/cmd/mesh-watcher/watcher_test.go index a335af11..bf77ce46 100644 --- a/modules/mesh-watcher/cmd/mesh-watcher/watcher_test.go +++ b/modules/mesh-watcher/cmd/mesh-watcher/watcher_test.go @@ -1,7 +1,6 @@ package main import ( - "encoding/json" "errors" "strings" "testing" @@ -214,16 +213,18 @@ func TestOnTheControlNodeItSaysItIsMisplaced(t *testing.T) { } } -func TestTheSeatsAnswerIsReadInEitherShape(t *testing.T) { - bare := `{"seats":[{"seat":"mesh-controller","holders":[{"node":"n1","module":"mesh-controller"}]}, - {"seat":"mesh-broker","holders":[{"node":"n1"},{"node":"n2"}]},{"seat":"git","holders":[{"node":"n3"}]}]}` - wrapped, _ := json.Marshal(map[string]any{"content": []map[string]string{{"type": "text", "text": bare}}}) - output, _ := json.Marshal(map[string]any{"ok": true, "output": bare}) - for _, raw := range []string{bare, string(wrapped), string(output)} { - got, err := holdersOf(json.RawMessage(raw), "mesh-controller", "mesh-broker") - if err != nil || strings.Join(got, ",") != "n1,n2" { - t.Errorf("%s: %v %v", raw, got, err) - } +func TestTheHoldersAreReadFromThePlacementSeatsState(t *testing.T) { + keys := map[string][]byte{ + "mesh-controller": []byte(`{"v":1,"seat":"mesh-controller","scope":"mesh","holders":[{"module":"mesh-controller","node":"n1"}]}`), + "mesh-broker": []byte(`{"v":1,"seat":"mesh-broker","scope":"mesh","holders":[{"module":"nats","node":"n1"},{"module":"nats","node":"n2"}]}`), + "git": []byte(`{"v":1,"seat":"git","scope":"mesh","holders":[{"module":"gitea","node":"n3"}]}`), + } + got, err := holderNodes(keys, "mesh-controller", "mesh-broker") + if err != nil || strings.Join(got, ",") != "n1,n2" { + t.Errorf("%v %v", got, err) + } + if _, err := holderNodes(map[string][]byte{}, "mesh-controller", "mesh-broker"); err == nil { + t.Error("no holder named, and no error") } } diff --git a/modules/mesh-watcher/go.mod b/modules/mesh-watcher/go.mod index 6c449fbb..37ea6996 100644 --- a/modules/mesh-watcher/go.mod +++ b/modules/mesh-watcher/go.mod @@ -1,5 +1,5 @@ module mesh-watcher -go 1.22 +go 1.26.0 -require git.novox.be/novox/mesh-sdk/go v0.1.7 +require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 diff --git a/modules/mesh-watcher/go.sum b/modules/mesh-watcher/go.sum index b474419e..09ad064e 100644 --- a/modules/mesh-watcher/go.sum +++ b/modules/mesh-watcher/go.sum @@ -1,2 +1,14 @@ -git.novox.be/novox/mesh-sdk/go v0.1.7 h1:C0sTQmtTiyYH7bnqZb7PusXnqA37gKuT7Nqjn9gG47w= -git.novox.be/novox/mesh-sdk/go v0.1.7/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 h1:lKU3HjNXGiVbLW2hUgFjU/TMTUYMlNnnmZxG5T7biQA= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA= +github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI= +github.com/nats-io/nats.go v1.54.0 h1:vsXoOxjHp/GmPUN+EcI7uOf/uB+iAP+kEsAFNQN0yzA= +github.com/nats-io/nats.go v1.54.0/go.mod h1:y+DZoD1oBOYfZTU681eTUiUjI0vbqYGixNVFHcjHJ0k= +github.com/nats-io/nkeys v0.4.16 h1:rd5oAuLOb8mnAycB0xleuEBNS1pVVnN0fv/FF34Eypg= +github.com/nats-io/nkeys v0.4.16/go.mod h1:llLgWoI0o4z/Q57q2R1kHfmocyhGV6VG/U18Glg1Afs= +github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= +github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= +golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M= +golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA= +golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo= +golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= diff --git a/modules/mesh-watcher/module.json b/modules/mesh-watcher/module.json index e9543a5d..ac05e105 100644 --- a/modules/mesh-watcher/module.json +++ b/modules/mesh-watcher/module.json @@ -3,11 +3,15 @@ "version": "1", "slug": "watch", "consumes": [ - "mesh-controller.doctor-heartbeat" + "mesh-controller.doctor-heartbeat", + "mesh-self-check.checked" + ], + "reads": [ + "mesh-conditions.takeover", + "mesh-delivery-assigner.seat-holders" ], "invokes": [ - "mesh-watcher.watcher_ping", - "seat:mesh-controller.seats" + "mesh-watcher.watcher_ping" ], "own-secrets": { "telegram-token": "${dir:state}/telegram-token" @@ -39,7 +43,9 @@ "type": "file", "path": "${dir:state}/settings.json", "mode": "0600", - "takes-config": ["telegram-chat-id"], + "takes-config": [ + "telegram-chat-id" + ], "merge": "json", "template": "files/settings.json" } diff --git a/modules/messenger/cmd/messenger/answers.go b/modules/messenger/cmd/messenger/answers.go index d92bf1af..6a3fbb5a 100644 --- a/modules/messenger/cmd/messenger/answers.go +++ b/modules/messenger/cmd/messenger/answers.go @@ -133,7 +133,7 @@ func pruneGrants(grants map[string]Grant, kind string, now time.Time, keep int) // stands, closes the ask with a warrant for its asker (asks.go); Details are shown where they were asked. // Anything else is refused, and said there. The same answer is acted on once. func (h *Holder) Chosen(kind string, c asks.Choice) (string, error) { - // The controller's root-free word is asked before the work is taken (the confirmation review of 2026-10-09): + // The machines' engines' root-free word is asked before the work is taken (the confirmation review of 2026-10-09): // up to RootFreeWithin, which no ask, answer or link should wait behind. Asked fresh here, for this answer, // and judged with in it. root := h.rootWordFor(kind) diff --git a/modules/messenger/cmd/messenger/answers_test.go b/modules/messenger/cmd/messenger/answers_test.go index cdffa473..c1ff0d0f 100644 --- a/modules/messenger/cmd/messenger/answers_test.go +++ b/modules/messenger/cmd/messenger/answers_test.go @@ -600,6 +600,16 @@ func TestAnAskIsRefusedInWordsToItsAsker(t *testing.T) { if _, kept := store.asks["mesh-delivery.d-1"]; kept { t.Error("a refused ask was recorded") } + // A node's engine is an asker under node-engine- (novox/hq ADR 0563 §3, changing ADR 0259 §2): its ask is + // recorded under that name, with its own bound of open asks; a name that is no asker's is dropped. + submit(t, h, "node-engine-ace", heldAsk(c, "hand-over-1")) + submit(t, h, "node.ace", heldAsk(c, "hand-over-2")) + if rec, kept := store.asks["node-engine-ace.hand-over-1"]; !kept || rec.Asker != "node-engine-ace" { + t.Errorf("a node's engine's ask was not recorded under its name: %+v", rec) + } + if _, kept := store.asks["node.ace.hand-over-2"]; kept { + t.Error("an ask under a name that is no asker's was recorded") + } // No channel can carry anything: held with the reason (ADR 0397), never queued in silence. h2, _, _, c2, store2, w2 := newRouter(t) h2.Bench.Stand("telegram", Standing{Ready: false, Why: "no bot token", At: c2.now()}) diff --git a/modules/messenger/cmd/messenger/asks.go b/modules/messenger/cmd/messenger/asks.go index 08a53a95..b719e68f 100644 --- a/modules/messenger/cmd/messenger/asks.go +++ b/modules/messenger/cmd/messenger/asks.go @@ -242,8 +242,9 @@ func (h *Holder) carriers(level asks.Level) []string { return out } -// RootVerdict is the controller's word on one machine (its verb root-free, novox/hq ADR 0259 §8): free only -// when it measured, recently, that no agent can become root there without a person. +// RootVerdict is a machine's engine's word on that machine (the node seat node-engine's root-free, novox/hq ADR 0259 +// §8 as ADR 0563 §6 changes it): free only when it measured, recently, that no agent can become root there without a +// person. type RootVerdict struct { Machine string `json:"machine"` Free bool `json:"free"` @@ -251,51 +252,64 @@ type RootVerdict struct { Judged string `json:"judged,omitempty"` } -// RootFreeVerb is the controller's verb the router asks, as this module, which machines are root-free. -const RootFreeVerb = "seat:" + ControllerSeat + ".root-free" +// EngineSeat is the node seat each machine's engine holds, and RootFreeVerb its verb the router asks of a machine, as +// this module: `seat:node-engine.root-free@`, which that machine's engine alone answers (ADR 0563 §6). +const ( + EngineSeat = "node-engine" + RootFreeVerb = "seat:" + EngineSeat + ".root-free" +) -// RootFreeWithin is how long the router waits for the controller's word; RootFreeKept how long a word is -// used to render a message's buttons. An answer is always judged on a word asked then. +// RootFreeWithin is how long the router waits for the machines' words; RootFreeKept how long a word is used to render a +// message's buttons. An answer is always judged on a word asked then. const ( RootFreeWithin = 10 * time.Second RootFreeKept = 30 * time.Second ) -// askRootFree asks the controller about machines, failing on no answer in time. +// askRootFree asks each machine's own engine about that machine, at once, failing on no answer in time. A machine's +// word is the verdict its engine gives on it; a verdict an engine gives on another machine is not taken. func (h *Holder) askRootFree(machines []string) (map[string]RootVerdict, error) { if h.RootFree != nil { return h.RootFree(machines) } if h.Ask == nil { - return nil, errors.New("this router cannot ask the controller") + return nil, errors.New("this router cannot ask a machine's engine") } type answer struct { - raw json.RawMessage - err error + machine string + raw json.RawMessage + err error } - done := make(chan answer, 1) - go func() { - raw, err := h.Ask(RootFreeVerb, map[string]any{"machines": machines}) - done <- answer{raw, err} - }() - var a answer - select { - case a = <-done: - case <-time.After(RootFreeWithin): - return nil, fmt.Errorf("the controller did not answer within %s", RootFreeWithin) - } - if a.err != nil { - return nil, a.err - } - var body struct { - Machines []RootVerdict `json:"machines"` - } - if err := json.Unmarshal(a.raw, &body); err != nil { - return nil, fmt.Errorf("the controller's answer could not be read: %v", err) + done := make(chan answer, len(machines)) + for _, m := range machines { + go func(m string) { + raw, err := h.Ask(RootFreeVerb+"@"+m, map[string]any{}) + done <- answer{m, raw, err} + }(m) } out := map[string]RootVerdict{} - for _, v := range body.Machines { - out[v.Machine] = v + deadline := time.After(RootFreeWithin) + for range machines { + var a answer + select { + case a = <-done: + case <-deadline: + return nil, fmt.Errorf("a machine's engine did not answer within %s", RootFreeWithin) + } + if a.err != nil { + return nil, fmt.Errorf("%s's engine: %w", a.machine, a.err) + } + var body struct { + Machines []RootVerdict `json:"machines"` + } + if err := json.Unmarshal(a.raw, &body); err != nil { + return nil, fmt.Errorf("%s's engine's answer could not be read: %v", a.machine, err) + } + for _, v := range body.Machines { + if v.Machine == a.machine { + out[v.Machine] = v + } + } } return out, nil } @@ -329,7 +343,7 @@ func (h *Holder) rootFree(k KindInfo, cached bool) string { case cached: // Read inside the work, which never waits on the controller: no recent word is no pass. warmRootFree // asks outside it. - return "the controller's word on whether an agent can become root there is not recent" + return "the machines' engines' word on whether an agent can become root there is not recent" default: got, err := h.askRootFree(machines) if err != nil { @@ -351,7 +365,7 @@ func (h *Holder) rootFree(k KindInfo, cached bool) string { v, said := verdicts[m] switch { case !said: - return "the controller said nothing of whether an agent can become root on " + m + return "its engine said nothing of whether an agent can become root on " + m case !v.Free: why := v.Why if why == "" { @@ -364,7 +378,7 @@ func (h *Holder) rootFree(k KindInfo, cached bool) string { return "" } -// warmRootFree asks the controller's root-free word for every kind that proves its sender, outside the work, +// warmRootFree asks the machines' engines' root-free word for every kind that proves its sender, outside the work, // so what is rendered and offered inside it reads a recent word. // force asks even a recent word again: taking an ask, whose refusal its asker keeps until something changes. func (h *Holder) warmRootFree(force bool) { @@ -390,7 +404,7 @@ func (h *Holder) warmRootFree(force bool) { } } -// rootWordFor asks the controller's root-free word for a kind fresh, before the work is taken, and answers +// rootWordFor asks the machines' engines' root-free word for a kind fresh, before the work is taken, and answers // what judging an answer of that kind then reads: the word for that very kind's placement, or why there is none. // A kind that does not prove its sender is asked nothing: its answers never approve. func (h *Holder) rootWordFor(kind string) func(KindInfo) string { @@ -453,7 +467,7 @@ func (h *Holder) TakeAsk(verb, asker string, body json.RawMessage) error { if verb == ConditionVerb { // Before the work lock: Condition takes it, and decides as for the controller's conditions. if !asks.UsableID(strings.ReplaceAll(asker, "-", "_")) { - h.Logf("[messenger] a condition from %q was taken and dropped: not a module's name", asker) + h.Logf("[messenger] a condition from %q was taken and dropped: not an asker's name (a module's, or a node's engine's)", asker) return nil } h.takeCondition(asker, body) @@ -466,7 +480,7 @@ func (h *Holder) TakeAsk(verb, asker string, body json.RawMessage) error { h.init() h.mu.Unlock() if !asks.UsableID(strings.ReplaceAll(asker, "-", "_")) { - h.Logf("[messenger] an ask from %q was taken and dropped: not a module's name", asker) + h.Logf("[messenger] an ask from %q was taken and dropped: not an asker's name (a module's, or a node's engine's)", asker) return nil } if verb == "cancel" { diff --git a/modules/messenger/cmd/messenger/bus_test.go b/modules/messenger/cmd/messenger/bus_test.go index 9c67bda6..c42fb278 100644 --- a/modules/messenger/cmd/messenger/bus_test.go +++ b/modules/messenger/cmd/messenger/bus_test.go @@ -256,16 +256,8 @@ func (r *busRuntime) runtime(id json.RawMessage, method string, params json.RawM case "mesh/seat.publish": return map[string]any{}, r.nc.Publish(p.Subject, p.Body) case "mesh/ask": - if p.Key == RootFreeVerb { - var a struct { - Machines []string `json:"machines"` - } - _ = json.Unmarshal(p.Body, &a) - var out []RootVerdict - for _, m := range a.Machines { - out = append(out, RootVerdict{Machine: m, Free: true}) - } - return map[string]any{"machines": out}, nil + if m, ok := strings.CutPrefix(p.Key, RootFreeVerb+"@"); ok { + return map[string]any{"machines": []RootVerdict{{Machine: m, Free: true}}}, nil } return nil, fmt.Errorf("the test runtime answers nothing for %s", p.Key) } diff --git a/modules/messenger/cmd/messenger/condition.go b/modules/messenger/cmd/messenger/condition.go index 1378d596..7a218fad 100644 --- a/modules/messenger/cmd/messenger/condition.go +++ b/modules/messenger/cmd/messenger/condition.go @@ -122,9 +122,12 @@ func (c Condition) SilencedAt(now time.Time) bool { // eventOf is the local event name of an envelope's key: `mesh-controller.condition-raised` is // `condition-raised`. Anything not of the controller's seat is not a condition event. -func eventOf(key string) (string, bool) { +func eventOf(key string) (string, bool) { return eventOfSeat(key, ControllerSeat) } + +// eventOfSeat is eventOf for the seat the take-over key says is followed: the controller's, or the keeper's (ADR 0564 clause 9). +func eventOfSeat(key, seat string) (string, bool) { emitter, event, ok := strings.Cut(key, ".") - if !ok || emitter != ControllerSeat { + if !ok || emitter != seat { return "", false } switch event { diff --git a/modules/messenger/cmd/messenger/holder.go b/modules/messenger/cmd/messenger/holder.go index 120daa75..b3dff885 100644 --- a/modules/messenger/cmd/messenger/holder.go +++ b/modules/messenger/cmd/messenger/holder.go @@ -315,7 +315,8 @@ type Holder struct { Emit func(event string, body any) error // Ask calls a seat verb through the mesh, as this module: the answers the operator chooses. Ask Asker - // RootFree asks the controller's verb root-free about machines; nil asks it through Ask (askRootFree). + // RootFree asks each machine's engine's root-free (the node seat node-engine's, ADR 0563 §6); nil asks it through Ask + // (askRootFree). RootFree func(machines []string) (map[string]RootVerdict, error) // Zone is the operator's time zone, which every time in a message is said in; nil is this machine's. Zone func() *time.Location diff --git a/modules/messenger/cmd/messenger/main.go b/modules/messenger/cmd/messenger/main.go index e6c963f8..2cc92c95 100644 --- a/modules/messenger/cmd/messenger/main.go +++ b/modules/messenger/cmd/messenger/main.go @@ -18,6 +18,7 @@ import ( stdio "git.novox.be/novox/mesh-sdk/go" "git.novox.be/novox/mesh-sdk/go/asks" + "git.novox.be/novox/mesh-sdk/go/health" ) func errorf(format string, a ...any) error { return fmt.Errorf(format, a...) } @@ -415,8 +416,9 @@ func run(h *Holder, l *listening) { // The state first: what is open now is read from the controller before any event is taken, so // what the stream replays is history (novox/hq issue 271). Not reachable yet, it is asked for // again in the loop below, and old events meanwhile are kept as state, never said. + follow := newTakeover() h.Start() - syncFromController(h) + syncFromController(h, follow) go func() { tick := time.NewTicker(5 * time.Second) defer tick.Stop() @@ -429,10 +431,11 @@ func run(h *Holder, l *listening) { } if h.WantsSync() && now.Sub(lastAsk) >= time.Minute { lastAsk = now - syncFromController(h) + syncFromController(h, follow) } } }() + go followTakeover(follow, h) handle := func(e stdio.Envelope) error { if event, ok := strings.CutPrefix(e.Key, ViewModule+"."); ok { // A module in motion on a node, settled or stuck: a walk's start or end now that plans are gone (ADR 0308 §1, ADR 0539 §17). @@ -446,7 +449,9 @@ func run(h *Holder, l *listening) { } return nil } - event, ok := eventOf(e.Key) + // Only the seat the take-over key names is heard: the other's events are dropped, so the keeper's conditions + // are never held a second time beside the controller's (ADR 0564 clause 9). + event, ok := eventOfSeat(e.Key, follow.seat()) if !ok { return nil } @@ -476,6 +481,9 @@ func run(h *Holder, l *listening) { }() for wait := 2 * time.Second; ; wait = min(wait*2, time.Minute) { err := stdio.Subscribe(ControllerSeat+".*", handle) + if err == nil { + err = stdio.Subscribe(health.KeeperModule+".*", handle) + } if err == nil { l.set("listening") logf("[messenger] listening for the controller's condition events") @@ -625,7 +633,7 @@ func followKinds(h *Holder) { } // syncFromController reads the controller's open conditions and gives them to the holder. -func syncFromController(h *Holder) { +func syncFromController(h *Holder, follow *takeover) { asked := h.Now() type answer struct { raw json.RawMessage @@ -633,7 +641,7 @@ func syncFromController(h *Holder) { } done := make(chan answer, 1) go func() { - raw, err := stdio.Ask("seat:"+ControllerSeat+".conditions", map[string]any{}) + raw, err := stdio.Ask("seat:"+follow.seat()+".conditions", map[string]any{}) done <- answer{raw, err} }() var a answer diff --git a/modules/messenger/cmd/messenger/manifest_test.go b/modules/messenger/cmd/messenger/manifest_test.go index 2281501b..6e61b0de 100644 --- a/modules/messenger/cmd/messenger/manifest_test.go +++ b/modules/messenger/cmd/messenger/manifest_test.go @@ -81,13 +81,14 @@ func TestItDeclaresTheSeatsOfAnAskAndHoldsTheOperatorChannel(t *testing.T) { // } if !reflect.DeepEqual(m.Consumes, []string{ ControllerSeat + "." + EventRaised, ControllerSeat + "." + EventChanged, ControllerSeat + "." + EventCleared, + "mesh-conditions." + EventRaised, "mesh-conditions." + EventChanged, "mesh-conditions." + EventCleared, ViewModule + "." + EventMoving, ViewModule + "." + EventSettled, ViewModule + "." + EventStuck, "intake.choice", "intake.link", "intake.failed", "intake.standing"}) { t.Fatalf("consumes %v", m.Consumes) } // It performs nothing: the verbs it calls read the open conditions, whether root is free, and which issues // track which conditions. if !reflect.DeepEqual(m.Emits, []string{"refused"}) || !reflect.DeepEqual(m.Invokes, []string{"seat:" + ControllerSeat + ".conditions", - "seat:" + ControllerSeat + ".root-free", TrackingVerb}) { + "seat:mesh-conditions.conditions", "seat:" + EngineSeat + ".root-free", TrackingVerb}) { t.Fatalf("emits %v invokes %v", m.Emits, m.Invokes) } var state []string diff --git a/modules/messenger/cmd/messenger/review_test.go b/modules/messenger/cmd/messenger/review_test.go index 3409c3a6..9c042edc 100644 --- a/modules/messenger/cmd/messenger/review_test.go +++ b/modules/messenger/cmd/messenger/review_test.go @@ -8,6 +8,7 @@ import ( "os" "path/filepath" "regexp" + "sort" "strings" "sync" "testing" @@ -66,26 +67,37 @@ func TestApprovalNeedsTheControllersWordThatRootIsFree(t *testing.T) { } } -// H3: the word is asked of the controller's verb, as this module, and read from its answer. -func TestTheRootWordIsAskedOfTheControllersVerb(t *testing.T) { +// H3, as ADR 0563 §6 changes it: the word on each machine is asked of that machine's own engine, the node seat +// node-engine's root-free there, as this module, and read from its answer; a machine's word is that machine's alone. +func TestTheRootWordIsAskedOfEachMachinesEngine(t *testing.T) { h, _, _, _, _, _ := newRouter(t) h.RootFree = nil var asked []string h.Ask = func(key string, body any) (json.RawMessage, error) { asked = append(asked, key) - raw, _ := json.Marshal(body) - if !strings.Contains(string(raw), `"machines":["anchor"]`) { - t.Errorf("asked about %s", raw) + machine := strings.TrimPrefix(key, "seat:node-engine.root-free@") + if machine == "elsewhere" { + machine = "anchor" // an engine that answers for another machine is not believed } - return json.RawMessage(`{"machines":[{"machine":"anchor","free":true,"why":"judged","judged":"2026-10-09T10:00:00Z"}]}`), nil + return json.RawMessage(`{"machines":[{"machine":"` + machine + `","free":true,"why":"judged","judged":"2026-10-09T10:00:00Z"}]}`), nil } info, _ := h.Bench.Kind("telegram") if why := h.rootFree(info, false); why != "" { t.Fatalf("free not read: %s", why) } - if len(asked) != 1 || asked[0] != "seat:mesh-controller.root-free" { + if len(asked) != 1 || asked[0] != "seat:node-engine.root-free@anchor" { t.Errorf("asked %v", asked) } + asked = nil + h.Node = "elsewhere" + if why := h.rootFree(info, false); !strings.Contains(why, "said nothing of whether an agent can become root on elsewhere") { + t.Errorf("one machine's word was taken for another's: %q", why) + } + sort.Strings(asked) + if len(asked) != 2 || asked[0] != "seat:node-engine.root-free@anchor" || asked[1] != "seat:node-engine.root-free@elsewhere" { + t.Errorf("the router's machine and the channel's were asked as %v", asked) + } + h.Node = "anchor" h.Ask = func(string, any) (json.RawMessage, error) { return json.RawMessage(`not json`), nil } if why := h.rootFree(info, false); !strings.Contains(why, "could not be read") { t.Errorf("an unreadable answer was a pass: %q", why) diff --git a/modules/messenger/cmd/messenger/sync.go b/modules/messenger/cmd/messenger/sync.go index efeaa6ee..34d1d696 100644 --- a/modules/messenger/cmd/messenger/sync.go +++ b/modules/messenger/cmd/messenger/sync.go @@ -180,3 +180,11 @@ func (h *Holder) WantsSync() bool { h.init() return h.wantSync || h.syncedAt.IsZero() || h.Now().Sub(h.syncedAt) >= ResyncEvery } + +// AskSync has the next pass of the loop read the open conditions again (the take-over key named another seat). +func (h *Holder) AskSync() { + h.mu.Lock() + h.init() + h.wantSync = true + h.mu.Unlock() +} diff --git a/modules/messenger/cmd/messenger/takeover.go b/modules/messenger/cmd/messenger/takeover.go new file mode 100644 index 00000000..e02ae421 --- /dev/null +++ b/modules/messenger/cmd/messenger/takeover.go @@ -0,0 +1,59 @@ +package main + +import ( + "sync" + "time" + + stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/health" +) + +// takeover is which seat's condition events the messenger hears: the controller's until the keeper's take-over key +// says otherwise (novox/hq ADR 0564 clause 9, issue 1952 R1). An absent key follows the controller. +type takeover struct { + mu sync.Mutex + t health.Takeover +} + +func newTakeover() *takeover { + return &takeover{t: health.Takeover{V: health.TakeoverVersion, Follow: health.FollowController}} +} + +func (f *takeover) seat() string { f.mu.Lock(); defer f.mu.Unlock(); return f.t.Seat() } + +// set takes the key as a watch handed it and says whether the followed seat changed. +func (f *takeover) set(value []byte, found bool) (changed bool, err error) { + t, err := health.DecodeTakeover(value, found) + if err != nil { + return false, err + } + f.mu.Lock() + defer f.mu.Unlock() + changed = f.t.Seat() != t.Seat() + f.t = t + return changed, nil +} + +// followTakeover watches the take-over key for the life of the process, and asks the holder to read the followed +// seat's open conditions again when it changes. A key it cannot read leaves the seat it followed. +func followTakeover(f *takeover, h *Holder) { + for wait := 2 * time.Second; ; wait = min(wait*2, time.Minute) { + err := stdio.State(health.KeeperModule+"."+health.TakeoverState).Watch(health.TakeoverKey, func(c stdio.StateChange) error { + changed, err := f.set(c.Value, c.Op != "delete" && len(c.Value) > 0) + if err != nil { + logf("[messenger] the take-over key is unreadable (%v); still hearing %s", err, f.seat()) + return nil + } + if changed { + logf("[messenger] the take-over key now names %s", f.seat()) + h.AskSync() + } + return nil + }) + if err == nil { + return + } + logf("[messenger] not watching the take-over key yet (%v); hearing %s", err, f.seat()) + time.Sleep(wait) + } +} diff --git a/modules/messenger/go.mod b/modules/messenger/go.mod index e70013a7..9ea72bd8 100644 --- a/modules/messenger/go.mod +++ b/modules/messenger/go.mod @@ -3,7 +3,7 @@ module messenger go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261019091011-aa68b67389c9 + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/messenger/go.sum b/modules/messenger/go.sum index aa13456d..a2b8e018 100644 --- a/modules/messenger/go.sum +++ b/modules/messenger/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261019091011-aa68b67389c9 h1:qHpN/fpTDqijfMVoepxUT6mPkfcydb4LiPKsIokv6iM= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261019091011-aa68b67389c9/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 h1:lKU3HjNXGiVbLW2hUgFjU/TMTUYMlNnnmZxG5T7biQA= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/messenger/module.json b/modules/messenger/module.json index 4aceaebe..eec860f0 100644 --- a/modules/messenger/module.json +++ b/modules/messenger/module.json @@ -77,6 +77,9 @@ "mesh-controller.condition-raised", "mesh-controller.condition-changed", "mesh-controller.condition-cleared", + "mesh-conditions.condition-raised", + "mesh-conditions.condition-changed", + "mesh-conditions.condition-cleared", "mesh-delivery-view.moving", "mesh-delivery-view.settled", "mesh-delivery-view.stuck", @@ -90,9 +93,13 @@ ], "invokes": [ "seat:mesh-controller.conditions", - "seat:mesh-controller.root-free", + "seat:mesh-conditions.conditions", + "seat:node-engine.root-free", "seat:issue-tracker.tracking" ], + "reads": [ + "mesh-conditions.takeover" + ], "state": [ "open", "sent", diff --git a/modules/minio/cmd/object-store/manifest_test.go b/modules/minio/cmd/object-store/manifest_test.go index 34e5ebb9..58494c7f 100644 --- a/modules/minio/cmd/object-store/manifest_test.go +++ b/modules/minio/cmd/object-store/manifest_test.go @@ -20,7 +20,7 @@ func manifest(t *testing.T) map[string]any { return m } -// 1516/AC6: minio claims object-store (scope mesh) serving link, invokes seat:mesh-controller.seats, builds the Go +// 1516/AC6: minio claims object-store (scope mesh) serving link, reads the placement seat's seat-holders, builds the Go // bundle, and no tool named minio_presigned_url remains. func TestManifestClaimsTheSeat(t *testing.T) { m := manifest(t) @@ -38,9 +38,9 @@ func TestManifestClaimsTheSeat(t *testing.T) { if c["name"] != "object-store" || c["scope"] != "mesh" || len(serves) != 1 || serves[0] != "link" { t.Fatalf("claim: %v", c) } - inv := m["invokes"].([]any) - if len(inv) != 1 || inv[0] != "seat:mesh-controller.seats" { - t.Fatalf("invokes: %v", inv) + reads, _ := m["reads"].([]any) + if _, invokes := m["invokes"]; invokes || len(reads) != 1 || reads[0] != "mesh-delivery-assigner.seat-holders" { + t.Fatalf("reads %v and invokes %v: it reads the placement seat's holders and calls nothing", reads, m["invokes"]) } var goBundle map[string]any for _, a := range m["build"].(map[string]any)["artifacts"].([]any) { diff --git a/modules/minio/cmd/object-store/server.go b/modules/minio/cmd/object-store/server.go index 9b4b59e5..789873b8 100644 --- a/modules/minio/cmd/object-store/server.go +++ b/modules/minio/cmd/object-store/server.go @@ -13,6 +13,7 @@ import ( "time" stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/delivery" "git.novox.be/novox/mesh-sdk/go/objectlink" ) @@ -59,11 +60,11 @@ func linkKeyID(bucket string) string { type server struct { configPath, linksDir, receives string now func() time.Time - // seats asks the controller for its seats; a test replaces it. - seats func() (json.RawMessage, error) + // seats reads the placement seat's state of who holds which seat, every key; a test replaces it. + seats func() (map[string][]byte, error) mu sync.Mutex - holders map[string]bool // "/." + keys map[string][]byte // seat-holders, as read readAt time.Time haveSet bool } @@ -77,8 +78,23 @@ func newServer(config, links, receives string) (*server, error) { return &server{configPath: config, linksDir: links, receives: receives, now: time.Now, seats: askSeats}, nil } -func askSeats() (json.RawMessage, error) { - return stdio.Ask("seat:mesh-controller.seats", map[string]any{}) +func askSeats() (map[string][]byte, error) { + state := stdio.State(delivery.SeatHoldersRead) + names, err := state.Keys() + if err != nil { + return nil, err + } + keys := map[string][]byte{} + for _, k := range names { + e, err := state.Get(k) + if err != nil { + return nil, err + } + if e != nil { + keys[k] = e.Value + } + } + return keys, nil } // link answers one ask. @@ -206,7 +222,7 @@ func (s *server) allowed(g grant, node, module, method string) error { if seat, isSeat := strings.CutPrefix(sh.To, "seat:"); isSeat { held, err := s.holds(seat, node, module) if err != nil { - return false, fmt.Errorf("a share names seat %s and the controller's seats cannot be read to tell whether this caller holds it: %w", seat, err) + return false, fmt.Errorf("a share names seat %s and the placement seat's holders cannot be read to tell whether this caller holds it: %w", seat, err) } return held, nil } @@ -249,58 +265,26 @@ func has(list []string, v string) bool { return false } -// holds says whether node.module holds the seat, from the controller's seats, read at most once a minute. A read that -// failed is not cached: the next ask reads again. +// holds says whether node.module holds the seat, from the placement seat's state of who holds which seat (novox/hq ADR +// 0571 clause 4), read at most once a minute. A read that failed is not cached: the next ask reads again. func (s *server) holds(seat, node, module string) (bool, error) { s.mu.Lock() defer s.mu.Unlock() if !s.haveSet || s.now().Sub(s.readAt) >= holdersFor { - raw, err := s.seats() + keys, err := s.seats() if err != nil { return false, err } - set, err := holdersIn(raw) - if err != nil { - return false, err - } - s.holders, s.readAt, s.haveSet = set, s.now(), true + s.keys, s.readAt, s.haveSet = keys, s.now(), true } - return s.holders[seat+"/"+node+"."+module], nil -} - -// holdersIn reads the controller's `seats` answer (answer.seats[].holders[] of {node, module}, and answer.outside[] of -// {Claim, Node, Module}) into "/.". The answer may arrive wrapped in {"answer": ...}. -func holdersIn(raw json.RawMessage) (map[string]bool, error) { - var doc struct { - Answer json.RawMessage `json:"answer"` - Seats []struct { - Seat string `json:"seat"` - Holders []struct { - Node string `json:"node"` - Module string `json:"module"` - } `json:"holders"` - } `json:"seats"` - Outside []struct { - Claim, Node, Module string - } `json:"outside"` + holders, err := delivery.HoldersOf(s.keys, seat, node) + if err != nil { + return false, err } - if err := json.Unmarshal(raw, &doc); err != nil { - return nil, fmt.Errorf("the controller's seats answer is not JSON: %w", err) - } - if len(doc.Answer) > 0 && string(doc.Answer) != "null" { - return holdersIn(doc.Answer) - } - if doc.Seats == nil && doc.Outside == nil { - return nil, errors.New("the controller's seats answer holds neither seats nor outside claims") - } - out := map[string]bool{} - for _, s := range doc.Seats { - for _, h := range s.Holders { - out[s.Seat+"/"+h.Node+"."+h.Module] = true + for _, h := range holders { + if h.Node == node && h.Module == module { + return true, nil } } - for _, o := range doc.Outside { - out[o.Claim+"/"+o.Node+"."+o.Module] = true - } - return out, nil + return false, nil } diff --git a/modules/minio/cmd/object-store/server_test.go b/modules/minio/cmd/object-store/server_test.go index 2f944a27..ba7bc956 100644 --- a/modules/minio/cmd/object-store/server_test.go +++ b/modules/minio/cmd/object-store/server_test.go @@ -1,7 +1,6 @@ package main import ( - "encoding/json" "errors" "net/url" "os" @@ -25,12 +24,13 @@ const grantFile = `{ ] }` -// The seats answer is the controller's (mesh-controller.seats read on 2026-10-21): seats[].holders[] of {node, module}, -// and outside[] of {Claim, Node, Module}. -const seatsAnswer = `{"answer": {"seats": [ - {"seat": "node-files", "scope": "node", "holders": [{"node": "g14", "module": "mesh-files"}, {"node": "shanks", "module": "mesh-files"}]}, - {"seat": "node-backup", "scope": "node", "holders": [{"node": "g14", "module": "restic"}]} - ], "outside": [{"Claim": "printer", "Scope": "mesh", "Node": "shanks", "Module": "brother", "Site": "home", "Kind": ""}]}}` +// The placement seat's state of who holds which seat (seat-holders): a key per node seat and node, a key per mesh seat. +var seatHolderKeys = map[string][]byte{ + "node-files.g14": []byte(`{"v":1,"seat":"node-files","scope":"node","node":"g14","holders":[{"module":"mesh-files","node":"g14"}]}`), + "node-files.shanks": []byte(`{"v":1,"seat":"node-files","scope":"node","node":"shanks","holders":[{"module":"mesh-files","node":"shanks"}]}`), + "node-backup.g14": []byte(`{"v":1,"seat":"node-backup","scope":"node","node":"g14","holders":[{"module":"restic","node":"g14"}]}`), + "printer": []byte(`{"v":1,"seat":"printer","scope":"mesh","holders":[{"module":"brother","node":"shanks"}]}`), +} type rig struct { s *server @@ -64,12 +64,12 @@ func newRig(t *testing.T) *rig { t.Fatal(err) } s.now = func() time.Time { return r.now } - s.seats = func() (json.RawMessage, error) { + s.seats = func() (map[string][]byte, error) { r.seatsCalls++ if r.seatsErr != nil { return nil, r.seatsErr } - return json.RawMessage(seatsAnswer), nil + return seatHolderKeys, nil } r.s = s return r diff --git a/modules/minio/go.mod b/modules/minio/go.mod index c09ccfda..c3197341 100644 --- a/modules/minio/go.mod +++ b/modules/minio/go.mod @@ -2,4 +2,4 @@ module minio go 1.26.0 -require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021122348-6c857e1c025f +require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 diff --git a/modules/minio/go.sum b/modules/minio/go.sum index 9728ce93..09ad064e 100644 --- a/modules/minio/go.sum +++ b/modules/minio/go.sum @@ -1,2 +1,14 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021122348-6c857e1c025f h1:7J/9jOfSfjfYpf8MOj96HwQRViQUkBNMb71RHP2mjn8= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021122348-6c857e1c025f/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 h1:lKU3HjNXGiVbLW2hUgFjU/TMTUYMlNnnmZxG5T7biQA= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA= +github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI= +github.com/nats-io/nats.go v1.54.0 h1:vsXoOxjHp/GmPUN+EcI7uOf/uB+iAP+kEsAFNQN0yzA= +github.com/nats-io/nats.go v1.54.0/go.mod h1:y+DZoD1oBOYfZTU681eTUiUjI0vbqYGixNVFHcjHJ0k= +github.com/nats-io/nkeys v0.4.16 h1:rd5oAuLOb8mnAycB0xleuEBNS1pVVnN0fv/FF34Eypg= +github.com/nats-io/nkeys v0.4.16/go.mod h1:llLgWoI0o4z/Q57q2R1kHfmocyhGV6VG/U18Glg1Afs= +github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= +github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= +golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M= +golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA= +golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo= +golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= diff --git a/modules/minio/module.json b/modules/minio/module.json index 01a966af..60b40a4b 100644 --- a/modules/minio/module.json +++ b/modules/minio/module.json @@ -12,6 +12,9 @@ "identity": { "max": 20, "in": "an S3 access key" + }, + "credential": { + "form": "pair" } } ], @@ -36,8 +39,8 @@ ] } ], - "invokes": [ - "seat:mesh-controller.seats" + "reads": [ + "mesh-delivery-assigner.seat-holders" ], "contributes": { "route": { diff --git a/modules/mongodb/module.json b/modules/mongodb/module.json index cbb12681..fc7d5212 100644 --- a/modules/mongodb/module.json +++ b/modules/mongodb/module.json @@ -12,6 +12,9 @@ "identity": { "max": 63, "in": "a MongoDB database name" + }, + "credential": { + "form": "pair" } } ], diff --git a/modules/mosquitto/module.json b/modules/mosquitto/module.json index 21058e0a..a7213d44 100644 --- a/modules/mosquitto/module.json +++ b/modules/mosquitto/module.json @@ -8,6 +8,9 @@ "scope": "mesh", "identity": { "in": "a Mosquitto client and topic prefix" + }, + "credential": { + "form": "pair" } } ], diff --git a/modules/mounts/cmd/mounts/bus.go b/modules/mounts/cmd/mounts/bus.go index ce3d0057..92505ce8 100644 --- a/modules/mounts/cmd/mounts/bus.go +++ b/modules/mounts/cmd/mounts/bus.go @@ -14,6 +14,7 @@ package main import ( "encoding/json" "fmt" + "git.novox.be/novox/mesh-sdk/go/health" "os" "path/filepath" "time" @@ -122,22 +123,22 @@ func (m *Machine) leave(p string, v any) error { // Asker calls a tool on the mesh as this module. type Asker func(key string, body any) (json.RawMessage, error) -// DataVerb is the controller's verb that says where the mesh keeps data on a machine. -const DataVerb = "seat:mesh-controller.data" +// Readings reads disk-load's readings of one machine's data items (its state `readings`, ADR 0571 clause 3). +type Readings func(node string) ([]health.Reading, error) -// LeaveData asks the controller where the mesh keeps data on this machine and leaves the paths for the +// LeaveData reads disk-load for where the mesh keeps data on this machine and leaves the paths for the // process (finding 4d). A failed ask keeps the last paths, with why; with none, the process sets up // nothing new. -func (m *Machine) LeaveData(node string, ask Asker, now time.Time) []string { - if node == "" || ask == nil { +func (m *Machine) LeaveData(node string, read Readings, now time.Time) []string { + if node == "" || read == nil { return nil } previous, _ := readJSON[DataPaths](m, DataFile) out := DataPaths{Machine: node, Asked: now.Format(time.RFC3339)} - raw, err := ask(DataVerb, map[string]any{"machine": node}) + readings, err := read(node) var paths []string if err == nil { - paths, err = DataFrom(raw) + paths, err = PathsFrom(readings) } var said []string if err != nil { diff --git a/modules/mounts/cmd/mounts/data.go b/modules/mounts/cmd/mounts/data.go index 803a1a84..02fc6da1 100644 --- a/modules/mounts/cmd/mounts/data.go +++ b/modules/mounts/cmd/mounts/data.go @@ -6,9 +6,9 @@ package main // mesh keeps data in on this machine, from the controller's `data` verb over the bus. import ( - "encoding/json" "errors" "fmt" + "git.novox.be/novox/mesh-sdk/go/health" "io" "os" "path/filepath" @@ -63,44 +63,16 @@ func sortWants(w []Want) { } } -// DataFrom reads the paths out of mesh-controller.data's answer: `answer.data`, `data`, or the same as -// text in `output`. An answer with no data list is an error, never an empty list: an empty list would -// let every mount point through. -func DataFrom(raw json.RawMessage) ([]string, error) { - type item struct { - Path string `json:"path"` - } - var whole struct { - Answer *struct { - Data *[]item `json:"data"` - } `json:"answer"` - Data *[]item `json:"data"` - Output string `json:"output"` - } - if err := json.Unmarshal(raw, &whole); err != nil { - return nil, fmt.Errorf("the controller's data answer is not JSON: %v", err) - } - var items *[]item - switch { - case whole.Answer != nil && whole.Answer.Data != nil: - items = whole.Answer.Data - case whole.Data != nil: - items = whole.Data - case whole.Output != "": - var inner struct { - Data *[]item `json:"data"` - } - if err := json.Unmarshal([]byte(whole.Output), &inner); err == nil && inner.Data != nil { - items = inner.Data - } - } - if items == nil { - return nil, errors.New("the controller's data answer names no data list") +// PathsFrom is the paths disk-load's readings name: where the mesh keeps data here. No reading is an error, never an +// empty list: an empty list would let every mount point through. +func PathsFrom(readings []health.Reading) ([]string, error) { + if len(readings) == 0 { + return nil, errors.New("disk-load keeps no reading of this machine's data yet") } paths := []string{} - for _, it := range *items { - if strings.HasPrefix(it.Path, "/") { - paths = append(paths, filepath.Clean(it.Path)) + for _, r := range readings { + if strings.HasPrefix(r.Path, "/") { + paths = append(paths, filepath.Clean(r.Path)) } } return paths, nil diff --git a/modules/mounts/cmd/mounts/main.go b/modules/mounts/cmd/mounts/main.go index ea5b32c0..7a69b8d0 100644 --- a/modules/mounts/cmd/mounts/main.go +++ b/modules/mounts/cmd/mounts/main.go @@ -13,8 +13,10 @@ import ( "context" "encoding/json" "fmt" + "git.novox.be/novox/mesh-sdk/go/health" "os" "os/signal" + "strings" "syscall" "time" @@ -62,13 +64,36 @@ func shareOnTheBus(m *Machine) { _, err := stdio.State(SharesState).Put(key, value) return err } - ask := func(key string, body any) (json.RawMessage, error) { return stdio.Ask(key, body) } + read := func(node string) ([]health.Reading, error) { + st := stdio.State(health.ReadingsRead) + keys, err := st.Keys() + if err != nil { + return nil, err + } + var out []health.Reading + for _, k := range keys { + if !strings.HasPrefix(k, node+".") { + continue + } + e, err := st.Get(k) + if err != nil { + return nil, err + } + if e == nil { + continue + } + if r, err := health.DecodeReading(e.Value); err == nil { + out = append(out, r) + } + } + return out, nil + } last, lastSaid := "", "" for n := 0; ; n++ { said := m.Share(node, get, put, &last, time.Now()) // Where the mesh keeps data here changes only with an assignment: asked at start and every five minutes. if n%10 == 0 { - said = append(said, m.LeaveData(node, ask, time.Now())...) + said = append(said, m.LeaveData(node, read, time.Now())...) } if s := fmt.Sprint(said); len(said) > 0 && s != lastSaid { fmt.Fprintln(os.Stderr, "mounts:", s) diff --git a/modules/mounts/cmd/mounts/manifest_test.go b/modules/mounts/cmd/mounts/manifest_test.go index 257ec6ce..43c6f67e 100644 --- a/modules/mounts/cmd/mounts/manifest_test.go +++ b/modules/mounts/cmd/mounts/manifest_test.go @@ -77,12 +77,10 @@ func TestTheManifestNamesWhatTheBundleServesAndWhatItNeeds(t *testing.T) { if strings.Join(m.Requires, ",") != "nfs-share" { t.Errorf("requires %v", m.Requires) } - if strings.Join(m.Reads, ",") != ExportsState { - t.Errorf("the module reads nfs-server's exports from the bus: reads %v", m.Reads) - } - // It calls one verb: the controller's, for where the mesh keeps data on this machine (review finding 4d). - if strings.Join(m.Invokes, ",") != DataVerb { - t.Errorf("invokes %v, want %s", m.Invokes, DataVerb) + // It reads nfs-server's exports, and disk-load's readings for where the mesh keeps data on this machine (review + // finding 4d, ADR 0571 clause 3); it calls no verb. + if strings.Join(m.Reads, ",") != "disk-load.readings,"+ExportsState || len(m.Invokes) != 0 { + t.Errorf("reads %v, invokes %v", m.Reads, m.Invokes) } if !strings.Contains(raw, `node=${machine:name}`) { t.Error("the config gives the process this machine's name, so it never mounts a share onto the machine that serves it") diff --git a/modules/mounts/cmd/mounts/review_test.go b/modules/mounts/cmd/mounts/review_test.go index 9e1c4c68..34778bf1 100644 --- a/modules/mounts/cmd/mounts/review_test.go +++ b/modules/mounts/cmd/mounts/review_test.go @@ -5,6 +5,8 @@ package main import ( "context" "encoding/json" + "git.novox.be/novox/mesh-sdk/go/catalogue" + "git.novox.be/novox/mesh-sdk/go/health" "os" "path/filepath" "strings" @@ -198,18 +200,13 @@ func TestWithoutTheMeshsDataNothingNewIsSetUp(t *testing.T) { func TestTheBundleLeavesTheMeshsDataForTheProcess(t *testing.T) { f := newSystem(t) f.write(ConfigFile, libraryOnly) - answer := json.RawMessage(`{"answer":{"data":[{"machine":"workstation","module":"plex","item":"data","path":"/mnt/plex/data"}]},"ok":true}`) - paths, err := DataFrom(answer) + rs := []health.Reading{{V: 1, Node: "workstation", PlacedDataItem: catalogue.PlacedDataItem{Module: "plex", ID: "data", Path: "/mnt/plex/data"}}} + paths, err := PathsFrom(rs) if err != nil || len(paths) != 1 || paths[0] != "/mnt/plex/data" { t.Fatalf("paths %q %v", paths, err) } - // The controller's verbs also answer as text in output. - paths, err = DataFrom(json.RawMessage(`{"ok":true,"output":"{\"data\":[{\"path\":\"/srv/x\"}]}"}`)) - if err != nil || len(paths) != 1 || paths[0] != "/srv/x" { - t.Fatalf("paths from output %q %v", paths, err) - } - if _, err := DataFrom(json.RawMessage(`{"ok":false}`)); err == nil { - t.Error("an answer with no data list is an error, not an empty list") + if _, err := PathsFrom(nil); err == nil { + t.Error("no reading is an error, not an empty list") } } @@ -287,23 +284,17 @@ func callVerb(t *testing.T, m *Machine, name string, args map[string]any) (any, func TestTheBundleKeepsTheLastDataWhenAnAskFails(t *testing.T) { f := newSystem(t) m := f.machine() - var asked []string - ok := func(key string, body any) (json.RawMessage, error) { - raw, _ := json.Marshal(body) - asked = append(asked, key+" "+string(raw)) - return json.RawMessage(`{"answer":{"data":[{"path":"/mnt/plex/data"}]}}`), nil + ok := func(node string) ([]health.Reading, error) { + return []health.Reading{{V: 1, Node: node, PlacedDataItem: catalogue.PlacedDataItem{Module: "plex", ID: "data", Path: "/mnt/plex/data"}}}, nil } if said := m.LeaveData("workstation", ok, time.Now()); len(said) != 0 { t.Fatalf("said %q", said) } - if len(asked) != 1 || asked[0] != `seat:mesh-controller.data {"machine":"workstation"}` { - t.Errorf("asked %q", asked) - } - failing := func(string, any) (json.RawMessage, error) { return nil, os.ErrDeadlineExceeded } + failing := func(string) ([]health.Reading, error) { return nil, os.ErrDeadlineExceeded } said := m.LeaveData("workstation", failing, time.Now()) d, err := readJSON[DataPaths](m, DataFile) if err != nil || len(d.Paths) != 1 || d.Paths[0] != "/mnt/plex/data" || d.Error == "" || len(said) == 0 { - t.Errorf("a failed ask keeps the last paths and says why: %+v %v %q", d, err, said) + t.Errorf("a failed read keeps the last paths and says why: %+v %v %q", d, err, said) } } diff --git a/modules/mounts/go.mod b/modules/mounts/go.mod index 76732a34..896fcdc5 100644 --- a/modules/mounts/go.mod +++ b/modules/mounts/go.mod @@ -1,5 +1,5 @@ module mounts -go 1.22 +go 1.26.0 -require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261010191001-33917f91ac3c +require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 diff --git a/modules/mounts/go.sum b/modules/mounts/go.sum index e655124d..f8bfbd2b 100644 --- a/modules/mounts/go.sum +++ b/modules/mounts/go.sum @@ -1,8 +1,2 @@ -git.novox.be/novox/mesh-sdk/go v0.1.6 h1:9qzdYONYbJdWcu6sxQcq9v1LI0JxcfkiKYkMUzJSkVQ= -git.novox.be/novox/mesh-sdk/go v0.1.6/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY= -git.novox.be/novox/mesh-sdk/go v0.1.7 h1:C0sTQmtTiyYH7bnqZb7PusXnqA37gKuT7Nqjn9gG47w= -git.novox.be/novox/mesh-sdk/go v0.1.7/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261010190644-622b6b4f556d h1:BR8uwv4BqyLaBwITuLylrd9DRUnu0D5DjJw0RuQSXbk= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261010190644-622b6b4f556d/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261010191001-33917f91ac3c h1:l5onJwoIeH8yE/PjVlEeoPyXBsHp2NQiWLllz2fwX7s= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261010191001-33917f91ac3c/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 h1:lKU3HjNXGiVbLW2hUgFjU/TMTUYMlNnnmZxG5T7biQA= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= diff --git a/modules/mounts/module.json b/modules/mounts/module.json index 488ebb97..dc1e3785 100644 --- a/modules/mounts/module.json +++ b/modules/mounts/module.json @@ -5,11 +5,9 @@ "nfs-share" ], "reads": [ + "disk-load.readings", "nfs-server.exports" ], - "invokes": [ - "seat:mesh-controller.data" - ], "capabilities": [ "package-manager", "service-manager" @@ -38,20 +36,20 @@ "per-machine": true } ], - "config": { - "shares": { - "type": "string", - "description": "The shares of another machine this machine mounts: space-separated name=mountpoint, each optionally :ro, or none (novox/hq ADR 0263 rule 5).", - "trusted": true, - "default": "none" - }, - "sources": { - "type": "string", - "description": "The occasional sources this machine mounts: space-separated name=smb://[@]/@ or name=disk:@, each optionally :ro, or none (novox/hq ADR 0263 rule 6); an SMB source's username is in its entry, its password the secret smb-password- (novox/hq ADR 0283).", - "trusted": true, - "default": "none" - } + "config": { + "shares": { + "type": "string", + "description": "The shares of another machine this machine mounts: space-separated name=mountpoint, each optionally :ro, or none (novox/hq ADR 0263 rule 5).", + "trusted": true, + "default": "none" }, + "sources": { + "type": "string", + "description": "The occasional sources this machine mounts: space-separated name=smb://[@]/@ or name=disk:@, each optionally :ro, or none (novox/hq ADR 0263 rule 6); an SMB source's username is in its entry, its password the secret smb-password- (novox/hq ADR 0283).", + "trusted": true, + "default": "none" + } + }, "own-secrets": { "smb-password-*": { "path": "${dir:state}/smb-password-*.secret", diff --git a/modules/mssql/module.json b/modules/mssql/module.json index ed4e4488..35cdbe6e 100644 --- a/modules/mssql/module.json +++ b/modules/mssql/module.json @@ -12,6 +12,9 @@ "identity": { "max": 128, "in": "a SQL Server login and database name" + }, + "credential": { + "form": "pair" } } ], diff --git a/modules/nats/cmd/mesh-nats-snapshot/certificate.go b/modules/nats/cmd/mesh-nats-snapshot/certificate.go new file mode 100644 index 00000000..7d03bc9e --- /dev/null +++ b/modules/nats/cmd/mesh-nats-snapshot/certificate.go @@ -0,0 +1,174 @@ +package main + +import ( + "crypto/rand" + "crypto/rsa" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "errors" + "fmt" + "math/big" + "net" + "os" + "path/filepath" + "time" +) + +// The bus's own certificate, made by the mesh rather than borrowed from an image. Moved here from mesh-controller's +// `broker certificate` with the same output (design 47's L3 s11): the command belongs to the bus module, and the +// controller's binary is retired after its lease is handed away (ADR 0547 §11). Nothing changes at the lease write: the +// certificate in /var/lib/mesh-broker-tls stays, and every pin stays valid. +// +// **The foundation asked a third-party image for a tool it never said must be there** (novox/hq +// 04-ISSUES/146). The bootstrap made this certificate by running `openssl` inside the broker's +// image, which worked while the broker was one that happened to carry it and stopped the day the +// bus changed: the new one has a shell and no openssl, so the step exited 127 and no mesh could be +// raised. Substituting another image the bundle names does not help — none of them carry it +// either. +// +// So the program that needs a certificate makes one. It is the mesh's own binary, already on the +// machine at this point in the bootstrap (the schema step ran it), and it needs nothing from the +// image it writes into but a mounted directory. +// +// **Self-signed, and that is the design** — a host pins this server's exact certificate and +// authenticates with a password (novox/hq ADR 0004). There is no authority above it to ask, and at +// this moment in a bootstrap there is no mesh to ask one of. +// +// Idempotent, because the step is applied again on every reconcile and a second certificate would +// be one the hosts that pinned the first no longer believe. + +// busCertificateNames is what the bus is reached by: the container name on a mesh network, and the +// loopback address the machine's own foundation dials. +var busCertificateNames = []string{"mesh-broker"} + +const busCertificateLife = 10 * 365 * 24 * time.Hour + +// busCertificate makes the bus's certificate in a directory, or says whether one is there. +// +// mesh-nats-snapshot certificate --into /tls make it if it is not there +// mesh-nats-snapshot certificate --check --into /tls exit non-zero unless a usable pair is +func busCertificate(args []string) error { + into, check := "", false + for i := 0; i < len(args); i++ { + switch args[i] { + case "--check": + check = true + case "--into": + if i+1 >= len(args) { + return errors.New("--into needs a directory") + } + into = args[i+1] + i++ + default: + return fmt.Errorf("certificate [--check] --into : %q", args[i]) + } + } + if into == "" { + return errors.New("certificate [--check] --into ") + } + crt, key := filepath.Join(into, "tls.crt"), filepath.Join(into, "tls.key") + + if usable, err := busCertificateUsable(crt, key); err != nil { + return err + } else if usable { + fmt.Printf("the bus already has a certificate at %s, and it was left alone\n", crt) + return nil + } + if check { + // Said as a failure, because that is what the caller asked: a bootstrap's verify runs + // this and a false answer is what makes the step run. + return fmt.Errorf("no usable certificate and key at %s", into) + } + return writeBusCertificate(crt, key) +} + +// busCertificateUsable says whether a certificate and its key are both there and parse. +// +// Both, and parsed rather than stat'ed: a half-written pair is the state a bootstrap interrupted +// between the two files leaves behind, and a step that treated it as done would hand the server a +// certificate with no key and report success. +func busCertificateUsable(crt, key string) (bool, error) { + certPEM, err := os.ReadFile(crt) + if errors.Is(err, os.ErrNotExist) { + return false, nil + } + if err != nil { + return false, err + } + keyPEM, err := os.ReadFile(key) + if errors.Is(err, os.ErrNotExist) { + return false, nil + } + if err != nil { + return false, err + } + if _, err := tlsPairParses(certPEM, keyPEM); err != nil { + return false, nil + } + return true, nil +} + +func tlsPairParses(certPEM, keyPEM []byte) (*x509.Certificate, error) { + block, _ := pem.Decode(certPEM) + if block == nil || block.Type != "CERTIFICATE" { + return nil, errors.New("not a certificate") + } + certificate, err := x509.ParseCertificate(block.Bytes) + if err != nil { + return nil, err + } + keyBlock, _ := pem.Decode(keyPEM) + if keyBlock == nil { + return nil, errors.New("not a key") + } + if _, err := x509.ParsePKCS8PrivateKey(keyBlock.Bytes); err != nil { + if _, err := x509.ParsePKCS1PrivateKey(keyBlock.Bytes); err != nil { + return nil, err + } + } + return certificate, nil +} + +func writeBusCertificate(crt, key string) error { + private, err := rsa.GenerateKey(rand.Reader, 2048) + if err != nil { + return err + } + serial, err := rand.Int(rand.Reader, new(big.Int).Lsh(big.NewInt(1), 128)) + if err != nil { + return err + } + template := &x509.Certificate{ + SerialNumber: serial, + Subject: pkix.Name{CommonName: busCertificateNames[0]}, + DNSNames: busCertificateNames, + IPAddresses: []net.IP{net.ParseIP("127.0.0.1")}, + NotBefore: time.Now().Add(-time.Hour), + NotAfter: time.Now().Add(busCertificateLife), + KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + BasicConstraintsValid: true, + } + der, err := x509.CreateCertificate(rand.Reader, template, template, &private.PublicKey, private) + if err != nil { + return err + } + pkcs8, err := x509.MarshalPKCS8PrivateKey(private) + if err != nil { + return err + } + + // **The key first, and only then the certificate**, so the pair a reader finds is never a + // certificate whose key has not been written yet — the one order in which an interruption + // leaves something that looks finished (novox/hq 04-ISSUES/014, a key present and unusable). + if err := os.WriteFile(key, pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: pkcs8}), 0o600); err != nil { + return err + } + if err := os.WriteFile(crt, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}), 0o644); err != nil { + return err + } + fmt.Printf("made the bus a certificate for %v, valid until %s\n %s\n %s\n", + busCertificateNames, template.NotAfter.Format(time.RFC3339), crt, key) + return nil +} diff --git a/modules/nats/cmd/mesh-nats-snapshot/certificate_test.go b/modules/nats/cmd/mesh-nats-snapshot/certificate_test.go new file mode 100644 index 00000000..e03df592 --- /dev/null +++ b/modules/nats/cmd/mesh-nats-snapshot/certificate_test.go @@ -0,0 +1,138 @@ +package main + +import ( + "crypto/tls" + "crypto/x509" + "os" + "path/filepath" + "strings" + "testing" +) + +// L3 s11 (design 47): the bus's certificate command, moved from mesh-controller with the same output, made once, and +// its check false without a usable pair. Its cases, as the controller's tests had them, are rows of one test. +func TestTheBusCertificate(t *testing.T) { + for _, row := range []struct { + name string + run func(*testing.T) + }{ + {"it loads as a server loads it", loadsAsAServersWould}, + {"it is made once", isMadeOnce}, + {"the check is false until there is a pair", checkIsFalseUntilAPair}, + {"a certificate without its key is not usable", withoutItsKeyIsNotUsable}, + {"where to write is required", whereToWriteIsRequired}, + } { + t.Run(row.name, row.run) + } +} + +// novox/hq 04-ISSUES/146. The bootstrap could not make the bus a certificate: it asked an image for +// `openssl` and the image it asks has none. What replaces it is this command, so what is checked is +// what the bootstrap needs from it — a pair a TLS server can actually load, made once and only once. + +func loadsAsAServersWould(t *testing.T) { + into := t.TempDir() + if err := busCertificate([]string{"--into", into}); err != nil { + t.Fatalf("the bus could not be given a certificate: %v", err) + } + + // The check a cheaper test would not make. The key was present and valid and the server could + // not start, once, because nothing loaded the pair the way a server loads it + // (novox/hq 04-ISSUES/014). + pair, err := tls.LoadX509KeyPair(filepath.Join(into, "tls.crt"), filepath.Join(into, "tls.key")) + if err != nil { + t.Fatalf("a TLS server cannot load what was written: %v", err) + } + leaf := pair.Leaf + if leaf == nil { + if leaf, err = x509.ParseCertificate(pair.Certificate[0]); err != nil { + t.Fatal(err) + } + } + if err := leaf.VerifyHostname("mesh-broker"); err != nil { + t.Errorf("the certificate is not for the name the bus is reached by: %v", err) + } + if len(leaf.IPAddresses) == 0 || leaf.IPAddresses[0].String() != "127.0.0.1" { + t.Errorf("the certificate does not cover the loopback address the foundation dials: %v", leaf.IPAddresses) + } + + // The key is not readable by anything else on the machine; the certificate is public and is. + key, err := os.Stat(filepath.Join(into, "tls.key")) + if err != nil { + t.Fatal(err) + } + if key.Mode().Perm() != 0o600 { + t.Errorf("the key is %v", key.Mode().Perm()) + } + crt, err := os.Stat(filepath.Join(into, "tls.crt")) + if err != nil { + t.Fatal(err) + } + if crt.Mode().Perm() != 0o644 { + t.Errorf("the certificate is %v, which the server runs as another user cannot read", crt.Mode().Perm()) + } +} + +// **Made once.** The step is applied again on every reconcile, and a second certificate is one the +// hosts that pinned the first no longer believe (novox/hq ADR 0004). +func isMadeOnce(t *testing.T) { + into := t.TempDir() + if err := busCertificate([]string{"--into", into}); err != nil { + t.Fatal(err) + } + first, err := os.ReadFile(filepath.Join(into, "tls.crt")) + if err != nil { + t.Fatal(err) + } + if err := busCertificate([]string{"--into", into}); err != nil { + t.Fatal(err) + } + again, err := os.ReadFile(filepath.Join(into, "tls.crt")) + if err != nil { + t.Fatal(err) + } + if string(first) != string(again) { + t.Fatal("running it twice replaced the certificate every host had pinned") + } +} + +// The verify half: false before, true after, which is what makes the bootstrap run the step at all. +func checkIsFalseUntilAPair(t *testing.T) { + into := t.TempDir() + if err := busCertificate([]string{"--check", "--into", into}); err == nil { + t.Fatal("an empty directory reported a usable certificate") + } + if err := busCertificate([]string{"--into", into}); err != nil { + t.Fatal(err) + } + if err := busCertificate([]string{"--check", "--into", into}); err != nil { + t.Fatalf("the certificate it just made does not satisfy its own check: %v", err) + } +} + +// A half-written pair is not a pair. An interrupted bootstrap leaves exactly this, and a step that +// called it done would hand the server a certificate with no key and report success. +func withoutItsKeyIsNotUsable(t *testing.T) { + into := t.TempDir() + if err := busCertificate([]string{"--into", into}); err != nil { + t.Fatal(err) + } + if err := os.Remove(filepath.Join(into, "tls.key")); err != nil { + t.Fatal(err) + } + if err := busCertificate([]string{"--check", "--into", into}); err == nil { + t.Fatal("a certificate with no key passed the check") + } + if err := busCertificate([]string{"--into", into}); err != nil { + t.Fatal(err) + } + if _, err := tls.LoadX509KeyPair(filepath.Join(into, "tls.crt"), filepath.Join(into, "tls.key")); err != nil { + t.Fatalf("it did not replace the unusable pair: %v", err) + } +} + +func whereToWriteIsRequired(t *testing.T) { + if err := busCertificate(nil); err == nil || !strings.Contains(err.Error(), "--into") { + t.Fatalf("it did not ask where to write: %v", err) + } +} diff --git a/modules/nats/cmd/mesh-nats-snapshot/connect.go b/modules/nats/cmd/mesh-nats-snapshot/connect.go index 97aad5b7..246a34fb 100644 --- a/modules/nats/cmd/mesh-nats-snapshot/connect.go +++ b/modules/nats/cmd/mesh-nats-snapshot/connect.go @@ -1,105 +1,16 @@ package main import ( - "crypto/sha256" - "crypto/tls" - "crypto/x509" - "encoding/hex" - "encoding/json" - "errors" - "fmt" - "io" - "os" - "strings" - "time" + "nats-tools/internal/busconn" "github.com/nats-io/nats.go" ) -// Credential is what the mesh seals to a module so it can reach the bus (mesh-controller -// cmd/mesh-controller issueWith): where, the server's certificate by fingerprint, who, and the -// password. The address is not used here — this runs beside the server, and the server is reached on -// loopback — but the fingerprint is: loopback or not, the certificate is the mesh's own. -type Credential struct { - URL string `json:"url"` - Fingerprint string `json:"fingerprint,omitempty"` - User string `json:"user"` - Password string `json:"password"` -} +// Credential is what the mesh seals to a module so it can reach the bus; see busconn. +type Credential = busconn.Credential -func readCredential(path string) (Credential, error) { - var raw []byte - var err error - if path == "-" { - raw, err = io.ReadAll(io.LimitReader(os.Stdin, 1<<20)) - } else { - raw, err = os.ReadFile(path) - } - if err != nil { - return Credential{}, fmt.Errorf("cannot read the credential: %w", err) - } - var c Credential - if err := json.Unmarshal(raw, &c); err != nil { - return Credential{}, errors.New("the credential is not the JSON the mesh seals to a module (url, fingerprint, user, password)") - } - if c.User == "" || c.Password == "" { - return Credential{}, errors.New("the credential names no user or password, so the bus would refuse it") - } - return c, nil -} +func readCredential(path string) (Credential, error) { return busconn.Read(path) } -// connect dials the bus as the credential's user. A `tls://` server is held to the credential's -// fingerprint and nothing else — the mesh's bus presents a certificate of its own, in no trust store, -// so a name check could only fail or be skipped. A plain `nats://` server is accepted only on -// loopback, and is what a restore's own private server is. func connect(server string, c Credential) (*nats.Conn, error) { - opts := []nats.Option{ - nats.UserInfo(c.User, c.Password), - nats.Name("mesh-nats-snapshot"), - // Answers come to the user's own inbox and no other: the bus grants every user its own - // `_INBOX..>` and nothing wider (novox/hq design 25 §4). - nats.CustomInboxPrefix("_INBOX." + c.User), - nats.Timeout(10 * time.Second), - // One attempt: a snapshot that cannot reach the bus fails the night, loudly, rather than - // waiting for it. - nats.NoReconnect(), - } - switch { - case strings.HasPrefix(server, "tls://"): - if c.Fingerprint == "" { - return nil, errors.New("the credential carries no fingerprint, so the bus's certificate cannot be checked") - } - opts = append(opts, nats.Secure(pinnedTo(c.Fingerprint))) - case strings.HasPrefix(server, "nats://127.0.0.1:"), strings.HasPrefix(server, "nats://localhost:"): - if c.Fingerprint != "" { - // The mesh's bus requires TLS; a plain connection to it would be refused anyway. - opts = append(opts, nats.Secure(pinnedTo(c.Fingerprint))) - } - default: - return nil, fmt.Errorf("%s: the bus is reached over tls:// (or plain nats:// on loopback, for a restore's own server)", server) - } - nc, err := nats.Connect(server, opts...) - if err != nil { - return nil, fmt.Errorf("cannot reach the bus at %s as %s: %w", server, c.User, err) - } - return nc, nil -} - -// pinnedTo accepts exactly the certificate with this SHA-256 — the same pin every machine holds -// (mesh-controller internal/broker PinnedToFingerprint). -func pinnedTo(want string) *tls.Config { - return &tls.Config{ - InsecureSkipVerify: true, //nolint:gosec // replaced by the pin below, which is stricter - MinVersion: tls.VersionTLS12, - VerifyPeerCertificate: func(rawCerts [][]byte, _ [][]*x509.Certificate) error { - if len(rawCerts) == 0 { - return errors.New("the bus presented no certificate") - } - sum := sha256.Sum256(rawCerts[0]) - if got := "sha256:" + hex.EncodeToString(sum[:]); got != want { - return fmt.Errorf("the bus presented a certificate this mesh does not know (%s)", got) - } - return nil - }, - } + return busconn.Connect(server, c, "mesh-nats-snapshot") } diff --git a/modules/nats/cmd/mesh-nats-snapshot/main.go b/modules/nats/cmd/mesh-nats-snapshot/main.go index f71e49d8..a3232c77 100644 --- a/modules/nats/cmd/mesh-nats-snapshot/main.go +++ b/modules/nats/cmd/mesh-nats-snapshot/main.go @@ -17,6 +17,7 @@ // mesh-nats-snapshot verify [--from FILE] // mesh-nats-snapshot restore --into DIR [--account MESH] [--from FILE] // mesh-nats-snapshot restore --server URL --credential FILE [--from FILE] +// mesh-nats-snapshot certificate [--check] --into DIR // // **Read-only on the live bus.** A snapshot neither pauses nor reconfigures a stream; the server // keeps accepting writes while it streams the copy out. The credential it uses is the bus's own @@ -47,6 +48,8 @@ func main() { err = runVerify(os.Args[2:]) case "restore": err = runRestore(os.Args[2:]) + case "certificate": + err = busCertificate(os.Args[2:]) case "-h", "--help", "help": usage() return @@ -67,6 +70,7 @@ func usage() { verify [--from bus.tar] every archive against the manifest restore --into DIR [--account MESH] [--from bus.tar] into a new store directory, by a server of its own restore --server URL --credential FILE [--from bus.tar] into a running server that holds none of them + certificate [--check] --into DIR the bus's self-signed certificate, made once The credential is the JSON the mesh seals to the bus's own module: url, fingerprint, user, password. `) diff --git a/modules/nats/cmd/nats-tools/bus.go b/modules/nats/cmd/nats-tools/bus.go new file mode 100644 index 00000000..ea37bc88 --- /dev/null +++ b/modules/nats/cmd/nats-tools/bus.go @@ -0,0 +1,41 @@ +package main + +import ( + "context" + "errors" + "fmt" + + stdio "git.novox.be/novox/mesh-sdk/go" + "github.com/nats-io/nats.go/jetstream" + "nats-tools/internal/busconn" +) + +// LiveBus is the bus as the module's own credential reaches it. Nothing connects until a tool that changes the +// bus is called, so the tools that read (through the monitoring endpoint) answer as they did with no credential. +// NATS_TOOLS_CREDENTIAL names the credential (a word of the bundle's own: the runtime strips MESH_BROKER_FILE) (the process nats-tools's own secret `broker-nats-tools`); MESH_NATS_SERVER where the bus is as +// seen from this machine (default: its own port on loopback). +func LiveBus() Bus { + return func(ctx context.Context) (jetstream.JetStream, func(), error) { + file := env("NATS_TOOLS_CREDENTIAL", "") + if file == "" { + return nil, nil, errors.New("NATS_TOOLS_CREDENTIAL is not set, so this tool has no credential to reach the bus with (the module's manifest sets it)") + } + c, err := busconn.Read(file) + if err != nil { + return nil, nil, fmt.Errorf("the credential at %s (NATS_TOOLS_CREDENTIAL) cannot be read: %w", file, err) + } + nc, err := busconn.Connect(env("MESH_NATS_SERVER", "tls://127.0.0.1:4222"), c, "nats-tools") + if err != nil { + return nil, nil, err + } + js, err := jetstream.New(nc) + if err != nil { + nc.Close() + return nil, nil, fmt.Errorf("JetStream on the bus: %w", err) + } + return js, nc.Close, nil + } +} + +// SaidByTheRuntime is the audit line: an event of this module, published through the runtime that launched it. +func SaidByTheRuntime(event string, body any) error { return stdio.Emit(event, body) } diff --git a/modules/nats/cmd/nats-tools/main.go b/modules/nats/cmd/nats-tools/main.go index 4de769dd..933abbea 100644 --- a/modules/nats/cmd/nats-tools/main.go +++ b/modules/nats/cmd/nats-tools/main.go @@ -1,6 +1,7 @@ // nats-tools: the mesh bus's own tools (novox/hq design 25), a Go bundle the node's runtime launches on the -// machine that runs the bus. Everything here reads — the server's monitoring API and the user list the -// controller composed — and nothing changes the bus. stdout is the MCP channel; this says nothing else. +// machine that runs the bus. Almost everything here reads — the server's monitoring API and the user list the +// controller composed. The two that change the bus (removal.go) are the only ones that connect to it, on the +// module's own credential, and say what they did. stdout is the MCP channel; this says nothing else. package main import ( @@ -86,7 +87,7 @@ func tools(monitor Monitor, users func(context.Context) ([]byte, error), written return Subscriptions(subsz, connz, arg(a, "subject"), limitOf(a, 100)) }}, {Name: "nats_streams", - Description: "JetStream: every stream with its subjects, retention, message count and size — or, given one stream, every consumer on it with what it has pending, unacknowledged and redelivered.", + Description: "JetStream: every stream with its subjects, retention, message count, size and size cap (max_bytes, when it has one) — or, given one stream, every consumer on it with what it has pending, unacknowledged and redelivered.", Input: map[string]any{"stream": str("one stream, e.g. EVENTS, CONTROL, ASSIGNMENTS, SEAT_NODE_BUILD_AGENT or KV_")}, Run: func(a map[string]any) (any, error) { doc, err := get(jsz) @@ -151,7 +152,7 @@ func tools(monitor Monitor, users func(context.Context) ([]byte, error), written } func main() { - if err := stdio.Serve("", tools(LiveMonitor(), ReadUsers, UsersWritten)); err != nil { + if err := stdio.Serve("", append(tools(LiveMonitor(), ReadUsers, UsersWritten), removalTools(LiveBus(), SaidByTheRuntime)...)); err != nil { fmt.Fprintln(os.Stderr, err) os.Exit(1) } diff --git a/modules/nats/cmd/nats-tools/manifest_test.go b/modules/nats/cmd/nats-tools/manifest_test.go index b9d699d0..2007b1ef 100644 --- a/modules/nats/cmd/nats-tools/manifest_test.go +++ b/modules/nats/cmd/nats-tools/manifest_test.go @@ -3,6 +3,7 @@ package main import ( "encoding/json" "os" + "slices" "strings" "testing" ) @@ -24,11 +25,14 @@ func TestTheBussManifestMountsItsProgramsIntoTheUpstreamImage(t *testing.T) { } `json:"backup"` } `json:"own"` } `json:"data"` + Tools []string `json:"tools"` + Emits []string `json:"emits"` Resources []map[string]any `json:"resources"` Build struct { On any `json:"on"` Artifacts []struct { Name, Kind, From, Language, System, Binary string + Env map[string]string } `json:"artifacts"` } `json:"build"` } @@ -38,7 +42,10 @@ func TestTheBussManifestMountsItsProgramsIntoTheUpstreamImage(t *testing.T) { if m.Build.On != nil { t.Error("build.on is declared: nothing is built on a base image any more") } - artifacts := map[string]struct{ Name, Kind, From, Language, System, Binary string }{} + artifacts := map[string]struct { + Name, Kind, From, Language, System, Binary string + Env map[string]string + }{} for _, a := range m.Build.Artifacts { artifacts[a.Name] = a if a.Kind == "image" { @@ -103,6 +110,23 @@ func TestTheBussManifestMountsItsProgramsIntoTheUpstreamImage(t *testing.T) { t.Errorf("%s is not in the container's restart-on: a new program would leave the old one running", p.resource) } } + // The tools that change the bus (issue 1952 N1-N3): listed, their audit events declared, and the credential + // they dial with the module's own secret `broker`. + for _, want := range []string{"nats_consumer_remove", "nats_bucket_remove"} { + if !slices.Contains(m.Tools, want) { + t.Errorf("tools do not list %s", want) + } + } + for _, want := range []string{"consumer-removed", "bucket-keys-removed"} { + if !slices.Contains(m.Emits, want) { + t.Errorf("emits do not list %s: the removal tools say it as the audit line", want) + } + } + for _, a := range m.Build.Artifacts { + if a.Name == "tools" && a.Env["NATS_TOOLS_CREDENTIAL"] != "${dir:tools-secret}/broker-nats-tools" { + t.Errorf("the tools' credential is %q: the process nats-tools' own secret broker-nats-tools, under a word of the bundle's own (the runtime strips MESH_BROKER_FILE)", a.Env["NATS_TOOLS_CREDENTIAL"]) + } + } args, _ := server["args"].([]any) if len(args) != 2 || args[0] != "sh" || args[1] != "/opt/mesh-entrypoint/entrypoint.sh" { t.Errorf("args are %v: the entrypoint run through sh, so no execute bit is relied on", args) diff --git a/modules/nats/cmd/nats-tools/removal.go b/modules/nats/cmd/nats-tools/removal.go new file mode 100644 index 00000000..3aff66e5 --- /dev/null +++ b/modules/nats/cmd/nats-tools/removal.go @@ -0,0 +1,246 @@ +package main + +// The two tools of this module that change the bus (novox/hq ADR 0564 clause 5, issue 1952 notes 6, 7 and 9): +// remove one named consumer, and remove one node's keys under a prefix of a key-value bucket. Everything else +// here reads. +// +// **Narrow by what the tool needs, and narrower by what the bus grants.** The tools reach the bus on the +// credential the module's manifest names (NATS_TOOLS_CREDENTIAL); the bus refuses whatever that credential is not +// granted, so a wrong stream, consumer or bucket is refused by the server and said here, not trusted to the +// arguments. Each tool refuses without `why`, reads what it would remove first (and says it, so a person can ask +// with dry_run and then for real), removes, reads that it is gone, and says what it did as an event of the module +// (the audit trail: who asked is in the call the controller audits, why and what are in the event). + +import ( + "context" + "errors" + "fmt" + "regexp" + "sort" + "strings" + "time" + + stdio "git.novox.be/novox/mesh-sdk/go" + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" +) + +// Bus opens JetStream on the bus as the module's credential for one call; close ends the connection. +type Bus func(ctx context.Context) (js jetstream.JetStream, close func(), err error) + +// Said keeps the audit line: an event of this module. +type Said func(event string, body any) error + +const ( + consumerRemoved = "consumer-removed" + bucketKeysRemoved = "bucket-keys-removed" + // maxKeysShown bounds the keys an answer lists; the count is always whole. + maxKeysShown = 200 +) + +var ( + // A stream, consumer or bucket is one subject token: no dot, no wildcard, no space. + name = regexp.MustCompile(`^[A-Za-z0-9_-]+$`) + // A key prefix is subject tokens and their dots; no wildcard, so it names keys and not a pattern. + keyPrefix = regexp.MustCompile(`^[A-Za-z0-9_./=-]+$`) +) + +func why(a map[string]any) (string, error) { + w := strings.TrimSpace(arg(a, "why")) + if w == "" { + return "", errors.New("why is required: say why this is removed, for the audit trail (novox/hq ADR 0564 clause 5); nothing was removed") + } + return w, nil +} + +// answerWithin is how long the bus is given to answer one request. **The bus does not refuse a request its +// credential is not granted: it drops it** (a permissions violation is told to the connection's error handler, +// never answered), so an unanswered request is how a refusal shows here. +var answerWithin = 10 * time.Second + +// unanswered says what a request with no answer most likely is. +func unanswered(err error, what string) error { + if errors.Is(err, context.DeadlineExceeded) || errors.Is(err, nats.ErrTimeout) { + return fmt.Errorf("%s: the bus did not answer within %s; it drops, without an answer, what the module's credential is not granted, "+ + "so this is most likely outside its grant (or the bus is not reachable): %w", what, answerWithin, err) + } + return fmt.Errorf("%s: %w", what, err) +} + +func step(ctx context.Context) (context.Context, context.CancelFunc) { + return context.WithTimeout(ctx, answerWithin) +} + +// ConsumerRemoved is what nats_consumer_remove answers and says. +type ConsumerRemoved struct { + Stream string `json:"stream"` + Consumer string `json:"consumer"` + Why string `json:"why"` + Was map[string]any `json:"was"` + Removed bool `json:"removed"` + At time.Time `json:"at"` +} + +// RemoveConsumer removes one named consumer of one stream, after reading it, and says so. +func RemoveConsumer(ctx context.Context, bus Bus, said Said, a map[string]any) (any, error) { + w, err := why(a) + if err != nil { + return nil, err + } + stream, consumer := arg(a, "stream"), arg(a, "consumer") + if !name.MatchString(stream) || !name.MatchString(consumer) { + return nil, errors.New("stream and consumer are required, each one name (letters, digits, - and _): nothing was removed") + } + js, closeBus, err := bus(ctx) + if err != nil { + return nil, err + } + defer closeBus() + sctx, cancel := step(ctx) + c, err := js.Consumer(sctx, stream, consumer) + cancel() + if err != nil { + if errors.Is(err, jetstream.ErrConsumerNotFound) || errors.Is(err, jetstream.ErrStreamNotFound) { + return nil, fmt.Errorf("%s has no consumer %s (%v): nothing was removed", stream, consumer, err) + } + return nil, unanswered(err, fmt.Sprintf("the consumer %s of %s could not be read, so it was not removed", consumer, stream)) + } + info := c.CachedInfo() + was := map[string]any{ + "durable": info.Config.Durable, "filter_subject": info.Config.FilterSubject, "filter_subjects": info.Config.FilterSubjects, + "deliver_subject": info.Config.DeliverSubject, "pending": info.NumPending, "ack_pending": info.NumAckPending, + "redelivered": info.NumRedelivered, "created": info.Created, + } + sctx, cancel = step(ctx) + err = js.DeleteConsumer(sctx, stream, consumer) + cancel() + if err != nil { + return nil, unanswered(err, fmt.Sprintf("consumer %s of %s was not removed", consumer, stream)) + } + sctx, cancel = step(ctx) + _, err = js.Consumer(sctx, stream, consumer) + cancel() + if err == nil { + return nil, fmt.Errorf("consumer %s of %s is still there after the removal was accepted", consumer, stream) + } + out := ConsumerRemoved{Stream: stream, Consumer: consumer, Why: w, Was: was, Removed: true, At: time.Now().UTC()} + if err := said(consumerRemoved, out); err != nil { + return nil, fmt.Errorf("consumer %s of %s WAS removed (%s) but the audit event could not be said: %w", consumer, stream, w, err) + } + return out, nil +} + +// KeysRemoved is what nats_bucket_remove answers and says. +type KeysRemoved struct { + Bucket string `json:"bucket"` + Prefix string `json:"prefix"` + Why string `json:"why"` + DryRun bool `json:"dry_run"` + Count int `json:"count"` + Keys []string `json:"keys"` + Cut bool `json:"keys_cut,omitempty"` + Removed int `json:"removed"` + At time.Time `json:"at"` +} + +// RemoveKeys removes the keys of a key-value bucket under a prefix: counted first, and only those. +func RemoveKeys(ctx context.Context, bus Bus, said Said, a map[string]any) (any, error) { + w, err := why(a) + if err != nil { + return nil, err + } + bucket, node, prefix := arg(a, "bucket"), arg(a, "node"), arg(a, "prefix") + if !name.MatchString(bucket) { + return nil, errors.New("bucket is required, a bucket's name as nats_buckets lists it: nothing was removed") + } + if (node == "") == (prefix == "") { + return nil, errors.New("give exactly one of node (that node's keys: those starting `.`) or prefix (keys starting so): " + + "a removal of a whole bucket is not this tool's; nothing was removed") + } + if node != "" { + if !name.MatchString(node) { + return nil, errors.New("node is one machine's name: nothing was removed") + } + prefix = node + "." + } + if !keyPrefix.MatchString(prefix) { + return nil, errors.New("prefix is key text without wildcards or spaces: nothing was removed") + } + dry, _ := a["dry_run"].(bool) + js, closeBus, err := bus(ctx) + if err != nil { + return nil, err + } + defer closeBus() + sctx, cancel := step(ctx) + kv, err := js.KeyValue(sctx, bucket) + cancel() + if err != nil { + return nil, unanswered(err, fmt.Sprintf("bucket %s could not be opened, nothing was removed", bucket)) + } + sctx, cancel = step(ctx) + stream, err := js.Stream(sctx, "KV_"+bucket) + cancel() + if err != nil { + return nil, unanswered(err, fmt.Sprintf("bucket %s has no stream to read its keys from", bucket)) + } + // The server narrows to the prefix's tokens when the prefix ends on a token; any other prefix is cut here. + filter := "$KV." + bucket + ".>" + if strings.HasSuffix(prefix, ".") { + filter = "$KV." + bucket + "." + prefix + ">" + } + sctx, cancel = step(ctx) + info, err := stream.Info(sctx, jetstream.WithSubjectFilter(filter)) + cancel() + if err != nil { + return nil, unanswered(err, fmt.Sprintf("the keys of %s could not be read, so none were removed", bucket)) + } + var keys []string + for subject := range info.State.Subjects { + if key := strings.TrimPrefix(subject, "$KV."+bucket+"."); strings.HasPrefix(key, prefix) { + keys = append(keys, key) + } + } + sort.Strings(keys) + out := KeysRemoved{Bucket: bucket, Prefix: prefix, Why: w, DryRun: dry, Count: len(keys), Keys: keys, At: time.Now().UTC()} + if len(keys) > maxKeysShown { + out.Keys, out.Cut = keys[:maxKeysShown], true + } + if want, ok := a["expect"].(float64); ok && int(want) != len(keys) { + return nil, fmt.Errorf("%d keys of %s start with %q, not the %d expected: nothing was removed", len(keys), bucket, prefix, int(want)) + } + if dry || len(keys) == 0 { + return out, nil + } + for _, k := range keys { + sctx, cancel := step(ctx) + err := kv.Purge(sctx, k) + cancel() + if err != nil { + return nil, unanswered(err, fmt.Sprintf("removing %q from %s failed after %d of %d keys, the rest were left", k, bucket, out.Removed, len(keys))) + } + out.Removed++ + } + if err := said(bucketKeysRemoved, out); err != nil { + return nil, fmt.Errorf("%d keys of %s under %q WERE removed (%s) but the audit event could not be said: %w", out.Removed, bucket, prefix, w, err) + } + return out, nil +} + +// removalTools are the module's two tools that change the bus; every other tool here reads. +func removalTools(bus Bus, said Said) []stdio.Tool { + run := func(f func(context.Context, Bus, Said, map[string]any) (any, error)) func(map[string]any) (any, error) { + return func(a map[string]any) (any, error) { return f(context.Background(), bus, said, a) } + } + return []stdio.Tool{ + {Name: "nats_consumer_remove", + Description: "Removes (deletes) one named JetStream consumer of one stream, e.g. the consumer `controller` of DEAD_LETTER_NOTICES at the lease write (novox/hq ADR 0564 clause 5). Needs `why`, kept in the audit trail; refuses an unknown stream or consumer; reads the consumer first and answers what it was (filters, pending, unacknowledged); the stream and its other consumers are not touched. The bus refuses any consumer the module's credential is not granted.", + Input: map[string]any{"stream": str("the stream, e.g. DEAD_LETTER_NOTICES"), "consumer": str("the consumer's name, e.g. controller"), "why": str("why it is removed, for the audit trail (required)")}, + Run: run(RemoveConsumer)}, + {Name: "nats_bucket_remove", + Description: "Removes (purges) keys from a key-value bucket: one node's keys (`node`, the keys starting `.`) or the keys under a `prefix`, e.g. a full acts bucket of the node-engine. Needs `why`, kept in the audit trail; never the whole bucket. Counts the keys first and lists them (`dry_run` only counts; `expect` refuses when the count differs); removes only those keys, and their history. See the buckets and their keys with nats_buckets. The bus refuses any bucket the module's credential is not granted.", + Input: map[string]any{"bucket": str("the bucket as nats_buckets lists it, e.g. mesh-host_acts"), "node": str("a node whose keys (`.`) go"), "prefix": str("key prefix instead of node"), + "why": str("why they are removed, for the audit trail (required)"), "dry_run": map[string]any{"type": "boolean", "description": "count and list the keys, remove none"}, "expect": num("the count of keys expected; refuses when it differs")}, + Run: run(RemoveKeys)}, + } +} diff --git a/modules/nats/cmd/nats-tools/removal_test.go b/modules/nats/cmd/nats-tools/removal_test.go new file mode 100644 index 00000000..2ed7799d --- /dev/null +++ b/modules/nats/cmd/nats-tools/removal_test.go @@ -0,0 +1,173 @@ +//go:build integration + +package main + +import ( + "context" + "fmt" + "os" + "path/filepath" + "sort" + "strings" + "testing" + "time" + + "github.com/nats-io/nats-server/v2/server" + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" +) + +// The two tools that change the bus, against a server shaped like the mesh's, on the credential the module's +// grant proposal gives them and nothing wider (issue 1952 N2, N3): the bus itself refuses what the grant leaves out, +// so this is also the proof that the proposed grant is enough and no wider than it needs to be. +func TestTheRemovalToolsChangeOnlyWhatTheyAreAskedAndTheGrantAllows(t *testing.T) { + dir := t.TempDir() + conf := filepath.Join(dir, "nats.conf") + if err := os.WriteFile(conf, []byte(fmt.Sprintf(`listen: "127.0.0.1:-1" +jetstream { store_dir: %q } +accounts { MESH { jetstream: enabled, users: [ + { user: "admin", password: "admin" }, + { user: "nats-tools", password: "tools", permissions: { + publish: { allow: [ + "$JS.API.CONSUMER.INFO.DEAD_LETTER_NOTICES.controller", + "$JS.API.CONSUMER.DELETE.DEAD_LETTER_NOTICES.controller", + "$JS.API.STREAM.INFO.KV_mesh-host_acts", + "$KV.mesh-host_acts.>" ] }, + subscribe: { allow: ["_INBOX.nats-tools.>"] } } } +] } } +`, filepath.Join(dir, "js"))), 0o600); err != nil { + t.Fatal(err) + } + opts, err := server.ProcessConfigFile(conf) + if err != nil { + t.Fatal(err) + } + opts.NoLog, opts.NoSigs = true, true + s, err := server.NewServer(opts) + if err != nil { + t.Fatal(err) + } + go s.Start() + if !s.ReadyForConnections(10 * time.Second) { + t.Fatal("the server did not start") + } + defer s.Shutdown() + + answerWithin = 2 * time.Second + ctx := context.Background() + admin, err := nats.Connect(s.ClientURL(), nats.UserInfo("admin", "admin")) + if err != nil { + t.Fatal(err) + } + defer admin.Close() + ajs, _ := jetstream.New(admin) + if _, err := ajs.CreateStream(ctx, jetstream.StreamConfig{Name: "DEAD_LETTER_NOTICES", Subjects: []string{"dl.>"}, Retention: jetstream.WorkQueuePolicy}); err != nil { + t.Fatal(err) + } + for _, c := range []struct{ name, filter string }{{"controller", "dl.a"}, {"watchman", "dl.b"}} { + if _, err := ajs.CreateConsumer(ctx, "DEAD_LETTER_NOTICES", jetstream.ConsumerConfig{Durable: c.name, FilterSubject: c.filter, AckPolicy: jetstream.AckExplicitPolicy}); err != nil { + t.Fatal(err) + } + } + acts, err := ajs.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: "mesh-host_acts"}) + if err != nil { + t.Fatal(err) + } + other, _ := ajs.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: "other_state"}) + for _, k := range []string{"g14.1", "g14.2", "g14.sub.3", "g14x.4", "ace.5"} { + acts.Put(ctx, k, []byte("v")) + } + other.Put(ctx, "g14.1", []byte("v")) + + bus := func(context.Context) (jetstream.JetStream, func(), error) { + nc, err := nats.Connect(s.ClientURL(), nats.UserInfo("nats-tools", "tools"), nats.CustomInboxPrefix("_INBOX.nats-tools")) + if err != nil { + return nil, nil, err + } + js, err := jetstream.New(nc) + return js, nc.Close, err + } + var said []string + say := func(event string, body any) error { said = append(said, event); return nil } + keys := func(kv jetstream.KeyValue) string { + ks, _ := kv.Keys(ctx) + sort.Strings(ks) + return strings.Join(ks, ",") + } + + // N2: the consumer goes, the stream and its other consumer stay; refused without why and for an unknown one. + if _, err := RemoveConsumer(ctx, bus, say, map[string]any{"stream": "DEAD_LETTER_NOTICES", "consumer": "controller"}); err == nil || !strings.Contains(err.Error(), "why is required") { + t.Fatalf("without why: %v", err) + } + if _, err := ajs.Consumer(ctx, "DEAD_LETTER_NOTICES", "controller"); err != nil { + t.Fatal("refused without why, but the consumer is gone") + } + // An unknown consumer: the granted name that does not exist is answered "not found"; an ungranted one is dropped. + if _, err := RemoveConsumer(ctx, bus, say, map[string]any{"stream": "DEAD_LETTER_NOTICES", "consumer": "nobody", "why": "x"}); err == nil || !strings.Contains(err.Error(), "not grant") && !strings.Contains(err.Error(), "no consumer") { + t.Fatalf("an unknown consumer: %v", err) + } + if _, err := RemoveConsumer(ctx, bus, say, map[string]any{"stream": "DEAD_LETTER_NOTICES", "consumer": "watchman", "why": "x"}); err == nil { + t.Fatal("a consumer outside the grant was removed") + } else if !strings.Contains(err.Error(), "grant") { + t.Fatalf("a consumer outside the grant: the refusal does not say so: %v", err) + } + if _, err := ajs.Consumer(ctx, "DEAD_LETTER_NOTICES", "watchman"); err != nil { + t.Fatalf("the grant did not hold, the watchman's consumer: %v", err) + } + got, err := RemoveConsumer(ctx, bus, say, map[string]any{"stream": "DEAD_LETTER_NOTICES", "consumer": "controller", "why": "the lease write"}) + if err != nil { + t.Fatal(err) + } + if r := got.(ConsumerRemoved); !r.Removed || r.Was["filter_subject"] != "dl.a" || r.Why != "the lease write" { + t.Fatalf("%+v", r) + } + if _, err := ajs.Consumer(ctx, "DEAD_LETTER_NOTICES", "controller"); err == nil { + t.Fatal("the consumer is still there") + } + if _, err := ajs.Consumer(ctx, "DEAD_LETTER_NOTICES", "watchman"); err != nil { + t.Fatalf("the other consumer went too: %v", err) + } + if _, err := RemoveConsumer(ctx, bus, say, map[string]any{"stream": "DEAD_LETTER_NOTICES", "consumer": "controller", "why": "again"}); err == nil || !strings.Contains(err.Error(), "no consumer") { + t.Fatalf("a consumer already gone: %v", err) + } + + // N3: only the node's keys go; counted first; refused without why, for a whole bucket, and outside the grant. + if _, err := RemoveKeys(ctx, bus, say, map[string]any{"bucket": "mesh-host_acts", "node": "g14"}); err == nil || !strings.Contains(err.Error(), "why is required") { + t.Fatalf("without why: %v", err) + } + if _, err := RemoveKeys(ctx, bus, say, map[string]any{"bucket": "mesh-host_acts", "why": "x"}); err == nil { + t.Fatal("a removal naming neither node nor prefix was not refused") + } + if _, err := RemoveKeys(ctx, bus, say, map[string]any{"bucket": "other_state", "node": "g14", "why": "x"}); err == nil { + t.Fatal("a bucket outside the grant was opened") + } + if k := keys(acts); k != "ace.5,g14.1,g14.2,g14.sub.3,g14x.4" { + t.Fatalf("keys after the refusals: %s", k) + } + dry, err := RemoveKeys(ctx, bus, say, map[string]any{"bucket": "mesh-host_acts", "node": "g14", "why": "look", "dry_run": true}) + if err != nil || dry.(KeysRemoved).Count != 3 || dry.(KeysRemoved).Removed != 0 { + t.Fatalf("dry run: %+v %v", dry, err) + } + if p, err := RemoveKeys(ctx, bus, say, map[string]any{"bucket": "mesh-host_acts", "prefix": "g14.s", "why": "look", "dry_run": true}); err != nil || p.(KeysRemoved).Count != 1 { + t.Fatalf("a prefix that ends inside a token: %+v %v", p, err) + } + if _, err := RemoveKeys(ctx, bus, say, map[string]any{"bucket": "mesh-host_acts", "node": "g14", "why": "x", "expect": 2.0}); err == nil { + t.Fatal("a count that differs from the expected was not refused") + } + got, err = RemoveKeys(ctx, bus, say, map[string]any{"bucket": "mesh-host_acts", "node": "g14", "why": "the acts bucket is full", "expect": 3.0}) + if err != nil { + t.Fatal(err) + } + if r := got.(KeysRemoved); r.Removed != 3 || r.Count != 3 { + t.Fatalf("%+v", r) + } + if k := keys(acts); k != "ace.5,g14x.4" { + t.Fatalf("keys left: %s", k) + } + if k := keys(other); k != "g14.1" { + t.Fatalf("another bucket changed: %s", k) + } + if strings.Join(said, ",") != "consumer-removed,bucket-keys-removed" { + t.Fatalf("the audit events said: %v", said) + } +} diff --git a/modules/nats/cmd/nats-tools/tools.go b/modules/nats/cmd/nats-tools/tools.go index e725a070..88d5b8b9 100644 --- a/modules/nats/cmd/nats-tools/tools.go +++ b/modules/nats/cmd/nats-tools/tools.go @@ -351,6 +351,10 @@ func Streams(jsz []byte, stream string) (map[string]any, error) { if s.Config.MaxMsgsPerSubject > 0 { row["max_per_subject"] = s.Config.MaxMsgsPerSubject } + // The cap on the stream's size, so how full it is reads against it (-1 and 0 are no cap). + if s.Config.MaxBytes > 0 { + row["max_bytes"] = s.Config.MaxBytes + } if stream != "" { row["description"] = s.Config.Description var consumers []map[string]any diff --git a/modules/nats/cmd/nats-tools/tools_test.go b/modules/nats/cmd/nats-tools/tools_test.go index bd0e7d0f..2a2eca4a 100644 --- a/modules/nats/cmd/nats-tools/tools_test.go +++ b/modules/nats/cmd/nats-tools/tools_test.go @@ -1,12 +1,15 @@ package main import ( + "context" "encoding/json" "errors" "fmt" "strings" "testing" "time" + + "github.com/nats-io/nats.go/jetstream" ) const connz = `{"connections":[ @@ -19,7 +22,7 @@ const subsz = `{"num_subscriptions":3,"subscriptions_list":[ {"subject":"mesh.node.anchor.declare","msgs":1,"cid":1}]}` const jsz = `{"streams":2,"consumers":2,"messages":12,"bytes":300,"account_details":[{"stream_detail":[ - {"name":"EVENTS","config":{"subjects":["mesh.mod.*.event.>"],"retention":"limits","max_age":604800000000000,"max_msgs_per_subject":10000}, + {"name":"EVENTS","config":{"subjects":["mesh.mod.*.event.>"],"retention":"limits","max_age":604800000000000,"max_msgs_per_subject":10000,"max_bytes":1048576}, "state":{"messages":10,"bytes":200,"num_subjects":3,"consumer_count":2,"last_ts":"2026-10-04T10:00:00Z"}, "consumer_detail":[ {"name":"anchor_audit","config":{"filter_subject":"mesh.mod.*.event.>"},"num_pending":5,"num_ack_pending":1,"num_redelivered":2,"delivered":{"last_active":"2026-10-04T10:00:00Z"}}, @@ -77,7 +80,7 @@ func TestStreamsBacklogAndBuckets(t *testing.T) { } one, _ := Streams([]byte(jsz), "EVENTS") row := one["detail"].([]map[string]any)[0] - if row["max_age_hours"].(int64) != 168 || len(row["consumer_detail"].([]map[string]any)) != 2 { + if row["max_age_hours"].(int64) != 168 || row["max_bytes"].(int64) != 1048576 || len(row["consumer_detail"].([]map[string]any)) != 2 { t.Fatalf("%v", row) } if _, err := Streams([]byte(jsz), "NOPE"); err == nil { @@ -208,3 +211,47 @@ func TestAGrantIsInForceOnlyWhenTheServerReloadedAfterTheListWasWritten(t *testi t.Error("a write time that is not one was read") } } + +// **The tools that change the bus are two, each refuses before it reaches the bus, and every other tool reads** +// (issue 1952 N1-N3). A row per refusal: the bus is never opened for them. +func TestTheRemovalToolsRefuseBeforeTheyReachTheBus(t *testing.T) { + bus := func(context.Context) (jetstream.JetStream, func(), error) { + t.Fatal("the bus was opened for a call that must be refused") + return nil, nil, nil + } + said := func(string, any) error { t.Fatal("an audit event was said for a refused call"); return nil } + for _, c := range []struct { + tool string + args map[string]any + want string + }{ + {"nats_consumer_remove", map[string]any{"stream": "DEAD_LETTER_NOTICES", "consumer": "controller"}, "why is required"}, + {"nats_consumer_remove", map[string]any{"stream": "DEAD_LETTER_NOTICES", "consumer": "controller", "why": " "}, "why is required"}, + {"nats_consumer_remove", map[string]any{"stream": "A.B", "consumer": "c", "why": "x"}, "stream and consumer are required"}, + {"nats_consumer_remove", map[string]any{"stream": "S", "consumer": ">", "why": "x"}, "stream and consumer are required"}, + {"nats_bucket_remove", map[string]any{"bucket": "mesh-host_acts", "node": "g14"}, "why is required"}, + {"nats_bucket_remove", map[string]any{"bucket": "mesh-host_acts", "why": "x"}, "exactly one of node"}, + {"nats_bucket_remove", map[string]any{"bucket": "mesh-host_acts", "node": "g14", "prefix": "g", "why": "x"}, "exactly one of node"}, + {"nats_bucket_remove", map[string]any{"bucket": "mesh-host_acts", "prefix": "g*", "why": "x"}, "without wildcards"}, + {"nats_bucket_remove", map[string]any{"bucket": "", "node": "g14", "why": "x"}, "bucket is required"}, + } { + var run func(map[string]any) (any, error) + for _, tl := range removalTools(bus, said) { + if tl.Name == c.tool { + run = tl.Run + } + } + if run == nil { + t.Fatalf("no tool %s", c.tool) + } + if _, err := run(c.args); err == nil || !strings.Contains(err.Error(), c.want) { + t.Errorf("%s %v: %v, want a refusal saying %q", c.tool, c.args, err, c.want) + } + } + // The read tools stay read tools: none of the first ten takes a `why`, and the list is these ten. + for _, tl := range tools(nil, nil, nil) { + if _, ok := tl.Input["why"]; ok || strings.Contains(tl.Name, "remove") { + t.Errorf("%s changes the bus; only the removal tools may", tl.Name) + } + } +} diff --git a/modules/nats/go.mod b/modules/nats/go.mod index 9f7e38c0..8e261655 100644 --- a/modules/nats/go.mod +++ b/modules/nats/go.mod @@ -1,12 +1,12 @@ module nats-tools -go 1.25.0 +go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.7 - github.com/klauspost/compress v1.18.5 + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 + github.com/klauspost/compress v1.20.0 github.com/nats-io/nats-server/v2 v2.11.17 - github.com/nats-io/nats.go v1.51.0 + github.com/nats-io/nats.go v1.54.0 ) require ( @@ -14,9 +14,9 @@ require ( github.com/google/go-tpm v0.9.8 // indirect github.com/minio/highwayhash v1.0.4 // indirect github.com/nats-io/jwt/v2 v2.8.1 // indirect - github.com/nats-io/nkeys v0.4.15 // indirect + github.com/nats-io/nkeys v0.4.16 // indirect github.com/nats-io/nuid v1.0.1 // indirect - golang.org/x/crypto v0.50.0 // indirect - golang.org/x/sys v0.43.0 // indirect + golang.org/x/crypto v0.57.0 // indirect + golang.org/x/sys v0.48.0 // indirect golang.org/x/time v0.15.0 // indirect ) diff --git a/modules/nats/go.sum b/modules/nats/go.sum index b28aa2f0..a2b8e018 100644 --- a/modules/nats/go.sum +++ b/modules/nats/go.sum @@ -1,27 +1,27 @@ -git.novox.be/novox/mesh-sdk/go v0.1.7 h1:C0sTQmtTiyYH7bnqZb7PusXnqA37gKuT7Nqjn9gG47w= -git.novox.be/novox/mesh-sdk/go v0.1.7/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 h1:lKU3HjNXGiVbLW2hUgFjU/TMTUYMlNnnmZxG5T7biQA= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= github.com/google/go-tpm v0.9.8/go.mod h1:h9jEsEECg7gtLis0upRBQU+GhYVH6jMjrFxI8u6bVUY= -github.com/klauspost/compress v1.18.5 h1:/h1gH5Ce+VWNLSWqPzOVn6XBO+vJbCNGvjoaGBFW2IE= -github.com/klauspost/compress v1.18.5/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= +github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA= +github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI= github.com/minio/highwayhash v1.0.4 h1:asJizugGgchQod2ja9NJlGOWq4s7KsAWr5XUc9Clgl4= github.com/minio/highwayhash v1.0.4/go.mod h1:GGYsuwP/fPD6Y9hMiXuapVvlIUEhFhMTh0rxU3ik1LQ= github.com/nats-io/jwt/v2 v2.8.1 h1:V0xpGuD/N8Mi+fQNDynXohVvp7ZztevW5io8CUWlPmU= github.com/nats-io/jwt/v2 v2.8.1/go.mod h1:nWnOEEiVMiKHQpnAy4eXlizVEtSfzacZ1Q43LIRavZg= github.com/nats-io/nats-server/v2 v2.11.17 h1:GKEghcFK6A+aFx11Yf1LjgLC3txAwvyhnYzhBIQZA8I= github.com/nats-io/nats-server/v2 v2.11.17/go.mod h1:B1sFVz4StNosQ903ak4N1G01Fl/9f8e06mXpFIE2K24= -github.com/nats-io/nats.go v1.51.0 h1:ByW84XTz6W03GSSsygsZcA+xgKK8vPGaa/FCAAEHnAI= -github.com/nats-io/nats.go v1.51.0/go.mod h1:26HypzazeOkyO3/mqd1zZd53STJN0EjCYF9Uy2ZOBno= -github.com/nats-io/nkeys v0.4.15 h1:JACV5jRVO9V856KOapQ7x+EY8Jo3qw1vJt/9Jpwzkk4= -github.com/nats-io/nkeys v0.4.15/go.mod h1:CpMchTXC9fxA5zrMo4KpySxNjiDVvr8ANOSZdiNfUrs= +github.com/nats-io/nats.go v1.54.0 h1:vsXoOxjHp/GmPUN+EcI7uOf/uB+iAP+kEsAFNQN0yzA= +github.com/nats-io/nats.go v1.54.0/go.mod h1:y+DZoD1oBOYfZTU681eTUiUjI0vbqYGixNVFHcjHJ0k= +github.com/nats-io/nkeys v0.4.16 h1:rd5oAuLOb8mnAycB0xleuEBNS1pVVnN0fv/FF34Eypg= +github.com/nats-io/nkeys v0.4.16/go.mod h1:llLgWoI0o4z/Q57q2R1kHfmocyhGV6VG/U18Glg1Afs= github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= -golang.org/x/crypto v0.50.0 h1:zO47/JPrL6vsNkINmLoo/PH1gcxpls50DNogFvB5ZGI= -golang.org/x/crypto v0.50.0/go.mod h1:3muZ7vA7PBCE6xgPX7nkzzjiUq87kRItoJQM1Yo8S+Q= +golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M= +golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA= golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.43.0 h1:Rlag2XtaFTxp19wS8MXlJwTvoh8ArU6ezoyFsMyCTNI= -golang.org/x/sys v0.43.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo= +golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= diff --git a/modules/nats/internal/busconn/busconn.go b/modules/nats/internal/busconn/busconn.go new file mode 100644 index 00000000..09ada3c3 --- /dev/null +++ b/modules/nats/internal/busconn/busconn.go @@ -0,0 +1,106 @@ +// Package busconn dials the bus as a credential the mesh sealed to a module: the one place the module's +// programs learn how (the snapshot, and the tools that change the bus). +package busconn + +import ( + "crypto/sha256" + "crypto/tls" + "crypto/x509" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "strings" + "time" + + "github.com/nats-io/nats.go" +) + +// Credential is what the mesh seals to a module so it can reach the bus (mesh-controller +// cmd/mesh-controller issueWith): where, the server's certificate by fingerprint, who, and the +// password. The address is not used here — this runs beside the server, and the server is reached on +// loopback — but the fingerprint is: loopback or not, the certificate is the mesh's own. +type Credential struct { + URL string `json:"url"` + Fingerprint string `json:"fingerprint,omitempty"` + User string `json:"user"` + Password string `json:"password"` +} + +func Read(path string) (Credential, error) { + var raw []byte + var err error + if path == "-" { + raw, err = io.ReadAll(io.LimitReader(os.Stdin, 1<<20)) + } else { + raw, err = os.ReadFile(path) + } + if err != nil { + return Credential{}, fmt.Errorf("cannot read the credential: %w", err) + } + var c Credential + if err := json.Unmarshal(raw, &c); err != nil { + return Credential{}, errors.New("the credential is not the JSON the mesh seals to a module (url, fingerprint, user, password)") + } + if c.User == "" || c.Password == "" { + return Credential{}, errors.New("the credential names no user or password, so the bus would refuse it") + } + return c, nil +} + +// Connect dials the bus as the credential's user. A `tls://` server is held to the credential's +// fingerprint and nothing else — the mesh's bus presents a certificate of its own, in no trust store, +// so a name check could only fail or be skipped. A plain `nats://` server is accepted only on +// loopback, and is what a restore's own private server is. +func Connect(server string, c Credential, name string) (*nats.Conn, error) { + opts := []nats.Option{ + nats.UserInfo(c.User, c.Password), + nats.Name(name), + // Answers come to the user's own inbox and no other: the bus grants every user its own + // `_INBOX..>` and nothing wider (novox/hq design 25 §4). + nats.CustomInboxPrefix("_INBOX." + c.User), + nats.Timeout(10 * time.Second), + // One attempt: a program that cannot reach the bus fails, loudly, rather than waiting for it. + nats.NoReconnect(), + } + switch { + case strings.HasPrefix(server, "tls://"): + if c.Fingerprint == "" { + return nil, errors.New("the credential carries no fingerprint, so the bus's certificate cannot be checked") + } + opts = append(opts, nats.Secure(PinnedTo(c.Fingerprint))) + case strings.HasPrefix(server, "nats://127.0.0.1:"), strings.HasPrefix(server, "nats://localhost:"): + if c.Fingerprint != "" { + // The mesh's bus requires TLS; a plain connection to it would be refused anyway. + opts = append(opts, nats.Secure(PinnedTo(c.Fingerprint))) + } + default: + return nil, fmt.Errorf("%s: the bus is reached over tls:// (or plain nats:// on loopback, for a restore's own server)", server) + } + nc, err := nats.Connect(server, opts...) + if err != nil { + return nil, fmt.Errorf("cannot reach the bus at %s as %s: %w", server, c.User, err) + } + return nc, nil +} + +// PinnedTo accepts exactly the certificate with this SHA-256 — the same pin every machine holds +// (mesh-controller internal/broker PinnedToFingerprint). +func PinnedTo(want string) *tls.Config { + return &tls.Config{ + InsecureSkipVerify: true, //nolint:gosec // replaced by the pin below, which is stricter + MinVersion: tls.VersionTLS12, + VerifyPeerCertificate: func(rawCerts [][]byte, _ [][]*x509.Certificate) error { + if len(rawCerts) == 0 { + return errors.New("the bus presented no certificate") + } + sum := sha256.Sum256(rawCerts[0]) + if got := "sha256:" + hex.EncodeToString(sum[:]); got != want { + return fmt.Errorf("the bus presented a certificate this mesh does not know (%s)", got) + } + return nil + }, + } +} diff --git a/modules/nats/module.json b/modules/nats/module.json index 106aac92..d2d02efe 100644 --- a/modules/nats/module.json +++ b/modules/nats/module.json @@ -8,7 +8,10 @@ "provides": [ { "name": "mesh-bus", - "scope": "mesh" + "scope": "mesh", + "credential": { + "form": "none" + } } ], "claims": [ @@ -19,12 +22,16 @@ ], "bus-users": "/var/lib/nats-module/conf/accounts.conf", "own-secrets": { - "broker": "${dir:mesh-state}/broker" + "broker": "${dir:mesh-state}/broker", + "broker-nats-tools": "${dir:tools-secret}/broker-nats-tools" }, "capabilities": [ "container-runtime" ], - "emits": [], + "emits": [ + "consumer-removed", + "bucket-keys-removed" + ], "consumes": [], "listens": [ { @@ -45,7 +52,9 @@ "nats_buckets", "nats_users", "nats_user_can", - "nats_snapshot_verify" + "nats_snapshot_verify", + "nats_consumer_remove", + "nats_bucket_remove" ], "guards": [ 8222 @@ -86,6 +95,13 @@ "mode": "0700", "unknown": "program" }, + { + "id": "tools-secret", + "type": "directory", + "path": "/var/lib/mesh/nats-tools", + "mode": "0711", + "unknown": "program" + }, { "id": "snapshots", "type": "directory", @@ -190,11 +206,18 @@ "loads": [ "nats-tools" ], + "bus": { + "removes": [ + "DEAD_LETTER_NOTICES.controller", + "mesh-host_acts" + ] + }, "env": { "MESH_NATS_MONITOR": "http://127.0.0.1:8222", "MESH_NATS_CONTAINER": "mesh-broker-nats", "MESH_NATS_USERS_FILE": "/etc/nats/accounts.conf", - "MESH_NATS_SNAPSHOTS_DIR": "${dir:snapshots}" + "MESH_NATS_SNAPSHOTS_DIR": "${dir:snapshots}", + "NATS_TOOLS_CREDENTIAL": "${dir:tools-secret}/broker-nats-tools" } }, { diff --git a/modules/nfs-server/.gitignore b/modules/nfs-server/.gitignore new file mode 100644 index 00000000..3cbd8705 --- /dev/null +++ b/modules/nfs-server/.gitignore @@ -0,0 +1 @@ +nfs-server diff --git a/modules/nfs-server/cmd/nfs-server/backup.go b/modules/nfs-server/cmd/nfs-server/backup.go index 8bf3d6fd..99c7ba59 100644 --- a/modules/nfs-server/cmd/nfs-server/backup.go +++ b/modules/nfs-server/cmd/nfs-server/backup.go @@ -14,12 +14,12 @@ package main // been read, what is exported stays exported and the status says the list is not known: the bus being // slow is no reason to take the library away, and the look below still guards against a repository. // -// The parsing of mesh-controller.data's answer is a small copy of the mounts module's (DataFrom). Shared -// code belongs in mesh-sdk: moving both there is mesh-sdk issue 13. import ( "encoding/json" "fmt" + "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/health" "os" "path" "sort" @@ -46,68 +46,46 @@ type BackupFromBus struct { Error string `json:"error,omitempty"` } -// BackupFrom reads the holder of node-backup on one machine from mesh-controller.seats' answer, and the -// paths that module keeps data in on that machine from mesh-controller.data's. A machine whose backup seat -// nobody holds, and an answer with no data list, are errors: never an empty list read as "nothing to fear". -func BackupFrom(machine string, seats, data json.RawMessage) (string, []string, error) { - var s struct { - Answer *struct { - Seats []struct { - Seat string `json:"seat"` - Holders []struct { - Module string `json:"module"` - Node string `json:"node"` - } `json:"holders"` - } `json:"seats"` - } `json:"answer"` - } - if err := json.Unmarshal(seats, &s); err != nil || s.Answer == nil { - return "", nil, fmt.Errorf("the controller's seats answer could not be read: %v", err) +// BackupFrom reads the holder of node-backup on one machine from the placement seat's state of who holds which seat +// (novox/hq ADR 0571 clause 4), and the paths that module keeps data in on that machine from disk-load's readings +// (clause 3). A machine whose backup seat nobody holds, and no readings at all, are errors: never an empty list read as +// "nothing to fear". +func BackupFrom(machine string, keys map[string][]byte, readings []health.Reading) (string, []string, error) { + holders, err := delivery.HoldersOf(keys, BackupSeat, machine) + if err != nil { + return "", nil, err } holder := "" - for _, seat := range s.Answer.Seats { - if seat.Seat != BackupSeat { - continue - } - for _, h := range seat.Holders { - if h.Node == machine { - holder = h.Module - } + for _, h := range holders { + if h.Node == machine { + holder = h.Module } } if holder == "" { return "", nil, fmt.Errorf("no module holds %s on %s", BackupSeat, machine) } - var d struct { - Answer *struct { - Data *[]struct { - Machine string `json:"machine"` - Module string `json:"module"` - Path string `json:"path"` - } `json:"data"` - } `json:"answer"` - } - if err := json.Unmarshal(data, &d); err != nil || d.Answer == nil || d.Answer.Data == nil { - return "", nil, fmt.Errorf("the controller's data answer holds no data list (%v)", err) + if len(readings) == 0 { + return "", nil, fmt.Errorf("disk-load keeps no reading of this machine's data yet") } var paths []string - for _, item := range *d.Answer.Data { - if item.Module == holder && (item.Machine == "" || item.Machine == machine) && path.IsAbs(item.Path) { - paths = append(paths, path.Clean(item.Path)) + for _, r := range readings { + if r.Module == holder && (r.Node == "" || r.Node == machine) && path.IsAbs(r.Path) { + paths = append(paths, path.Clean(r.Path)) } } sort.Strings(paths) return holder, paths, nil } -// TakeBackup asks the controller over the bus, as the module, through the runtime. +// TakeBackup reads the bus, as the module, through the runtime. func (m *Machine) TakeBackup(machine string) { - m.TakeBackupWith(machine, askBus) + m.TakeBackupWith(machine, readingsOf, seatHolderKeys) } -// TakeBackupWith is TakeBackup with the ask given, for tests. A failed ask keeps the last holder and +// TakeBackupWith is TakeBackup with the ask and the reading of the seat-holders state given, for tests. A failed ask keeps the last holder and // paths and adds why. -func (m *Machine) TakeBackupWith(machine string, ask func(key string, body any) (json.RawMessage, error)) { +func (m *Machine) TakeBackupWith(machine string, readings func(node string) ([]health.Reading, error), + holderKeys func() (map[string][]byte, error)) { var last BackupFromBus if text, err := m.Read(BackupFile); err == nil { _ = json.Unmarshal([]byte(text), &last) @@ -115,10 +93,10 @@ func (m *Machine) TakeBackupWith(machine string, ask func(key string, body any) next := last next.Looked = m.Now().UTC().Format(time.RFC3339) next.Error = "" - seats, err := ask("seat:mesh-controller.seats", map[string]any{}) - var data json.RawMessage + seats, err := holderKeys() + var data []health.Reading if err == nil { - data, err = ask("seat:mesh-controller.data", map[string]any{"machine": machine}) + data, err = readings(machine) } if err == nil { var holder string diff --git a/modules/nfs-server/cmd/nfs-server/backup_test.go b/modules/nfs-server/cmd/nfs-server/backup_test.go index 7ae3298b..dd4da159 100644 --- a/modules/nfs-server/cmd/nfs-server/backup_test.go +++ b/modules/nfs-server/cmd/nfs-server/backup_test.go @@ -4,6 +4,8 @@ import ( "context" "encoding/json" "errors" + "git.novox.be/novox/mesh-sdk/go/catalogue" + "git.novox.be/novox/mesh-sdk/go/health" "strings" "testing" "time" @@ -81,8 +83,8 @@ func TestWithoutTheBackupListTheExportsStayAndItIsSaid(t *testing.T) { } func TestBackupFromTheSeatsAndTheData(t *testing.T) { - seats := json.RawMessage(`{"answer":{"seats":[{"seat":"node-backup","scope":"node","holders":[{"module":"restic","node":"ace"},{"module":"borg","node":"shanks"}]}]},"ok":true}`) - data := json.RawMessage(`{"answer":{"data":[{"machine":"ace","module":"restic","item":"repository","path":"/storage/restic"},{"machine":"ace","module":"restic","item":"state","path":"/var/lib/restic"},{"machine":"ace","module":"plex","item":"data","path":"/mnt/plex/data"}]},"ok":true}`) + seats := backupHolders("ace", "restic") + data := readingsOfPaths("ace", map[string]string{"restic/repository": "/storage/restic", "restic/state": "/var/lib/restic", "plex/data": "/mnt/plex/data"}) holder, paths, err := BackupFrom("ace", seats, data) if err != nil || holder != "restic" || strings.Join(paths, " ") != "/storage/restic /var/lib/restic" { t.Errorf("got %q %v %v", holder, paths, err) @@ -90,8 +92,8 @@ func TestBackupFromTheSeatsAndTheData(t *testing.T) { if _, _, err := BackupFrom("g14", seats, data); err == nil { t.Errorf("a machine whose backup seat nobody holds is not said") } - if _, _, err := BackupFrom("ace", seats, json.RawMessage(`{"ok":true}`)); err == nil { - t.Errorf("an answer with no data list reads as an empty list") + if _, _, err := BackupFrom("ace", seats, nil); err == nil { + t.Errorf("no readings read as an empty list") } } @@ -99,13 +101,13 @@ func TestBackupFromTheSeatsAndTheData(t *testing.T) { func TestAFailedAskKeepsTheLastBackupPaths(t *testing.T) { f := newFake(t) m := f.machine() - m.TakeBackupWith("ace", func(string, any) (json.RawMessage, error) { + m.TakeBackupWith("ace", func(string) ([]health.Reading, error) { return nil, errors.New("no responders") - }) + }, noHolders) f.backup("restic", "/storage/restic") - m.TakeBackupWith("ace", func(string, any) (json.RawMessage, error) { + m.TakeBackupWith("ace", func(string) ([]health.Reading, error) { return nil, errors.New("timeout") - }) + }, func() (map[string][]byte, error) { return backupHolders("ace", "restic"), nil }) var b BackupFromBus if err := json.Unmarshal([]byte(f.read(BackupFile)), &b); err != nil { t.Fatal(err) @@ -152,15 +154,13 @@ func TestAStaleOrUnknownBackupListIsSaidWithItsAge(t *testing.T) { t.Errorf("an unknown backup list is not a warning in the health: %+v", h) } - ok := func(key string, body any) (json.RawMessage, error) { - if strings.Contains(key, "seats") { - return json.RawMessage(`{"answer":{"seats":[{"seat":"node-backup","holders":[{"module":"restic","node":"hs"}]}]}}`), nil - } - return json.RawMessage(`{"answer":{"data":[{"machine":"hs","module":"restic","path":"/srv/restic"}]}}`), nil + ok := func(string) ([]health.Reading, error) { + return readingsOfPaths("hs", map[string]string{"restic/repository": "/srv/restic"}), nil } - m.TakeBackupWith("hs", ok) + m.TakeBackupWith("hs", ok, func() (map[string][]byte, error) { return backupHolders("hs", "restic"), nil }) now = now.Add(2 * time.Hour) - m.TakeBackupWith("hs", func(string, any) (json.RawMessage, error) { return nil, errors.New("the bus did not answer") }) + m.TakeBackupWith("hs", func(string) ([]health.Reading, error) { return nil, errors.New("the bus did not answer") }, + func() (map[string][]byte, error) { return backupHolders("hs", "restic"), nil }) s := m.Pass(context.Background(), nil) said := strings.Join(append(append([]string{}, s.Said...), s.Warnings...), "\n") if !strings.Contains(said, "2h0m0s") || !strings.Contains(said, "the bus did not answer") { @@ -171,3 +171,21 @@ func TestAStaleOrUnknownBackupListIsSaidWithItsAge(t *testing.T) { t.Errorf("the health does not say the backup list's age: %+v", h) } } + +// backupHolders is the placement seat's state with one holder of node-backup on one node. +func backupHolders(node, module string) map[string][]byte { + return map[string][]byte{"node-backup." + node: []byte(`{"v":1,"seat":"node-backup","scope":"node","node":"` + node + + `","holders":[{"module":"` + module + `","node":"` + node + `"}]}`)} +} + +func noHolders() (map[string][]byte, error) { return nil, errors.New("the state is not read yet") } + +// readingsOfPaths is disk-load's readings of one node's items, by "/", each at its path. +func readingsOfPaths(node string, items map[string]string) []health.Reading { + var out []health.Reading + for k, p := range items { + module, item, _ := strings.Cut(k, "/") + out = append(out, health.Reading{V: 1, Node: node, PlacedDataItem: catalogue.PlacedDataItem{Module: module, ID: item, Path: p}}) + } + return out +} diff --git a/modules/nfs-server/cmd/nfs-server/guards_test.go b/modules/nfs-server/cmd/nfs-server/guards_test.go index add729ff..457af6f6 100644 --- a/modules/nfs-server/cmd/nfs-server/guards_test.go +++ b/modules/nfs-server/cmd/nfs-server/guards_test.go @@ -4,7 +4,7 @@ package main import ( "context" - "encoding/json" + "git.novox.be/novox/mesh-sdk/go/health" "strings" "testing" ) @@ -63,12 +63,10 @@ func TestAFailedReloadIsTriedAgainWhenNothingElseAsks(t *testing.T) { } } -func TestBackupFromRefusesAnAnswerWithoutADataList(t *testing.T) { - seats := json.RawMessage(`{"answer":{"seats":[{"seat":"node-backup","holders":[{"module":"restic","node":"ace"}]}]}}`) - for _, data := range []string{`{"answer":{}}`, `{"answer":{"data":null}}`} { - if _, paths, err := BackupFrom("ace", seats, json.RawMessage(data)); err == nil { - t.Errorf("%s was read as the empty list %v: never \"nothing to fear\"", data, paths) - } +func TestBackupFromRefusesNoReadingsAtAll(t *testing.T) { + seats := backupHolders("ace", "restic") + if _, paths, err := BackupFrom("ace", seats, []health.Reading{}); err == nil { + t.Errorf("no readings were read as the empty list %v: never \"nothing to fear\"", paths) } } diff --git a/modules/nfs-server/cmd/nfs-server/main.go b/modules/nfs-server/cmd/nfs-server/main.go index 3406d5a8..6edda7de 100644 --- a/modules/nfs-server/cmd/nfs-server/main.go +++ b/modules/nfs-server/cmd/nfs-server/main.go @@ -16,10 +16,13 @@ import ( "os" "os/signal" "sort" + "strings" "syscall" "time" stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/health" ) // ExportsState is the state the exports are kept in on the bus; MountsState the `mounts` modules' state, @@ -88,9 +91,50 @@ func ShareExports(m *Machine) { } } -// askBus calls a seat's verb over the bus as this module, through the runtime. -func askBus(key string, body any) (json.RawMessage, error) { - return stdio.Ask(key, body) +// readingsOf reads disk-load's readings of one machine's data items, every key `..`. +func readingsOf(node string) ([]health.Reading, error) { + state := stdio.State(health.ReadingsRead) + names, err := state.Keys() + if err != nil { + return nil, err + } + var out []health.Reading + for _, k := range names { + if !strings.HasPrefix(k, node+".") { + continue + } + e, err := state.Get(k) + if err != nil { + return nil, err + } + if e == nil { + continue + } + if r, err := health.DecodeReading(e.Value); err == nil { + out = append(out, r) + } + } + return out, nil +} + +// seatHolderKeys reads the placement seat's state of who holds which seat, every key. +func seatHolderKeys() (map[string][]byte, error) { + state := stdio.State(delivery.SeatHoldersRead) + names, err := state.Keys() + if err != nil { + return nil, err + } + keys := map[string][]byte{} + for _, k := range names { + e, err := state.Get(k) + if err != nil { + return nil, err + } + if e != nil { + keys[k] = e.Value + } + } + return keys, nil } // TakeWants reads every node's `mounts` state from the bus and writes what is asked of this machine to diff --git a/modules/nfs-server/cmd/nfs-server/manifest_test.go b/modules/nfs-server/cmd/nfs-server/manifest_test.go index d43d5776..ae048eb2 100644 --- a/modules/nfs-server/cmd/nfs-server/manifest_test.go +++ b/modules/nfs-server/cmd/nfs-server/manifest_test.go @@ -2,6 +2,8 @@ package main import ( "encoding/json" + "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/health" "os" "regexp" "sort" @@ -137,7 +139,7 @@ func TestTheManifestCarriesGrantsTheRosterTheWantsAndTheRemovalOrder(t *testing. if err := json.Unmarshal([]byte(raw), &full); err != nil { t.Fatal(err) } - if len(full.Reads) != 1 || full.Reads[0] != MountsState { + if len(full.Reads) != 3 || full.Reads[0] != health.ReadingsRead || full.Reads[1] != delivery.SeatHoldersRead || full.Reads[2] != MountsState { t.Errorf("reads: %v", full.Reads) } roster := full.Facts["machines"] diff --git a/modules/nfs-server/go.mod b/modules/nfs-server/go.mod index 6e122025..ffeb87e5 100644 --- a/modules/nfs-server/go.mod +++ b/modules/nfs-server/go.mod @@ -1,5 +1,5 @@ module nfsserver -go 1.22 +go 1.26.0 -require git.novox.be/novox/mesh-sdk/go v0.1.7 +require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 diff --git a/modules/nfs-server/go.sum b/modules/nfs-server/go.sum index b474419e..09ad064e 100644 --- a/modules/nfs-server/go.sum +++ b/modules/nfs-server/go.sum @@ -1,2 +1,14 @@ -git.novox.be/novox/mesh-sdk/go v0.1.7 h1:C0sTQmtTiyYH7bnqZb7PusXnqA37gKuT7Nqjn9gG47w= -git.novox.be/novox/mesh-sdk/go v0.1.7/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 h1:lKU3HjNXGiVbLW2hUgFjU/TMTUYMlNnnmZxG5T7biQA= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA= +github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI= +github.com/nats-io/nats.go v1.54.0 h1:vsXoOxjHp/GmPUN+EcI7uOf/uB+iAP+kEsAFNQN0yzA= +github.com/nats-io/nats.go v1.54.0/go.mod h1:y+DZoD1oBOYfZTU681eTUiUjI0vbqYGixNVFHcjHJ0k= +github.com/nats-io/nkeys v0.4.16 h1:rd5oAuLOb8mnAycB0xleuEBNS1pVVnN0fv/FF34Eypg= +github.com/nats-io/nkeys v0.4.16/go.mod h1:llLgWoI0o4z/Q57q2R1kHfmocyhGV6VG/U18Glg1Afs= +github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= +github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= +golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M= +golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA= +golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo= +golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= diff --git a/modules/nfs-server/module.json b/modules/nfs-server/module.json index f9994d16..c7082176 100644 --- a/modules/nfs-server/module.json +++ b/modules/nfs-server/module.json @@ -13,7 +13,10 @@ { "name": "nfs-share", "scope": "mesh", - "identity": false + "identity": false, + "credential": { + "form": "none" + } } ], "data": { @@ -45,26 +48,24 @@ } ], "reads": [ + "disk-load.readings", + "mesh-delivery-assigner.seat-holders", "mounts.shares" ], - "invokes": [ - "seat:mesh-controller.seats", - "seat:mesh-controller.data" - ], - "config": { - "shares": { - "type": "string", - "description": "The folders this machine exports, as space-separated name=path; the operator's, with no default (novox/hq ADR 0263).", - "trusted": true, - "required": true - }, - "grants": { - "type": "string", - "description": "Which nodes may mount which share, and how: share=node:rw|ro[,node:rw|ro], entries separated by spaces, or none to export to no node whatever a node wants (novox/hq ADR 0263).", - "trusted": true, - "default": "none" - } + "config": { + "shares": { + "type": "string", + "description": "The folders this machine exports, as space-separated name=path; the operator's, with no default (novox/hq ADR 0263).", + "trusted": true, + "required": true }, + "grants": { + "type": "string", + "description": "Which nodes may mount which share, and how: share=node:rw|ro[,node:rw|ro], entries separated by spaces, or none to export to no node whatever a node wants (novox/hq ADR 0263).", + "trusted": true, + "default": "none" + } + }, "facts": { "machines": { "path": "/etc/nfs-server/machines", diff --git a/modules/nodejs/module.json b/modules/nodejs/module.json index c3573444..2f4a049a 100644 --- a/modules/nodejs/module.json +++ b/modules/nodejs/module.json @@ -7,7 +7,10 @@ "provides": [ { "name": "node-toolchain", - "reach": "machine" + "reach": "machine", + "credential": { + "form": "none" + } } ], "resources": [ diff --git a/modules/ollama/module.json b/modules/ollama/module.json index 2d5fb48c..312ef6bf 100644 --- a/modules/ollama/module.json +++ b/modules/ollama/module.json @@ -7,7 +7,10 @@ "provides": [ { "name": "model-access", - "scope": "node" + "scope": "node", + "credential": { + "form": "none" + } } ], "listens": [ diff --git a/modules/postgres/cmd/postgres-provider/harness_test.go b/modules/postgres/cmd/postgres-provider/harness_test.go index 26b0246c..7c04bdba 100644 --- a/modules/postgres/cmd/postgres-provider/harness_test.go +++ b/modules/postgres/cmd/postgres-provider/harness_test.go @@ -120,8 +120,8 @@ func TestProvisionerCreatesTheDatabaseThenItsExtensions(t *testing.T) { if err := a.Retire(ctx, "mesh_ace_letta", nil, "test", time.Now()); err != nil { t.Fatal(err) } - noDrop(t, f.statements()) - if !has(f.statements(), `NOLOGIN`) { + // On the operator's yes the login goes and the database stays (ADR 0570 clause 5). + if !has(f.statements(), `^DROP ROLE "mesh_ace_letta"$`) || has(f.statements(), `DROP DATABASE`) { t.Fatal(f.statements()) } } diff --git a/modules/postgres/cmd/postgres-provider/live_test.go b/modules/postgres/cmd/postgres-provider/live_test.go index 38f91ef5..5e847973 100644 --- a/modules/postgres/cmd/postgres-provider/live_test.go +++ b/modules/postgres/cmd/postgres-provider/live_test.go @@ -86,6 +86,14 @@ func TestLive(t *testing.T) { if err != nil || len(r.Rows) != 1 || cell(r.Rows[0], 0) != "1" { t.Fatal("retiring lost the data:", r, err) } + // The login is removed, its database kept by its keeper (ADR 0570 clause 5). + if has, _ := c.exists(ctx, "SELECT 1 FROM pg_roles WHERE rolname = "+Literal(p.As)); has { + t.Fatal("the retired login is still there") + } + if has, _ := c.exists(ctx, "SELECT 1 FROM pg_database d JOIN pg_roles r ON r.oid = d.datdba WHERE d.datname = "+ + Literal(p.As)+" AND r.rolname = "+Literal(p.As+"~retired")); !has { + t.Fatal("the database is not its keeper's") + } inv, err := a.Inventory(ctx) if err != nil { t.Fatal(err) diff --git a/modules/postgres/cmd/postgres-provider/provisioner.go b/modules/postgres/cmd/postgres-provider/provisioner.go index a0d4ed06..b9bbb02d 100644 --- a/modules/postgres/cmd/postgres-provider/provisioner.go +++ b/modules/postgres/cmd/postgres-provider/provisioner.go @@ -35,10 +35,14 @@ func (a provisioner) Create(ctx context.Context, p harness.Provision) error { if err := a.pg.CreateDatabaseAndRole(ctx, database, p.As, p.Password); err != nil { return err } + // A login retired and asked for again takes back what its keeper kept (retire_pg.go). + if err := a.pg.Reclaim(ctx, database, p.As); err != nil { + return err + } if err := a.pg.EnsureExtensions(ctx, database, extensions); err != nil { return err } - // The active mark: a retired consumer asked for again loses its mark to delete here, its LOGIN + // The active mark: a login locked before ADR 0570 and asked for again loses its mark to delete here, its LOGIN // already set again by the role statement above (novox/hq ADR 0230). if err := a.pg.MarkActive(ctx, p.As, p.Consumer); err != nil { return err @@ -47,11 +51,10 @@ func (a provisioner) Create(ctx context.Context, p harness.Provision) error { return nil } -// Retire locks the login, never drops (novox/hq issue 241, ADR 0230): the database is kept under its -// own name, the role marked retired with when and why (retire_pg.go). On 2026-10-04 a misread -// contributions file withdrew every consumer at once, and dropping made that a loss of seven databases. -// Deleting is `cleanup delete`, a person's act; taking a database out of service by hand is -// postgres_retire_database, which renames rather than drops. +// Retire removes the login on the operator's yes (novox/hq ADR 0570 clause 5, ADR 0114 §2): what it owns is handed +// to its keeper, its sessions end and the login is dropped; the database and its data are kept (retire_pg.go). +// Deleting is `cleanup delete`, a person's act; taking a database out of service by hand is postgres_retire_database, +// which renames rather than drops. func (a provisioner) Retire(ctx context.Context, as string, _ map[string]any, why string, at time.Time) error { if err := a.pg.RetireRole(ctx, as, why, at); err != nil { return err diff --git a/modules/postgres/cmd/postgres-provider/retire_pg.go b/modules/postgres/cmd/postgres-provider/retire_pg.go index e07a5640..24e911c8 100644 --- a/modules/postgres/cmd/postgres-provider/retire_pg.go +++ b/modules/postgres/cmd/postgres-provider/retire_pg.go @@ -1,20 +1,22 @@ package main -// What retired means in postgres (novox/hq ADR 0230). +// What retired means in postgres (novox/hq ADR 0570 clause 5, ADR 0114 §2 and §9; ADR 0230 for the delete). // -// **Retired is the login locked, the database kept, and the role marked.** `ALTER ROLE … NOLOGIN` — -// the consumer's login can no longer connect, as itself, to anything — and its open sessions are -// ended. The database, its owner, its grants and every byte in it stay exactly as they were: CONNECT is -// not revoked and the database still allows connections, because the backup contribution dumps every -// database and a retired one is still worth backing up, and because re-enabling must give it back as it -// was. The mark is the role's comment, a JSON object the mesh owns: +// **Retired is the login removed, the database kept under a keeper role, and the keeper marked.** Retiring happens +// only on the operator's yes to a retiring provision (the SDK's moved path, Gone). What the login owns, its database +// first, is handed to its keeper, a role of its own that cannot log in (`~retired`, KeeperOf), its open +// sessions are ended, and the login is dropped: nothing can connect as it any more, and no credential of it is left. +// The database, its grants to others and every byte in it stay: the backup contribution dumps every database and a +// retired one is still worth backing up. The mark is the keeper's comment, a JSON object the mesh owns: // -// {"mesh":"consumer","node":"ace"} active -// {"mesh":"consumer","node":"ace","retired":"2026-10-06T…Z","why":"…"} retired +// {"mesh":"consumer","node":"ace"} an active login +// {"mesh":"consumer","node":"ace","retired":"2026-10-06T…Z","why":"…","login":"…"} a keeper // -// Asked for again, the ordinary create sets LOGIN and the password again and writes the active mark. -// Deleted — only through `cleanup delete` — the database is dropped, then the role, unless it still owns -// another database (a set-aside copy), which is said and kept. +// Asked for again, the ordinary create makes the login again and its keeper hands everything back (Reclaim) and +// goes. Deleted — only through `cleanup delete` — the database is dropped, then the keeper. +// +// **A login retired before this record** was only locked (`NOLOGIN`, its own role marked retired); it is listed and +// deleted as before, and create sets its LOGIN again. // // **What the mesh made is recognised by its mark, or by its shape before the mark existed**: a role // valid until 'infinity' (only the mesh's role statement says that) that owns a database of its own @@ -41,6 +43,17 @@ type roleMark struct { Node string `json:"node,omitempty"` Retired string `json:"retired,omitempty"` Why string `json:"why,omitempty"` + // Login is, on a keeper, the login it keeps for. + Login string `json:"login,omitempty"` +} + +// KeeperOf is the role a retired login's database and objects are handed to: the login's name with `~retired`. +func KeeperOf(login string) (string, error) { + k := login + "~retired" + if len(k) > 63 { + return "", fmt.Errorf("%s is too long a login for a keeper role (%s is over postgres's 63 bytes)", login, k) + } + return k, nil } func readMark(comment string) (roleMark, bool) { @@ -64,8 +77,9 @@ func (c *Client) MarkActive(ctx context.Context, role, node string) error { return err } -// RetireRole locks the login, ends its sessions and marks it retired with when and why. Its database -// is not touched. A role that does not exist has nothing to retire. +// RetireRole removes a login on the operator's yes (ADR 0570 clause 5): what it owns is handed to its keeper, marked +// with when and why, its sessions end and the login is dropped. The database and its data are kept. A login that does +// not exist has nothing to retire. func (c *Client) RetireRole(ctx context.Context, role, why string, at time.Time) error { r, err := c.Query(ctx, "", "SELECT coalesce(shobj_description(oid, 'pg_authid'), '') FROM pg_roles WHERE rolname = "+ Literal(role)) @@ -76,10 +90,66 @@ func (c *Client) RetireRole(ctx context.Context, role, why string, at time.Time) return nil } prev, _ := readMark(cell(r.Rows[0], 0)) + keeper, err := KeeperOf(role) + if err != nil { + return err + } + has, err := c.exists(ctx, "SELECT 1 FROM pg_roles WHERE rolname = "+Literal(keeper)) + if err != nil { + return err + } + if !has { + if _, err := c.Query(ctx, "", fmt.Sprintf("CREATE ROLE %s NOLOGIN", Ident(keeper))); err != nil { + return err + } + } + if _, err := c.Query(ctx, "", MarkStatement(keeper, roleMark{Node: prev.Node, Retired: at.UTC().Format(time.RFC3339), + Why: why, Login: role})); err != nil { + return err + } + // No new session, and none left open. if err := c.LockRole(ctx, role); err != nil { return err } - _, err = c.Query(ctx, "", MarkStatement(role, roleMark{Node: prev.Node, Retired: at.UTC().Format(time.RFC3339), Why: why})) + // In its own database, where its objects are; the database itself is a shared object, handed with them. + in := "" + if own, err := c.exists(ctx, "SELECT 1 FROM pg_database WHERE datname = "+Literal(role)); err != nil { + return err + } else if own { + in = role + } + for _, sql := range []string{ + fmt.Sprintf("REASSIGN OWNED BY %s TO %s", Ident(role), Ident(keeper)), + fmt.Sprintf("DROP OWNED BY %s", Ident(role)), + } { + if _, err := c.Query(ctx, in, sql); err != nil { + return err + } + } + _, err = c.Query(ctx, "", fmt.Sprintf("DROP ROLE %s", Ident(role))) + return err +} + +// Reclaim gives a login made again what its keeper kept (ADR 0570 clause 5: a provision needed again): the keeper's +// database and objects are handed back, and the keeper goes. A login with no keeper has nothing to reclaim. +func (c *Client) Reclaim(ctx context.Context, database, role string) error { + keeper, err := KeeperOf(role) + if err != nil { + return err + } + has, err := c.exists(ctx, "SELECT 1 FROM pg_roles WHERE rolname = "+Literal(keeper)) + if err != nil || !has { + return err + } + for _, sql := range []string{ + fmt.Sprintf("REASSIGN OWNED BY %s TO %s", Ident(keeper), Ident(role)), + fmt.Sprintf("DROP OWNED BY %s", Ident(keeper)), + } { + if _, err := c.Query(ctx, database, sql); err != nil { + return err + } + } + _, err = c.Query(ctx, "", fmt.Sprintf("DROP ROLE %s", Ident(keeper))) return err } @@ -89,7 +159,8 @@ var setAside = regexp.MustCompile(`_deleted_(\d{8})$`) // of its own database. const InventoryStatement = `SELECT r.rolname, r.rolcanlogin, coalesce(shobj_description(r.oid, 'pg_authid'), '') AS mark, coalesce(r.rolvaliduntil = 'infinity', false) AS mesh_shaped, - (SELECT pg_database_size(d.datname) FROM pg_database d WHERE d.datname = r.rolname AND d.datdba = r.oid) AS size + (SELECT pg_database_size(d.datname) FROM pg_database d WHERE d.datdba = r.oid + AND (d.datname = r.rolname OR d.datname || '~retired' = r.rolname)) AS size FROM pg_roles r WHERE r.rolname !~ '^pg_' AND NOT r.rolsuper AND r.rolname <> ` + "'" + Reader + "'" + ` ORDER BY r.rolname` @@ -118,6 +189,9 @@ func (c *Client) Inventory(ctx context.Context) (harness.Inventory, error) { continue } at, _ := time.Parse(time.RFC3339, m.Retired) + if m.Login != "" { + name = m.Login // a keeper: listed under the login it keeps for + } inv.Retired = append(inv.Retired, harness.Retired{Consumer: name, Node: m.Node, RetiredAt: at, Why: m.Why, SizeBytes: sizeOf(size), Kind: harness.KindConsumer}) } @@ -160,7 +234,19 @@ func (c *Client) DeleteRetired(ctx context.Context, r harness.Retired) (int64, e return freed, err } if len(row.Rows) == 0 { - return freed, nil + // Retired by removing the login: its keeper goes with the database it kept. + keeper, err := KeeperOf(r.Consumer) + if err != nil { + return freed, err + } + if has, err := c.exists(ctx, "SELECT 1 FROM pg_roles WHERE rolname = "+Literal(keeper)); err != nil || !has { + return freed, err + } + if _, err := c.Query(ctx, "", fmt.Sprintf("DROP OWNED BY %s", Ident(keeper))); err != nil { + return freed, err + } + _, err = c.Query(ctx, "", fmt.Sprintf("DROP ROLE %s", Ident(keeper))) + return freed, err } owns, err := c.Query(ctx, "", "SELECT datname FROM pg_database WHERE datdba = (SELECT oid FROM pg_roles WHERE rolname = "+ Literal(r.Consumer)+")") diff --git a/modules/postgres/cmd/postgres-provider/retire_pg_test.go b/modules/postgres/cmd/postgres-provider/retire_pg_test.go index ee9a476a..75feec90 100644 --- a/modules/postgres/cmd/postgres-provider/retire_pg_test.go +++ b/modules/postgres/cmd/postgres-provider/retire_pg_test.go @@ -10,31 +10,74 @@ import ( "time" ) -func TestRetiringLocksMarksAndDropsNothing(t *testing.T) { +// ADR 0570 clause 5, on the operator's yes: the login's database and objects go to its keeper, marked; its sessions +// end; the login is dropped; the database is not. +func TestRetiringHandsTheDatabaseToAKeeperAndRemovesTheLogin(t *testing.T) { f, c := newFake(admin) f.answer(`shobj_description\(oid`, []string{"c"}, []string{`{"mesh":"consumer","node":"ace"}`}) - f.answer(`SELECT 1 FROM pg_roles`, []string{"?column?"}, []string{"1"}) + f.answer(`pg_roles WHERE rolname = 'mesh_ace_letta'$`, []string{"?column?"}, []string{"1"}) + f.answer(`pg_database WHERE datname = 'mesh_ace_letta'$`, []string{"?column?"}, []string{"1"}) at := time.Date(2026, 10, 6, 12, 0, 0, 0, time.UTC) - if err := c.RetireRole(ctx, "mesh_ace_letta", "the mesh stopped asking for it", at); err != nil { + if err := c.RetireRole(ctx, "mesh_ace_letta", "the operator approved its retirement", at); err != nil { t.Fatal(err) } - sql := f.statements() - if !has(sql, `ALTER ROLE "mesh_ace_letta" NOLOGIN`) || !has(sql, `pg_terminate_backend.*usename = 'mesh_ace_letta'`) || - !has(sql, `COMMENT ON ROLE "mesh_ace_letta" IS '\{"mesh":"consumer","node":"ace","retired":"2026-10-06T12:00:00Z","why":"the mesh stopped asking for it"\}'`) { - t.Fatal(strings.Join(sql, "\n")) + var order []string + for _, call := range f.calls { + switch { + case strings.HasPrefix(call.SQL, "SELECT"): + case strings.HasPrefix(call.SQL, "REASSIGN"), strings.HasPrefix(call.SQL, "DROP OWNED"): + order = append(order, call.Database+": "+call.SQL) + default: + order = append(order, call.SQL) + } } - noDrop(t, sql) - if has(sql, `REVOKE|ALLOW_CONNECTIONS|RENAME`) { - t.Fatal("retiring touched the database:", strings.Join(sql, "\n")) + if !has(f.statements(), `pg_terminate_backend.*usename = 'mesh_ace_letta'`) { + t.Fatal(strings.Join(f.statements(), "\n")) + } + want := []string{ + `CREATE ROLE "mesh_ace_letta~retired" NOLOGIN`, + `COMMENT ON ROLE "mesh_ace_letta~retired" IS '{"mesh":"consumer","node":"ace","retired":"2026-10-06T12:00:00Z","why":"the operator approved its retirement","login":"mesh_ace_letta"}'`, + `ALTER ROLE "mesh_ace_letta" NOLOGIN`, + `mesh_ace_letta: REASSIGN OWNED BY "mesh_ace_letta" TO "mesh_ace_letta~retired"`, + `mesh_ace_letta: DROP OWNED BY "mesh_ace_letta"`, + `DROP ROLE "mesh_ace_letta"`, + } + if strings.Join(order, "\n") != strings.Join(want, "\n") { + t.Fatalf("sent:\n%s\nwant:\n%s", strings.Join(order, "\n"), strings.Join(want, "\n")) + } + if has(f.statements(), `DROP DATABASE|REVOKE|ALLOW_CONNECTIONS|RENAME`) { + t.Fatal("retiring touched the database:", strings.Join(f.statements(), "\n")) } // No such role: nothing to retire, nothing done. f, c = newFake(admin) - if err := c.RetireRole(ctx, "gone", "x", at); err != nil || has(f.statements(), `ALTER|COMMENT`) { + if err := c.RetireRole(ctx, "gone", "x", at); err != nil || has(f.statements(), `ALTER|COMMENT|CREATE|DROP`) { t.Fatal(f.statements(), err) } } +// A login made again takes back what its keeper kept, and the keeper goes; with no keeper, nothing is sent. +func TestALoginMadeAgainReclaimsWhatItsKeeperKept(t *testing.T) { + f, c := newFake(admin) + if err := c.Reclaim(ctx, "mesh_ace_letta", "mesh_ace_letta"); err != nil || has(f.statements(), `REASSIGN|DROP`) { + t.Fatal(f.statements(), err) + } + f, c = newFake(admin) + f.answer(`pg_roles WHERE rolname = 'mesh_ace_letta~retired'$`, []string{"?column?"}, []string{"1"}) + if err := c.Reclaim(ctx, "mesh_ace_letta", "mesh_ace_letta"); err != nil { + t.Fatal(err) + } + var got []string + for _, call := range f.calls[1:] { + got = append(got, call.Database+": "+call.SQL) + } + want := `mesh_ace_letta: REASSIGN OWNED BY "mesh_ace_letta~retired" TO "mesh_ace_letta"` + "\n" + + `mesh_ace_letta: DROP OWNED BY "mesh_ace_letta~retired"` + "\n" + `postgres: DROP ROLE "mesh_ace_letta~retired"` + if strings.Join(got, "\n") != want { + t.Fatal(strings.Join(got, "\n")) + } +} + func TestTheInventoryIsWhatTheMeshMadeByMarkOrShape(t *testing.T) { f, c := newFake(admin) f.answer(`FROM pg_roles r`, []string{"rolname", "rolcanlogin", "mark", "mesh_shaped", "size"}, @@ -44,6 +87,7 @@ func TestTheInventoryIsWhatTheMeshMadeByMarkOrShape(t *testing.T) { []string{"locked_before", "f", "", "t", "400"}, []string{"hal_registry", "t", "", "f", "500"}, []string{"jochens", "t", "", "t", ""}, + []string{"mesh_ace_umami~retired", "f", `{"mesh":"consumer","node":"ace","retired":"2026-10-29T12:00:00Z","why":"approved","login":"mesh_ace_umami"}`, "f", "600"}, ) f.answer(`_deleted_`, []string{"datname", "size"}, []string{"mesh_novox_gitea_deleted_20261005", "7771827"}) inv, err := c.Inventory(ctx) @@ -53,7 +97,7 @@ func TestTheInventoryIsWhatTheMeshMadeByMarkOrShape(t *testing.T) { if strings.Join(inv.Active, ",") != "active_marked,active_shaped" { t.Fatalf("active: %v", inv.Active) } - if len(inv.Retired) != 3 { + if len(inv.Retired) != 4 { t.Fatalf("%+v", inv.Retired) } r := inv.Retired[0] @@ -64,7 +108,10 @@ func TestTheInventoryIsWhatTheMeshMadeByMarkOrShape(t *testing.T) { if r := inv.Retired[1]; r.Consumer != "locked_before" || !r.RetiredAt.IsZero() || r.Kind != harness.KindConsumer { t.Fatalf("found locked without a mark: %+v", r) } - if r := inv.Retired[2]; r.Kind != KindSetAside || r.RetiredAt.Format("20060102") != "20261005" || r.SizeBytes != 7771827 { + if r := inv.Retired[2]; r.Consumer != "mesh_ace_umami" || r.SizeBytes != 600 || r.Why != "approved" { + t.Fatalf("a keeper is listed under the login it keeps for: %+v", r) + } + if r := inv.Retired[3]; r.Kind != KindSetAside || r.RetiredAt.Format("20060102") != "20261005" || r.SizeBytes != 7771827 { t.Fatalf("set aside: %+v", r) } } @@ -97,6 +144,18 @@ func TestDeletingDropsTheDatabaseThenTheRole(t *testing.T) { if db < 0 || role < db { t.Fatal(sql) } + // A login removed on retiring: the database, then its keeper. + f, c = newFake(admin) + f.answer(`SELECT pg_database_size`, []string{"size"}, []string{"9000"}) + f.answer(`pg_roles WHERE rolname = 'mesh_ace_umami~retired'$`, []string{"?column?"}, []string{"1"}) + if _, err := c.DeleteRetired(ctx, harness.Retired{Consumer: "mesh_ace_umami", Kind: harness.KindConsumer}); err != nil { + t.Fatal(err) + } + sql = strings.Join(f.statements(), "\n") + db, keeper := strings.Index(sql, `DROP DATABASE "mesh_ace_umami"`), strings.Index(sql, `DROP ROLE "mesh_ace_umami~retired"`) + if db < 0 || keeper < db { + t.Fatal(sql) + } // A role still owning a set-aside database is kept, and said. f, c = newFake(admin) f.answer(`SELECT rolcanlogin`, []string{"rolcanlogin"}, []string{"f"}) diff --git a/modules/postgres/go.mod b/modules/postgres/go.mod index b1598d4e..fe566892 100644 --- a/modules/postgres/go.mod +++ b/modules/postgres/go.mod @@ -3,7 +3,7 @@ module postgres go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021114014-66e8fdaa62e2 + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 github.com/jackc/pgx/v5 v5.11.0 ) diff --git a/modules/postgres/go.sum b/modules/postgres/go.sum index 923b4ba3..3d4a2c90 100644 --- a/modules/postgres/go.sum +++ b/modules/postgres/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021114014-66e8fdaa62e2 h1:Cdl5N56QPfmFBoLHrDfvvyc5Q+PUnTjf3fgYD7imcco= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021114014-66e8fdaa62e2/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 h1:lKU3HjNXGiVbLW2hUgFjU/TMTUYMlNnnmZxG5T7biQA= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= diff --git a/modules/postgres/module.json b/modules/postgres/module.json index e8a59069..8f9f0579 100644 --- a/modules/postgres/module.json +++ b/modules/postgres/module.json @@ -9,6 +9,7 @@ "provides": [ { "name": "postgres-database", + "credential": {"form": "pair"}, "scope": "mesh", "identity": { "max": 63, @@ -195,6 +196,7 @@ "bound" ], "reads": [ - "mesh-controller.assignments" + "mesh-provisioner.plans", + "mesh-provisioner.provisions" ] } diff --git a/modules/redis/module.json b/modules/redis/module.json index 58e11064..4cc6c4e5 100644 --- a/modules/redis/module.json +++ b/modules/redis/module.json @@ -7,6 +7,9 @@ "scope": "mesh", "identity": { "in": "a Redis ACL user and key prefix" + }, + "credential": { + "form": "pair" } } ], diff --git a/modules/restic/cmd/restic-backups/.gitignore b/modules/restic/cmd/restic-backups/.gitignore new file mode 100644 index 00000000..066e2897 --- /dev/null +++ b/modules/restic/cmd/restic-backups/.gitignore @@ -0,0 +1 @@ +restic-backups diff --git a/modules/restic/cmd/restic-backups/backups.go b/modules/restic/cmd/restic-backups/backups.go index 03a4e598..f0954581 100644 --- a/modules/restic/cmd/restic-backups/backups.go +++ b/modules/restic/cmd/restic-backups/backups.go @@ -25,6 +25,7 @@ import ( "encoding/json" "errors" "fmt" + "git.novox.be/novox/mesh-sdk/go/health" "os" "os/exec" "path/filepath" @@ -32,6 +33,7 @@ import ( "slices" "strconv" "strings" + "sync" "syscall" "time" ) @@ -188,6 +190,15 @@ type Backups struct { // measuring run as processes of their own). Node is the machine's name, said in the events. Emit func(event string, body any) error Node string + // Readings is every reading disk-load keeps for this machine (its state `readings`, ADR 0571 clause 3); nil where the + // process has no runtime to read it. + Readings func() ([]health.Reading, error) + // PutGood writes one row of the state `good` (health.GoodRow): the night's process through its own bus account, the + // tools through the runtime; nil where there is no bus to write to. + PutGood func(key string, row health.GoodRow) error + + goodMu sync.Mutex + goodWrote map[string][]byte } // The locks, files in this module's state. The night runs in a process of its own and `now`, `restore` and a diff --git a/modules/restic/cmd/restic-backups/data.go b/modules/restic/cmd/restic-backups/data.go index a7a3d5fc..67fb4d46 100644 --- a/modules/restic/cmd/restic-backups/data.go +++ b/modules/restic/cmd/restic-backups/data.go @@ -1,26 +1,9 @@ -// The data the modules on this machine declare, measured (novox/hq ADR 0233). +// The data the modules on this machine declare (novox/hq ADR 0233), and what the readings say of them. // -// The mesh composes a second file beside the backup lines: every data item every module on the -// machine declares, one line each, -// -// item -// -// where covered-by is the path whose snapshot keeps it (the item itself, or the directory its dump -// writes into), or `-` when it is not backed up; protection is backup, redundancy, both, or none; and -// measure is how the item is measured. This holder measures each item that is not a cache — its size, -// its newest write, and the redundant storage it is on and whether that is healthy (ZFS, md, btrfs) — -// and says it, with each module's last good backup and the precision of the measurement, in -// `backed-up`. -// -// **Never an unbounded walk of something large.** Three methods, the item's own choice: -// -// - `walk` (the default, for small items): every file summed and the newest change found, in one walk -// as root — at most once a day, and stopped after walkFor or walkFiles, said as "measured partially"; -// - `dataset`: the ZFS dataset holding the path — its `used` from the filesystem's own counters — and -// the newest change among the path's top-level entries; hourly, and nothing is walked; -// - `shallow`: the newest change among the top-level entries only, and no size; hourly. The controller keeps the readings and says when an item shrinks, stops being written, -// -// goes without a backup, or is replaced by an empty copy of itself; this holder only measures. +// The mesh composes a file beside the backup lines: every data item every module on the machine declares, one line +// each, which mesh-sdk reads (catalogue.ParseDataItems). disk-load measures the items and keeps each reading and its +// seven-day peak as its own state (ADR 0571 clause 3); this holder reads them there and says each, with each module's +// last good backup, in `backed-up`. // // And it deletes, when a person has decided: an item the mesh retired — its module gone from this // machine — is removed by `backup_delete_retired`, and only after a last restore point of it has been @@ -32,104 +15,32 @@ import ( "encoding/json" "errors" "fmt" + "git.novox.be/novox/mesh-sdk/go/catalogue" + "git.novox.be/novox/mesh-sdk/go/health" "os" "path/filepath" "regexp" - "strconv" "strings" "sync" "time" ) -// Item is one data item a module declares. -type Item struct { - // Measure is walk, dataset or shallow. - Measure string `json:"measure,omitempty"` - Module string `json:"-"` - ID string `json:"item"` - Class string `json:"class"` - Path string `json:"path"` - CoveredBy string `json:"covered_by,omitempty"` - Protection string `json:"protection,omitempty"` -} +// Item is one data item a module declares, as placed on this machine (mesh-sdk's reading of the holder's data text). +type Item = catalogue.PlacedDataItem -// Redundancy is the redundant storage an item is on, as read here. -type Redundancy struct { - // Kind is zfs, md or btrfs. - Kind string `json:"kind"` - // Where is the pool, the array device or the filesystem. - Where string `json:"where"` - // Healthy is nil when its state could not be read. - Healthy *bool `json:"healthy"` - Said string `json:"said"` -} - -// Measured is what one measurement found. -type Measured struct { - SizeBytes *int64 `json:"size_bytes,omitempty"` - LastWrite *time.Time `json:"last_write,omitempty"` - MeasuredAt *time.Time `json:"measured_at,omitempty"` - // Precision says what the size is: "exact", "dataset : its whole size", "partial: …" (a lower - // bound, never compared), or "no size: …". - Precision string `json:"precision,omitempty"` - Error string `json:"error,omitempty"` - Redundancy *Redundancy `json:"redundancy,omitempty"` -} - -// ItemReport is one item as `backed-up` says it. +// ItemReport is one item as `backed-up` says it: the item, its newest reading from disk-load (ADR 0571 clause 3) and +// its newest good backup. type ItemReport struct { Item - Measured + health.Measured + // Peak is the largest size in the last seven days, on PeakDay. + Peak *int64 `json:"peak_bytes,omitempty"` + PeakDay string `json:"peak_day,omitempty"` LastBackup *time.Time `json:"last_backup,omitempty"` } -// The classes the mesh declares; a cache is listed and never measured. -const classCache = "cache" - var safePath = regexp.MustCompile(`^/[^\s'"\\$` + "`" + `]*$`) -// parseItems reads the composed data file into each module's items. A line this holder does not read -// is refused, naming the module, as a backup line is. -func parseItems(text string) (map[string][]Item, error) { - out := map[string][]Item{} - module := "" - for _, raw := range strings.Split(text, "\n") { - line := strings.TrimSpace(raw) - if line == "" { - continue - } - if m := moduleHeader.FindStringSubmatch(line); m != nil { - module = m[1] - continue - } - if strings.HasPrefix(line, "#") || module == "" { - continue - } - f := strings.Fields(line) - if len(f) < 5 || len(f) > 7 || f[0] != "item" || !safePath.MatchString(f[3]) || (f[4] != "-" && !safePath.MatchString(f[4])) { - return nil, fmt.Errorf("%s declares a data line this holder does not read: %s", module, line) - } - it := Item{Module: module, ID: f[1], Class: f[2], Path: f[3]} - if len(f) >= 6 { - it.Protection = f[5] - } - it.Measure = measureWalk - if len(f) == 7 { - it.Measure = f[6] - } - switch it.Measure { - case measureWalk, measureDataset, measureShallow: - default: - return nil, fmt.Errorf("%s declares a data line this holder does not read: %s", module, line) - } - if f[4] != "-" { - it.CoveredBy = f[4] - } - out[module] = append(out[module], it) - } - return out, nil -} - // Items is what the modules on this machine declare; none when the mesh composed no data file — an // older controller, which composes none. func (b *Backups) Items() (map[string][]Item, error) { @@ -143,257 +54,30 @@ func (b *Backups) Items() (map[string][]Item, error) { if err != nil { return nil, err } - return parseItems(string(raw)) + return catalogue.ParseDataItems(string(raw)) } -func (b *Backups) measuredFile() string { return filepath.Join(b.Where.State, "measured.json") } - -// Measurements is the newest measurement of each item, by module and item. -func (b *Backups) Measurements() map[string]map[string]Measured { - out := map[string]map[string]Measured{} - if raw, err := os.ReadFile(b.measuredFile()); err == nil { - _ = json.Unmarshal(raw, &out) +// Measurements is the newest reading disk-load keeps of each item on this machine, by module and item (its state +// `readings`, ADR 0571 clause 3); none where it cannot be read, which `backed-up` then says as no reading. +func (b *Backups) Measurements() map[string]map[string]health.Reading { + out := map[string]map[string]health.Reading{} + if b.Readings == nil { + return out + } + rs, err := b.Readings() + if err != nil { + b.Say("disk-load's readings could not be read: %v", err) + return out + } + for _, r := range rs { + if out[r.Module] == nil { + out[r.Module] = map[string]health.Reading{} + } + out[r.Module][r.ID] = r } return out } -// The ways an item is measured. -const ( - measureWalk = "walk" - measureDataset = "dataset" - measureShallow = "shallow" -) - -// The bounds on a walk, and how often one runs: a walk is for small items, and one that meets a large -// one stops and says so rather than loading the disks for hours. -var ( - walkFor = 10 * time.Minute - walkFiles = 2_000_000 - walkEvery = 23 * time.Hour -) - -// walkCommand sums the files under a path and finds its newest change, in one walk, as root — bounded -// by time and by files. One line out: " ". -func walkCommand(path string) string { - return fmt.Sprintf("timeout %d find '%s' -xdev -printf '%%y %%s %%T@\\n' 2>/dev/null | head -n %d | "+ - "awk 'BEGIN{s=0;m=0;n=0} {n++; if ($1==\"f\") s+=$2; if ($3>m) m=$3} END {printf \"%%d %%.0f %%d\\n\", s, m, n}'; "+ - "echo \"${PIPESTATUS[0]:-0}\"", int(walkFor.Seconds()), path, walkFiles) -} - -// topCommand is the newest change among a path and its top-level entries, and how many there are: -// one directory read, bounded. " ". -func topCommand(path string) string { - return "timeout 60 find '" + path + "' -maxdepth 1 -printf '%T@\\n' 2>/dev/null | " + - "awk 'BEGIN{m=0;n=0} {n++; if ($1>m) m=$1} END {printf \"%.0f %d\\n\", m, n-1}'" -} - -func epochTime(s string) *time.Time { - epoch, err := strconv.ParseInt(s, 10, 64) - if err != nil || epoch <= 0 { - return nil - } - t := time.Unix(epoch, 0).UTC() - return &t -} - -// measure is one item, measured now, by its own method. -func (b *Backups) measure(ctx context.Context, it Item) Measured { - at := b.Now().UTC() - m := Measured{MeasuredAt: &at} - if !b.exists(ctx, it.Path) { - m.Error = it.Path + " does not exist" - zero := int64(0) - m.SizeBytes, m.Precision = &zero, "exact" - return m - } - m.Redundancy = b.redundancyOf(ctx, it.Path) - switch it.Measure { - case measureDataset, measureShallow: - out, err := b.Run(ctx, "bash", "-c", topCommand(it.Path)) - if f := strings.Fields(out); err == nil && len(f) == 2 { - m.LastWrite = epochTime(f[0]) - } else if err != nil { - m.Error = err.Error() - } - m.Precision = "no size: the newest change among its top-level entries only" - if it.Measure == measureShallow { - return m - } - out, err = b.Run(ctx, "zfs", "list", "-Hp", "-o", "name,used", it.Path) - f := strings.Fields(out) - if err != nil || len(f) != 2 { - m.Precision = "no size: it is on no ZFS dataset to read one from; the newest change among its top-level entries only" - return m - } - used, err := strconv.ParseInt(f[1], 10, 64) - if err != nil { - return m - } - m.SizeBytes = &used - m.Precision = "dataset " + f[0] + ": its whole size, from the filesystem's counters; the newest change among the top-level entries" - return m - } - out, err := b.Run(ctx, "bash", "-c", walkCommand(it.Path)) - if err != nil { - m.Error = err.Error() - return m - } - lines := strings.Split(strings.TrimSpace(out), "\n") - f := strings.Fields(lines[0]) - if len(f) != 3 || len(lines) < 2 { - m.Error = "the measurement said " + strings.TrimSpace(out) - return m - } - size, err1 := strconv.ParseInt(f[0], 10, 64) - files, err2 := strconv.Atoi(f[2]) - if err1 != nil || err2 != nil { - m.Error = "the measurement said " + strings.TrimSpace(out) - return m - } - m.SizeBytes, m.LastWrite = &size, epochTime(f[1]) - m.Precision = "exact" - if status := strings.TrimSpace(lines[len(lines)-1]); status == "124" || files >= walkFiles { - m.Precision = fmt.Sprintf("partial: measured partially — stopped after %s or %d files; the size is a lower bound, "+ - "and this item wants measure: dataset or shallow", walkFor, walkFiles) - } - return m -} - -// redundancyOf is the redundant storage a path is on, read as root: a ZFS pool's health and its own -// verdict, an md array's members, a btrfs filesystem's error counters. Nil for storage with no -// redundancy this holder knows how to read — which, for an item said to be protected by redundancy, the -// controller says is no protection at all. -func (b *Backups) redundancyOf(ctx context.Context, path string) *Redundancy { - out, err := b.Run(ctx, "findmnt", "-no", "SOURCE,FSTYPE", "--target", path) - if err != nil { - return nil - } - f := strings.Fields(out) - if len(f) < 2 { - return nil - } - source, fstype := f[0], f[1] - yes, no := true, false - switch { - case fstype == "zfs": - pool, _, _ := strings.Cut(source, "/") - r := &Redundancy{Kind: "zfs", Where: pool} - health, err := b.Run(ctx, "zpool", "list", "-H", "-o", "health", pool) - if err != nil { - r.Said = "zpool list: " + err.Error() - return r - } - status, err := b.Run(ctx, "zpool", "status", "-x", pool) - if err != nil { - r.Said = "zpool status: " + err.Error() - return r - } - health, status = strings.TrimSpace(health), strings.TrimSpace(status) - r.Said = "health " + health + "; " + firstLineOf(status) - if health == "ONLINE" && strings.Contains(status, "is healthy") { - r.Healthy = &yes - } else { - r.Healthy = &no - r.Said = "health " + health + "; " + oneLineOf(status) - } - return r - case strings.HasPrefix(source, "/dev/md"): - device := strings.TrimPrefix(source, "/dev/") - r := &Redundancy{Kind: "md", Where: device} - mdstat, err := b.Run(ctx, "cat", "/proc/mdstat") - if err != nil { - r.Said = err.Error() - return r - } - healthy, said, found := mdHealth(mdstat, device) - if !found { - r.Said = device + " is not in /proc/mdstat" - return r - } - r.Said = said - if healthy { - r.Healthy = &yes - } else { - r.Healthy = &no - } - return r - case fstype == "btrfs": - r := &Redundancy{Kind: "btrfs", Where: source} - stats, err := b.Run(ctx, "btrfs", "device", "stats", "--check", path) - if err != nil { - r.Healthy, r.Said = &no, "device errors: "+oneLineOf(err.Error()) - return r - } - r.Healthy, r.Said = &yes, firstLineOf(stats) - return r - } - return nil -} - -var mdMembers = regexp.MustCompile(`\[([U_]+)\]`) - -// mdHealth reads one array's block of /proc/mdstat: healthy when every member is up and it is not -// recovering. -func mdHealth(mdstat, device string) (bool, string, bool) { - lines := strings.Split(mdstat, "\n") - for i, l := range lines { - if !strings.HasPrefix(l, device+" ") { - continue - } - block := l - for j := i + 1; j < len(lines) && strings.HasPrefix(lines[j], " "); j++ { - block += " " + strings.TrimSpace(lines[j]) - } - members := mdMembers.FindStringSubmatch(block) - healthy := members != nil && !strings.Contains(members[1], "_") && !strings.Contains(block, "recovery") - return healthy, oneLineOf(block), true - } - return false, "", false -} - -func firstLineOf(s string) string { - line, _, _ := strings.Cut(strings.TrimSpace(s), "\n") - return line -} - -func oneLineOf(s string) string { return strings.Join(strings.Fields(s), " ") } - -// MeasureAll measures every item that is not a cache, one at a time, and keeps what it found. An item -// measured by a walk is walked only when walks is true or its last walk is older than walkEvery: the -// hourly round reads counters, and a walk runs at most once a day. -func (b *Backups) MeasureAll(ctx context.Context, walks bool) error { - // The hourly round runs in its own process and the walk after a night in the night's (novox/hq issue 454). - release, err := b.hold(measureLock) - if err != nil { - return err - } - defer release() - items, err := b.Items() - if err != nil { - return err - } - before := b.Measurements() - found := map[string]map[string]Measured{} - for module, its := range items { - for _, it := range its { - if it.Class == classCache { - continue - } - if found[module] == nil { - found[module] = map[string]Measured{} - } - if was, ok := before[module][it.ID]; ok && it.Measure == measureWalk && !walks && was.MeasuredAt != nil && - b.Now().Sub(*was.MeasuredAt) < walkEvery { - found[module][it.ID] = was - continue - } - found[module][it.ID] = b.measure(ctx, it) - } - } - raw, _ := json.MarshalIndent(found, "", " ") - return os.WriteFile(b.measuredFile(), append(raw, '\n'), 0o600) -} - func (b *Backups) goodFile() string { return filepath.Join(b.Where.State, "good.json") } // Good is each module's newest good night: what a night that failed since does not erase. @@ -418,14 +102,18 @@ func (b *Backups) recordGood(module string, at time.Time) { if err := os.WriteFile(b.goodFile(), append(raw, '\n'), 0o600); err != nil { b.Say("recording %s's good night failed: %v", module, err) } + b.keepGood() } -// itemReports is every item of one module, with its newest measurement and its newest good backup: the +// itemReports is every item of one module, with its newest reading and its newest good backup: the // module's newest good night, where that night keeps the path that covers the item. -func itemReports(its []Item, measured map[string]Measured, paths []string, good time.Time) []ItemReport { +func itemReports(its []Item, measured map[string]health.Reading, paths []string, good time.Time) []ItemReport { out := []ItemReport{} for _, it := range its { - r := ItemReport{Item: it, Measured: measured[it.ID]} + r := ItemReport{Item: it} + if m, ok := measured[it.ID]; ok { + r.Measured, r.Peak, r.PeakDay = m.Measured, m.Peak, m.PeakAt + } if it.CoveredBy != "" && !good.IsZero() { for _, p := range paths { if p == it.CoveredBy { diff --git a/modules/restic/cmd/restic-backups/data_test.go b/modules/restic/cmd/restic-backups/data_test.go index 0834ad61..9e41411d 100644 --- a/modules/restic/cmd/restic-backups/data_test.go +++ b/modules/restic/cmd/restic-backups/data_test.go @@ -6,7 +6,8 @@ package main import ( "context" "errors" - "fmt" + "git.novox.be/novox/mesh-sdk/go/catalogue" + "git.novox.be/novox/mesh-sdk/go/health" "os" "path/filepath" "strings" @@ -28,53 +29,34 @@ func withData(t *testing.T, declared, data string) Where { } func TestTheComposedDataFileIsReadIntoEachModulesItems(t *testing.T) { - got, err := parseItems(composedData) + b := &Backups{Where: withData(t, composed, composedData), Now: time.Now, Say: quiet} + got, err := b.Items() must(t, err) if len(got["postgres"]) != 2 || got["postgres"][0].CoveredBy != "/var/lib/mesh-store/dumps" || got["postgres"][1].CoveredBy != "" || got["searxng"][0].Class != "cache" { t.Fatalf("%+v", got) } - for _, bad := range []string{"# pg\nitem a irreplaceable relative -\n", "# pg\nitem a irreplaceable /x\n", "# pg\nitem a b /x; rm -rf / -\n"} { - if _, err := parseItems(bad); err == nil || !strings.Contains(err.Error(), "pg declares a data line") { - t.Errorf("%q: %v", bad, err) - } - } - // An older controller composes no data file: nothing is measured, nothing fails. - b := &Backups{Where: placed(t, composed), Now: time.Now, Say: quiet} + // An older controller composes no data file: nothing is declared, nothing fails. + b = &Backups{Where: placed(t, composed), Now: time.Now, Say: quiet} if items, err := b.Items(); err != nil || len(items) != 0 { t.Fatalf("%v, %v", items, err) } } -// Every item but a cache is measured — its files' size and its newest change, in one walk as root — and -// `backed-up` says each with the newest good night of the module that keeps the path covering it. -func TestEveryItemButACacheIsMeasuredAndSaidWithItsLastGoodBackup(t *testing.T) { +// `backed-up` says every item with disk-load's newest reading and its peak, and the newest good night of the module that +// keeps the path covering it; an item with no reading yet is said with none. +func TestEveryItemIsSaidWithItsReadingAndItsLastGoodBackup(t *testing.T) { where := withData(t, composed, composedData) - var mu sync.Mutex - var walked []string - run := func(_ context.Context, name string, args ...string) (string, error) { - mu.Lock() - defer mu.Unlock() - switch { - case name == "test": - return "", nil - case name == "bash" && strings.Contains(args[1], "find '"): - walked = append(walked, args[1]) - if strings.Contains(args[1], "'/var/lib/mesh-store'") { - return "1073741824 1759744800 4000\n0\n", nil - } - return "5000 1759700000 3\n0\n", nil - case name == "restic": - return "[]", nil - } - return "", nil - } + size, peak := int64(1<<30), int64(2<<30) + at := time.Date(2026, 10, 6, 4, 0, 0, 0, time.UTC) + write := time.Unix(1759744800, 0).UTC() night := time.Date(2026, 10, 6, 3, 0, 0, 0, time.UTC) - b := &Backups{Where: where, Run: run, Now: func() time.Time { return night.Add(time.Hour) }, Say: quiet} - must(t, b.MeasureAll(context.Background(), true)) - if len(walked) != 2 { - t.Fatalf("walked %d paths, want the two that are not a cache: %v", len(walked), walked) - } + b := &Backups{Where: where, Now: func() time.Time { return night.Add(time.Hour) }, Say: quiet, + Run: func(context.Context, string, ...string) (string, error) { return "[]", nil }, + Readings: func() ([]health.Reading, error) { + return []health.Reading{{V: 1, Node: "ace", PlacedDataItem: catalogue.PlacedDataItem{Module: "postgres", ID: "store"}, + Measured: health.Measured{SizeBytes: &size, LastWrite: &write, MeasuredAt: &at, Precision: "exact"}, Peak: &peak, PeakAt: "2026-10-05"}}, nil + }} b.recordGood("postgres", night) got, err := b.BackedUp(context.Background(), "") must(t, err) @@ -89,12 +71,23 @@ func TestEveryItemButACacheIsMeasuredAndSaidWithItsLastGoodBackup(t *testing.T) } } } - if store == nil || store.SizeBytes == nil || *store.SizeBytes != 1<<30 || store.LastWrite == nil || - store.LastWrite.Unix() != 1759744800 || store.LastBackup == nil || !store.LastBackup.Equal(night) { + if store == nil || store.SizeBytes == nil || *store.SizeBytes != 1<<30 || store.Peak == nil || *store.Peak != 2<<30 || + store.LastWrite == nil || store.LastWrite.Unix() != 1759744800 || store.LastBackup == nil || !store.LastBackup.Equal(night) { t.Fatalf("the store is said as %+v", store) } - if dumps == nil || dumps.LastBackup != nil { - t.Fatalf("an item not backed up is said backed up: %+v", dumps) + if dumps == nil || dumps.LastBackup != nil || dumps.SizeBytes != nil { + t.Fatalf("an item with no reading or no backup is said with one: %+v", dumps) + } + // The same last good night is kept on the bus beside good.json, one row per item a backup path covers. + b.Node = "ace" + wrote := map[string]health.GoodRow{} + b.PutGood = func(k string, r health.GoodRow) error { wrote[k] = r; return nil } + b.recordGood("postgres", night) // a good night writes the rows, once; the same night again writes nothing + n := len(wrote) + b.recordGood("postgres", night) + row, ok := wrote[health.GoodKey("ace", "postgres", "store")] + if len(wrote) != 1 || n != 1 || !ok || !row.LastGood.Equal(night) || row.CoveredBy != "/var/lib/mesh-store/dumps" || row.V != health.GoodVersion { + t.Fatalf("the good rows are %+v", wrote) } } @@ -202,104 +195,3 @@ func TestARetiredItemIsDeletedOnlyAfterALastRestorePoint(t *testing.T) { t.Fatalf("%+v, %v", again, err) } } - -// The redundant storage under an item is read: a ZFS pool healthy, then degraded; an md array with a -// member missing; and plain storage, which is no redundancy at all. -func TestTheRedundantStorageUnderAnItemIsRead(t *testing.T) { - health, statusX := "ONLINE\n", "pool 'storage' is healthy\n" - fs := "storage/media zfs\n" - run := func(_ context.Context, name string, args ...string) (string, error) { - switch name { - case "findmnt": - return fs, nil - case "zpool": - if args[0] == "list" { - return health, nil - } - return statusX, nil - case "cat": - return "Personalities : [raid1]\nmd127 : active raid1 sdb1[1] sda1[0]\n 976630464 blocks super 1.2 [2/1] [U_]\n\nunused devices: \n", nil - } - return "", nil - } - b := &Backups{Run: run, Now: time.Now, Say: quiet} - r := b.redundancyOf(context.Background(), "/storage/media/movies") - if r == nil || r.Kind != "zfs" || r.Where != "storage" || r.Healthy == nil || !*r.Healthy { - t.Fatalf("a healthy pool: %+v", r) - } - health, statusX = "DEGRADED\n", " pool: storage\n state: DEGRADED\nstatus: One or more devices has been removed\n" - if r := b.redundancyOf(context.Background(), "/storage/media/movies"); r.Healthy == nil || *r.Healthy || !strings.Contains(r.Said, "DEGRADED") { - t.Fatalf("a degraded pool: %+v", r) - } - fs = "/dev/md127 ext4\n" - if r := b.redundancyOf(context.Background(), "/srv/x"); r == nil || r.Kind != "md" || r.Healthy == nil || *r.Healthy { - t.Fatalf("an array missing a member: %+v", r) - } - fs = "/dev/nvme0n1p3 ext4\n" - if r := b.redundancyOf(context.Background(), "/var/lib/x"); r != nil { - t.Fatalf("plain storage read as redundant: %+v", r) - } -} - -// Large items are never walked: a dataset is measured from the filesystem's counters and its top-level -// entries, a shallow item from its top-level entries only; a walk that meets more than its bound stops -// and says so; and a walk runs at most once a day — the hourly round keeps the last one. -func TestNothingLargeIsWalkedAndAWalkIsBoundedAndDaily(t *testing.T) { - data := "# media\nitem movies irreplaceable /storage/media/movies - redundancy dataset\n" + - "item meta rebuildable /mnt/plex/config /mnt/plex/config backup shallow\n" + - "item big valuable /srv/big /srv/big backup\n" - where := withData(t, composed, data) - var mu sync.Mutex - var walks []string - truncated := true - run := func(_ context.Context, name string, args ...string) (string, error) { - mu.Lock() - defer mu.Unlock() - line := name + " " + strings.Join(args, " ") - switch { - case name == "test": - return "", nil - case name == "zfs" && args[len(args)-1] == "/storage/media/movies": - return "storage/media\t97067690722560\n", nil - case name == "zfs": - return "", errors.New("not a ZFS dataset") - case name == "bash" && strings.Contains(line, "-maxdepth 1"): - return "1759744800 12673\n", nil - case name == "bash": - walks = append(walks, line) - if strings.Contains(line, "-xdev") && !strings.Contains(line, "'/srv/big'") { - t.Errorf("walked something declared not to be walked: %s", line) - } - if truncated { - return fmt.Sprintf("123 1759700000 %d\n141\n", walkFiles), nil - } - return "999 1759700000 10\n0\n", nil - } - return "", nil - } - now := time.Date(2026, 10, 6, 12, 0, 0, 0, time.UTC) - b := &Backups{Where: where, Run: run, Now: func() time.Time { return now }, Say: quiet} - must(t, b.MeasureAll(context.Background(), false)) - got := b.Measurements()["media"] - if m := got["movies"]; m.SizeBytes == nil || *m.SizeBytes != 97067690722560 || !strings.HasPrefix(m.Precision, "dataset storage/media") || m.LastWrite == nil { - t.Fatalf("the library from its dataset: %+v", m) - } - if m := got["meta"]; m.SizeBytes != nil || !strings.HasPrefix(m.Precision, "no size") || m.LastWrite == nil { - t.Fatalf("a shallow item: %+v", m) - } - if m := got["big"]; !strings.HasPrefix(m.Precision, "partial") { - t.Fatalf("a walk stopped at its bound: %+v", m) - } - // An hour later the hourly round reads counters again and walks nothing. - now = now.Add(time.Hour) - truncated = false - must(t, b.MeasureAll(context.Background(), false)) - if len(walks) != 1 { - t.Fatalf("walked %d times in two rounds an hour apart: %v", len(walks), walks) - } - // After the night, it walks. - must(t, b.MeasureAll(context.Background(), true)) - if len(walks) != 2 || b.Measurements()["media"]["big"].Precision != "exact" { - t.Fatalf("%d walks, %+v", len(walks), b.Measurements()["media"]["big"]) - } -} diff --git a/modules/restic/cmd/restic-backups/good.go b/modules/restic/cmd/restic-backups/good.go new file mode 100644 index 00000000..5ea3c180 --- /dev/null +++ b/modules/restic/cmd/restic-backups/good.go @@ -0,0 +1,128 @@ +package main + +// **Each item's last good night, on the bus** (novox/hq ADR 0571 clause 3, ADR 0564 §6): beside good.json, the module's +// own state `good` (health.GoodRow), one key `..`, written when a night's files change, so the +// self-check's probe of a stale backup reads it there. + +import ( + "encoding/json" + "path/filepath" + "strings" + "syscall" + "time" + "unsafe" + + "git.novox.be/novox/mesh-sdk/go/health" +) + +// GoodRows is the rows of this machine (health.GoodRow): every item a module declares that a path of the module's backup +// covers, with the module's newest good night. +func (b *Backups) GoodRows() (map[string]health.GoodRow, error) { + items, err := b.Items() + if err != nil { + return nil, err + } + declared, err := b.Declared() + if err != nil { + return nil, err + } + paths := map[string][]string{} + for _, d := range declared { + paths[d.Module] = d.Paths + } + good := b.Good() + out := map[string]health.GoodRow{} + for module, its := range items { + for _, it := range its { + if it.Class == "cache" || it.CoveredBy == "" { + continue + } + covered := false + for _, p := range paths[module] { + covered = covered || p == it.CoveredBy + } + if !covered { + continue + } + out[health.GoodKey(b.Node, module, it.ID)] = health.GoodRow{V: health.GoodVersion, Node: b.Node, Module: module, + Item: it.ID, CoveredBy: it.CoveredBy, LastGood: good[module].UTC()} + } + } + return out, nil +} + +// keepGood writes the rows that differ from what was last written, when a night succeeded (recordGood). A bus that +// cannot be reached loses nothing: good.json is the local copy, and the next good night writes the rows again. +func (b *Backups) keepGood() { + if b.PutGood == nil { + return + } + rows, err := b.GoodRows() + if err != nil { + b.Say("the last good nights could not be read: %v", err) + return + } + b.goodMu.Lock() + defer b.goodMu.Unlock() + if b.goodWrote == nil { + b.goodWrote = map[string][]byte{} + } + for k, r := range rows { + raw, _ := json.Marshal(r) + if string(b.goodWrote[k]) == string(raw) { + continue + } + if err := b.PutGood(k, r); err != nil { + b.Say("the last good night of %s was not put on the bus: %v", k, err) + continue + } + b.goodWrote[k] = raw + } +} + +// watchGood writes the rows once at start and then each time the night's process changes what it keeps in the state +// directory (good.json, the nights' file), as the file system says it: the night runs as the operator account, which a +// process with a bus account of its own may not (ADR 0436 §2), so the tools' runtime, which has the module's account, +// writes at the night's event instead of the night itself. No timer. +func watchGood(b *Backups) { + time.Sleep(20 * time.Second) // the runtime serves first + b.keepGood() + for { + err := whenWritten(b.Where.State, func(name string) bool { return name == "good.json" || name == filepath.Base(b.nightsFile()) }, b.keepGood) + b.Say("watching the state directory for a night failed (%v); trying again in a minute", err) + time.Sleep(time.Minute) + b.keepGood() + } +} + +// whenWritten calls then each time a file of dir that match names is written or put in place, until the watch fails. +func whenWritten(dir string, match func(name string) bool, then func()) error { + fd, err := syscall.InotifyInit1(syscall.IN_CLOEXEC) + if err != nil { + return err + } + defer syscall.Close(fd) + if _, err := syscall.InotifyAddWatch(fd, dir, syscall.IN_CLOSE_WRITE|syscall.IN_MOVED_TO); err != nil { + return err + } + buf := make([]byte, 64*syscall.SizeofInotifyEvent+4096) + for { + n, err := syscall.Read(fd, buf) + if err != nil { + if err == syscall.EINTR { + continue + } + return err + } + heard := false + for off := 0; off+syscall.SizeofInotifyEvent <= n; { + ev := (*syscall.InotifyEvent)(unsafe.Pointer(&buf[off])) + name := strings.TrimRight(string(buf[off+syscall.SizeofInotifyEvent:off+syscall.SizeofInotifyEvent+int(ev.Len)]), "\x00") + heard = heard || match(name) + off += syscall.SizeofInotifyEvent + int(ev.Len) + } + if heard { + then() + } + } +} diff --git a/modules/restic/cmd/restic-backups/health_test.go b/modules/restic/cmd/restic-backups/health_test.go index 838aa88e..76ce1550 100644 --- a/modules/restic/cmd/restic-backups/health_test.go +++ b/modules/restic/cmd/restic-backups/health_test.go @@ -263,8 +263,8 @@ func TestTheManifestRunsTheNightJudgedByItsHealthTool(t *testing.T) { } env := m.Build.Artifacts[0].Env want := map[string]struct{ run, kind, tool string }{ - "night": {"night", "tool", "backup_health"}, - "measure": {"measure", "unit", ""}, + "night": {"night", "unit", ""}, + "awake": {"awake", "tool", "backup_health"}, } for _, r := range m.Resources { w, ok := want[r["id"].(string)] diff --git a/modules/restic/cmd/restic-backups/main.go b/modules/restic/cmd/restic-backups/main.go index f4e08e32..20c0a76b 100644 --- a/modules/restic/cmd/restic-backups/main.go +++ b/modules/restic/cmd/restic-backups/main.go @@ -24,6 +24,8 @@ import ( "time" stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/announce" + "git.novox.be/novox/mesh-sdk/go/health" ) // Seat is the role this module holds. @@ -46,20 +48,29 @@ func main() { } if way == "" { b.Emit = func(event string, body any) error { return stdio.Emit(event, body) } + b.Readings = busReadings(b.Node) } switch way { case "night": nights(b) - case "measure": - measurements(b) + case "awake": + counting(b) case "": + if err := stdio.Keeps(announce.State{Name: health.GoodState}); err != nil { + say("%v", err) + os.Exit(1) + } + b.PutGood = func(key string, row health.GoodRow) error { + _, err := stdio.State(health.GoodState).Put(key, row) + return err + } + go watchGood(b) if err := stdio.Serve("", tools(b)); err != nil { say("%v", err) os.Exit(1) } default: - say("%q is not a way to run; with no argument it serves its tools, with night it runs the nights, with "+ - "measure it measures the data declared here", way) + say("%q is not a way to run; with no argument it serves its tools, with night it runs the nights, with awake it counts how long the machine is up since a night", way) os.Exit(2) } } @@ -72,7 +83,6 @@ func nights(b *Backups) { if h, err := strconv.Atoi(os.Getenv("MESH_BACKUP_HOUR")); err == nil && h >= 0 && h < 24 { hour = h } - go counting(b) if missedANight(b.Nights(), time.Now()) { time.Sleep(10 * time.Minute) night(b) @@ -83,23 +93,6 @@ func nights(b *Backups) { } } -// measurements measures every declared item once an hour (MESH_BACKUP_MEASURE_EVERY to change it), -// the first a few minutes after start — counters hourly, a walk at most daily — and every walk after -// each night (novox/hq ADR 0233). -func measurements(b *Backups) { - every := time.Hour - if d, err := time.ParseDuration(os.Getenv("MESH_BACKUP_MEASURE_EVERY")); err == nil && d >= time.Minute { - every = d - } - time.Sleep(3 * time.Minute) - for { - if err := b.MeasureAll(context.Background(), false); err != nil { - say("measuring the data declared here failed: %v", err) - } - time.Sleep(every) - } -} - func night(b *Backups) { ctx := context.Background() outcome, err := b.BackUp(ctx, "") @@ -116,9 +109,6 @@ func night(b *Backups) { if len(failed) > 0 { say("the night left %s without a backup", strings.Join(failed, ", ")) } - if err := b.MeasureAll(ctx, true); err != nil { - say("measuring the data declared here failed: %v", err) - } // Sundays, the repository's own integrity with a sample of the data read back. if time.Now().Weekday() == time.Sunday { if err := b.Check(ctx); err != nil { @@ -204,3 +194,28 @@ func tools(b *Backups) []stdio.Tool { Run: func(a map[string]any) (any, error) { return b.DeletedOutcome(arg(a, "path")) }}, } } + +// busReadings reads disk-load's readings of this machine from its state, every key `..`. +func busReadings(node string) func() ([]health.Reading, error) { + return func() ([]health.Reading, error) { + st := stdio.State(health.ReadingsRead) + keys, err := st.Keys() + if err != nil { + return nil, err + } + var out []health.Reading + for _, k := range keys { + if !strings.HasPrefix(k, node+".") { + continue + } + e, err := st.Get(k) + if err != nil || e == nil { + continue + } + if r, err := health.DecodeReading(e.Value); err == nil { + out = append(out, r) + } + } + return out, nil + } +} diff --git a/modules/restic/go.mod b/modules/restic/go.mod index 7ec16d84..cc18cada 100644 --- a/modules/restic/go.mod +++ b/modules/restic/go.mod @@ -1,5 +1,5 @@ module restic -go 1.25.0 +go 1.26.0 -require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261010191001-33917f91ac3c +require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 diff --git a/modules/restic/go.sum b/modules/restic/go.sum index dfc73823..f8bfbd2b 100644 --- a/modules/restic/go.sum +++ b/modules/restic/go.sum @@ -1,2 +1,2 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261010191001-33917f91ac3c h1:l5onJwoIeH8yE/PjVlEeoPyXBsHp2NQiWLllz2fwX7s= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261010191001-33917f91ac3c/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 h1:lKU3HjNXGiVbLW2hUgFjU/TMTUYMlNnnmZxG5T7biQA= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= diff --git a/modules/restic/module.json b/modules/restic/module.json index ea5ec8b9..05927fe2 100644 --- a/modules/restic/module.json +++ b/modules/restic/module.json @@ -41,40 +41,40 @@ } ] }, - "config": { - "keep-daily": { - "type": "number", - "description": "How many nightly restore points are kept, a count from 1 to 9999: two weeks undo a mistake noticed within a fortnight. Read by the night, which thins nothing while it does not read (novox/hq issue 416); lowering it deletes the restore points beyond the new count at the next night, and that cannot be undone.", - "minimum": 1, - "maximum": 9999, - "trusted": true, - "default": 14 - }, - "keep-weekly": { - "type": "number", - "description": "How many weekly restore points are kept behind the dailies, a count from 1 to 9999, refused by the night like keep-daily; lowering it deletes the restore points beyond the new count at the next night, and that cannot be undone.", - "minimum": 1, - "maximum": 9999, - "trusted": true, - "default": 8 - }, - "keep-monthly": { - "type": "number", - "description": "How many monthly restore points are kept behind the weeklies, a count from 1 to 9999, refused by the night like keep-daily; lowering it deletes the restore points beyond the new count at the next night, and that cannot be undone.", - "minimum": 1, - "maximum": 9999, - "trusted": true, - "default": 6 - }, - "keep-hand-taken-days": { - "type": "number", - "description": "How many days a restore point taken by hand (node-backup.now, or the mesh's before a module moves) is kept at least, a count from 1 to 9999; it is kept until a later good night of its module exists, and this many days in any case, so the next thinning never takes the one taken against a risky change (novox/hq issue 453). Lowering it lets go of the older ones taken by hand at the next thinning, and that cannot be undone.", - "minimum": 1, - "maximum": 9999, - "trusted": true, - "default": 7 - } + "config": { + "keep-daily": { + "type": "number", + "description": "How many nightly restore points are kept, a count from 1 to 9999: two weeks undo a mistake noticed within a fortnight. Read by the night, which thins nothing while it does not read (novox/hq issue 416); lowering it deletes the restore points beyond the new count at the next night, and that cannot be undone.", + "minimum": 1, + "maximum": 9999, + "trusted": true, + "default": 14 }, + "keep-weekly": { + "type": "number", + "description": "How many weekly restore points are kept behind the dailies, a count from 1 to 9999, refused by the night like keep-daily; lowering it deletes the restore points beyond the new count at the next night, and that cannot be undone.", + "minimum": 1, + "maximum": 9999, + "trusted": true, + "default": 8 + }, + "keep-monthly": { + "type": "number", + "description": "How many monthly restore points are kept behind the weeklies, a count from 1 to 9999, refused by the night like keep-daily; lowering it deletes the restore points beyond the new count at the next night, and that cannot be undone.", + "minimum": 1, + "maximum": 9999, + "trusted": true, + "default": 6 + }, + "keep-hand-taken-days": { + "type": "number", + "description": "How many days a restore point taken by hand (node-backup.now, or the mesh's before a module moves) is kept at least, a count from 1 to 9999; it is kept until a later good night of its module exists, and this many days in any case, so the next thinning never takes the one taken against a risky change (novox/hq issue 453). Lowering it lets go of the older ones taken by hand at the next thinning, and that cannot be undone.", + "minimum": 1, + "maximum": 9999, + "trusted": true, + "default": 7 + } + }, "resources": [ { "id": "state", @@ -139,22 +139,17 @@ "MESH_BACKUP_DATA": "${dir:state}/data.conf" }, "health": { - "kind": "tool", - "tool": "backup_health", - "interval": "60s", - "timeout": "20s", - "looks": 2, - "grace": "60s" + "kind": "unit" } }, { - "id": "measure", + "id": "awake", "type": "process", - "name": "restic-measure", + "name": "restic-awake", "artifact": "tools", "run": [ "./restic-backups", - "measure" + "awake" ], "user": "${machine:account}", "env": { @@ -165,7 +160,12 @@ "MESH_BACKUP_DATA": "${dir:state}/data.conf" }, "health": { - "kind": "unit" + "kind": "tool", + "tool": "backup_health", + "interval": "60s", + "timeout": "20s", + "looks": 2, + "grace": "60s" } } ], @@ -195,5 +195,11 @@ "from": "files" } ] - } + }, + "reads": [ + "disk-load.readings" + ], + "state": [ + "good" + ] } diff --git a/modules/route-adapter/cmd/route-provider/provider_test.go b/modules/route-adapter/cmd/route-provider/provider_test.go index abb464e1..44ac3e40 100644 --- a/modules/route-adapter/cmd/route-provider/provider_test.go +++ b/modules/route-adapter/cmd/route-provider/provider_test.go @@ -7,6 +7,7 @@ import ( "testing" "git.novox.be/novox/mesh-sdk/go/provider" + "git.novox.be/novox/mesh-sdk/go/provisions" ) // bucketBus is the helper's bus over in-memory buckets. @@ -45,12 +46,22 @@ func (b *bucketBus) Delete(_ context.Context, bucket, key string) error { // the entry's values are the controller's composition, copied unchanged, and no vault is asked. func TestAConsumerOfARouteIsToldItsNamesFromItsPlanEntry(t *testing.T) { bus := &bucketBus{buckets: map[string]map[string][]byte{}} - plan, _ := json.Marshal(map[string]any{"node": "g14", "module": "gitea", "provisioning": map[string]any{ + plan, _ := json.Marshal(map[string]any{"v": 1, "node": "g14", "module": "gitea", "plan": map[string]any{ "provisions": []any{map[string]any{"provision": ProvisionName, "provider": "novox", "provider-module": ModuleName, "at": "10.7.0.1", "from": "novox", "as": "mesh_g14_gitea", "names": map[string]any{"name": "git.example.org", "internal-name": "git.novox.internal", "name-web": "git.example.org"}}}}}) - if err := bus.Put(context.Background(), provider.AssignmentsBucket, "g14.gitea", plan); err != nil { + if err := bus.Put(context.Background(), provisions.PlansBucket, "g14.gitea", plan); err != nil { + t.Fatal(err) + } + // The key the seat composed for the entry, asked of route-adapter (novox/hq ADR 0570 clauses 1, 3). + key, _ := provisions.Key("g14", "gitea", ProvisionName, "") + asked, err := provisions.Encode(provisions.Provision{Provision: ProvisionName, State: provisions.Asked, + Consumer: provisions.End{Node: "g14", Module: "gitea"}, Provider: provisions.End{Node: "novox", Module: ModuleName}}) + if err != nil { + t.Fatal(err) + } + if err := bus.Put(context.Background(), provisions.ProvisionsBucket, key, asked); err != nil { t.Fatal(err) } p := newProvider("novox", bus, t.Logf) diff --git a/modules/route-adapter/go.mod b/modules/route-adapter/go.mod index a4ec98b8..a4fe90d2 100644 --- a/modules/route-adapter/go.mod +++ b/modules/route-adapter/go.mod @@ -2,4 +2,4 @@ module route-adapter go 1.26.0 -require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021114014-66e8fdaa62e2 +require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 diff --git a/modules/route-adapter/go.sum b/modules/route-adapter/go.sum index 6162079d..f8bfbd2b 100644 --- a/modules/route-adapter/go.sum +++ b/modules/route-adapter/go.sum @@ -1,2 +1,2 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021114014-66e8fdaa62e2 h1:Cdl5N56QPfmFBoLHrDfvvyc5Q+PUnTjf3fgYD7imcco= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021114014-66e8fdaa62e2/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 h1:lKU3HjNXGiVbLW2hUgFjU/TMTUYMlNnnmZxG5T7biQA= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= diff --git a/modules/route-adapter/module.json b/modules/route-adapter/module.json index 5234c78b..2827462c 100644 --- a/modules/route-adapter/module.json +++ b/modules/route-adapter/module.json @@ -9,6 +9,7 @@ "provides": [ { "name": "route", + "credential": {"form": "none"}, "scope": "mesh", "identity": false } @@ -129,6 +130,7 @@ "bound" ], "reads": [ - "mesh-controller.assignments" + "mesh-provisioner.plans", + "mesh-provisioner.provisions" ] } diff --git a/modules/showcase/module.json b/modules/showcase/module.json index e657e2ab..6b0db672 100644 --- a/modules/showcase/module.json +++ b/modules/showcase/module.json @@ -9,7 +9,10 @@ "provides": [ { "name": "greeting", - "scope": "mesh" + "scope": "mesh", + "credential": { + "form": "none" + } } ], "serves": { diff --git a/modules/step-ca/module.json b/modules/step-ca/module.json index b11d2b99..3f44ead7 100644 --- a/modules/step-ca/module.json +++ b/modules/step-ca/module.json @@ -7,7 +7,10 @@ "provides": [ { "name": "internal-acme-ca", - "scope": "mesh" + "scope": "mesh", + "credential": { + "form": "none" + } } ], "serves": { diff --git a/modules/systemd-resolved/go.mod b/modules/systemd-resolved/go.mod index c50c4f22..79ddf566 100644 --- a/modules/systemd-resolved/go.mod +++ b/modules/systemd-resolved/go.mod @@ -1,5 +1,5 @@ module systemd-resolved -go 1.25.0 +go 1.26.0 -require git.novox.be/novox/mesh-sdk/go v0.1.7 +require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 diff --git a/modules/systemd-resolved/go.sum b/modules/systemd-resolved/go.sum index b474419e..f8bfbd2b 100644 --- a/modules/systemd-resolved/go.sum +++ b/modules/systemd-resolved/go.sum @@ -1,2 +1,2 @@ -git.novox.be/novox/mesh-sdk/go v0.1.7 h1:C0sTQmtTiyYH7bnqZb7PusXnqA37gKuT7Nqjn9gG47w= -git.novox.be/novox/mesh-sdk/go v0.1.7/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 h1:lKU3HjNXGiVbLW2hUgFjU/TMTUYMlNnnmZxG5T7biQA= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= diff --git a/modules/systemd-resolved/module.json b/modules/systemd-resolved/module.json index b850afcf..24b3bc85 100644 --- a/modules/systemd-resolved/module.json +++ b/modules/systemd-resolved/module.json @@ -8,7 +8,10 @@ "provides": [ { "name": "split-dns", - "reach": "machine" + "reach": "machine", + "credential": { + "form": "none" + } } ], "requires": [ diff --git a/modules/umami/module.json b/modules/umami/module.json index 22ead419..e3b9aa0d 100644 --- a/modules/umami/module.json +++ b/modules/umami/module.json @@ -32,7 +32,10 @@ "provides": [ { "name": "analytics", - "scope": "mesh" + "scope": "mesh", + "credential": { + "form": "none" + } } ], "serves": { diff --git a/modules/xorg/go.mod b/modules/xorg/go.mod index 356d1874..95dbdc00 100644 --- a/modules/xorg/go.mod +++ b/modules/xorg/go.mod @@ -1,5 +1,5 @@ module xorg -go 1.22 +go 1.26.0 -require git.novox.be/novox/mesh-sdk/go v0.1.7 +require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 diff --git a/modules/xorg/go.sum b/modules/xorg/go.sum index b474419e..f8bfbd2b 100644 --- a/modules/xorg/go.sum +++ b/modules/xorg/go.sum @@ -1,2 +1,2 @@ -git.novox.be/novox/mesh-sdk/go v0.1.7 h1:C0sTQmtTiyYH7bnqZb7PusXnqA37gKuT7Nqjn9gG47w= -git.novox.be/novox/mesh-sdk/go v0.1.7/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 h1:lKU3HjNXGiVbLW2hUgFjU/TMTUYMlNnnmZxG5T7biQA= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= diff --git a/modules/xorg/module.json b/modules/xorg/module.json index 7b0b6b1e..1685dcb4 100644 --- a/modules/xorg/module.json +++ b/modules/xorg/module.json @@ -9,7 +9,10 @@ "provides": [ { "name": "x11-display", - "reach": "machine" + "reach": "machine", + "credential": { + "form": "none" + } } ], "claims": [