From 685cb1cb1ba6de4960afdead4b5a0b6b62d9c054 Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 6 Oct 2026 16:47:49 +0200 Subject: [PATCH 1/2] Declare every module's data; the backup holder measures it (hq ADR 0233) Backup lines are derived from each module's data section instead of written by hand; the holder measures declared items, reads the array under them, and deletes a retired item only after a last restore point; the Go providers say each held consumer's size so an empty replacement is seen. --- modules/audit-logger/module.json | 10 + modules/baserow/module.json | 10 + modules/build-agent/module.json | 10 + modules/claude-code/module.json | 10 + modules/distribution/module.json | 11 + modules/gitea/module.json | 24 +- modules/grafana/module.json | 10 + modules/home-assistant/module.json | 11 + modules/icecast/module.json | 10 + modules/influxdb/module.json | 32 +- .../cmd/keycloak-provider/retirement.go | 11 +- modules/keycloak/module.json | 9 + modules/lab/module.json | 10 + modules/mailu/module.json | 90 ++- modules/matrix/module.json | 10 + modules/mesh-vault/module.json | 36 +- modules/minio/module.json | 26 +- modules/model-usage/module.json | 10 + modules/mongodb/module.json | 36 +- modules/mosquitto/module.json | 17 + modules/mssql/module.json | 36 +- modules/n8n/module.json | 16 + modules/nats/module.json | 11 + modules/nextcloud/module.json | 19 +- modules/nodered/module.json | 10 + modules/ollama/module.json | 11 + modules/only-office/module.json | 46 ++ .../cmd/postgres-provider/retire_pg.go | 5 +- .../cmd/postgres-provider/retirement.go | 11 +- modules/postgres/module.json | 36 +- modules/records/module.json | 10 + modules/redis/module.json | 16 + modules/restic/cmd/restic-backups/backups.go | 41 +- modules/restic/cmd/restic-backups/data.go | 512 ++++++++++++++++++ .../restic/cmd/restic-backups/data_test.go | 241 +++++++++ modules/restic/cmd/restic-backups/main.go | 37 ++ modules/restic/module.json | 27 +- modules/route-proxy/module.json | 16 + modules/searxng/module.json | 10 + modules/ssh-client/module.json | 10 + modules/step-ca/module.json | 10 + modules/supabase/module.json | 28 + modules/umami/module.json | 9 + modules/unifi/module.json | 10 + 44 files changed, 1493 insertions(+), 78 deletions(-) create mode 100644 modules/restic/cmd/restic-backups/data.go create mode 100644 modules/restic/cmd/restic-backups/data_test.go diff --git a/modules/audit-logger/module.json b/modules/audit-logger/module.json index f8c5e74..84ef472 100644 --- a/modules/audit-logger/module.json +++ b/modules/audit-logger/module.json @@ -23,6 +23,16 @@ } ] }, + "data": { + "own": [ + { + "id": "trail", + "path": "${dir:trail}", + "class": "valuable", + "why": "the audit trail, written nowhere else" + } + ] + }, "resources": [ { "id": "state", diff --git a/modules/baserow/module.json b/modules/baserow/module.json index 3dd921a..4b9c81d 100644 --- a/modules/baserow/module.json +++ b/modules/baserow/module.json @@ -36,6 +36,16 @@ "why": "the Baserow web UI and REST API, served by the image's own Caddy; a public name is the route's" } ], + "data": { + "own": [ + { + "id": "data", + "path": "${dir:data}", + "class": "valuable", + "why": "uploaded files and media; the tables are in the database" + } + ] + }, "resources": [ { "id": "mesh-state", diff --git a/modules/build-agent/module.json b/modules/build-agent/module.json index d63440a..a7539ad 100644 --- a/modules/build-agent/module.json +++ b/modules/build-agent/module.json @@ -25,6 +25,16 @@ "own-secrets": { "broker": "${dir:mesh-state}/broker" }, + "data": { + "own": [ + { + "id": "workspace", + "path": "${dir:workspace}", + "class": "cache", + "why": "a build's working copy, cloned again for every build" + } + ] + }, "resources": [ { "id": "mesh-state", diff --git a/modules/claude-code/module.json b/modules/claude-code/module.json index c172367..84c37f5 100644 --- a/modules/claude-code/module.json +++ b/modules/claude-code/module.json @@ -56,6 +56,16 @@ "claude_code_config_status", "claude_code_config_import" ], + "data": { + "own": [ + { + "id": "agent-home", + "path": "${dir:agent-home}", + "class": "valuable", + "why": "the operator's agent's own files: its memory, history and projects" + } + ] + }, "resources": [ { "id": "package", diff --git a/modules/distribution/module.json b/modules/distribution/module.json index 3a7c85f..06446f7 100644 --- a/modules/distribution/module.json +++ b/modules/distribution/module.json @@ -36,6 +36,17 @@ "why": "every machine pulls images and artifacts from here" } ], + "data": { + "own": [ + { + "id": "registry", + "path": "${dir:registry-data}", + "class": "rebuildable", + "backup": "none", + "why": "the artifact store: every image is built again from its source, and too large to copy every night (ADR 0214 left it out)" + } + ] + }, "resources": [ { "id": "state", diff --git a/modules/gitea/module.json b/modules/gitea/module.json index 33ea9d7..d5a29f7 100644 --- a/modules/gitea/module.json +++ b/modules/gitea/module.json @@ -85,6 +85,23 @@ "scope": "mesh" } ], + "data": { + "own": [ + { + "id": "data", + "path": "${dir:data}", + "class": "valuable", + "why": "every repository, its issues and attachments, and the package registry" + } + ], + "consumers": { + "npm-package-registry": { + "class": "rebuildable", + "in": "data", + "why": "a consumer's packages are published again from its source" + } + } + }, "resources": [ { "id": "mesh-state", @@ -226,12 +243,5 @@ "failregex": "^.*Failed authentication attempt for .* from (?::\\d+)?\\s*$\n ^.*Invalid user .* from port \\d+\\s*$\n ^.*User \\S+ from not allowed because .*$", "jail": "backend = systemd\njournalmatch = CONTAINER_NAME=gitea\nport = http,https,222\nmaxretry = 3\nfindtime = 1d\nbantime = 1d" } - ], - "contributions": [ - { - "seat": "node-backup", - "kind": "backup", - "content": "path ${dir:data}\n" - } ] } diff --git a/modules/grafana/module.json b/modules/grafana/module.json index 7645f6b..9ebebb0 100644 --- a/modules/grafana/module.json +++ b/modules/grafana/module.json @@ -19,6 +19,16 @@ "why": "the dashboards. Also 3000 inside, like the forge - which is the mesh's port assignment earning its keep" } ], + "data": { + "own": [ + { + "id": "data", + "path": "${dir:data}", + "class": "valuable", + "why": "dashboards, users and alert rules" + } + ] + }, "resources": [ { "id": "mesh-state", diff --git a/modules/home-assistant/module.json b/modules/home-assistant/module.json index ada4c9b..6db00c8 100644 --- a/modules/home-assistant/module.json +++ b/modules/home-assistant/module.json @@ -34,6 +34,17 @@ "why": "the bundled go2rtc's WebRTC port, which camera streams to a browser use" } ], + "data": { + "own": [ + { + "id": "config", + "path": "${dir:config}", + "class": "valuable", + "active": "1d", + "why": "the house's configuration, automations and history; written all the time" + } + ] + }, "resources": [ { "id": "mesh-state", diff --git a/modules/icecast/module.json b/modules/icecast/module.json index f3098e7..2e32075 100644 --- a/modules/icecast/module.json +++ b/modules/icecast/module.json @@ -37,6 +37,16 @@ "why": "streams in from sources (HTTP PUT) and out to listeners, plus the status and admin pages; a public name is its route" } ], + "data": { + "own": [ + { + "id": "logs", + "path": "${dir:logs}", + "class": "cache", + "why": "the streaming server's logs" + } + ] + }, "resources": [ { "id": "mesh-state", diff --git a/modules/influxdb/module.json b/modules/influxdb/module.json index e3445fd..bad5aee 100644 --- a/modules/influxdb/module.json +++ b/modules/influxdb/module.json @@ -40,6 +40,29 @@ "grants": { "influxdb-api": "${dir:grants}" }, + "data": { + "own": [ + { + "id": "data", + "path": "${dir:data}", + "class": "valuable", + "why": "every consumer's time series" + }, + { + "id": "config", + "path": "${dir:config}", + "class": "valuable", + "why": "the server's own configuration and its operator token, made at its first start" + } + ], + "consumers": { + "influxdb-api": { + "class": "valuable", + "in": "data", + "why": "a consumer's measurements are the only copy" + } + } + }, "resources": [ { "id": "mesh-state", @@ -135,12 +158,5 @@ } } ] - }, - "contributions": [ - { - "seat": "node-backup", - "kind": "backup", - "content": "path ${dir:data}\npath ${dir:config}\n" - } - ] + } } diff --git a/modules/keycloak/cmd/keycloak-provider/retirement.go b/modules/keycloak/cmd/keycloak-provider/retirement.go index f15f181..3c3f578 100644 --- a/modules/keycloak/cmd/keycloak-provider/retirement.go +++ b/modules/keycloak/cmd/keycloak-provider/retirement.go @@ -89,6 +89,9 @@ type Retired struct { type Inventory struct { Active []string Retired []Retired + // Sizes is what each active consumer keeps on the backend, in bytes, where the backend can say: what + // the controller tells an empty replacement of a consumer's data by (novox/hq ADR 0233). + Sizes map[string]int64 } // retirement is the loop's memory of the sets it is counting, waiting on and was told to keep. @@ -421,7 +424,13 @@ func (h *Harness) Retirement(ctx context.Context) (map[string]any, error) { retired = append(retired, map[string]any{"consumer": r.Consumer, "node": r.Node, "retired_at": stampOr(r.RetiredAt), "why": r.Why, "size_bytes": r.SizeBytes, "kind": kindOf(r)}) } - out := map[string]any{"resource": h.Resource, "node": h.Node, "held": orEmptyList(held), + sizes := map[string]int64{} + for _, as := range held { + if size, ok := inv.Sizes[as]; ok && size >= 0 { + sizes[as] = size + } + } + out := map[string]any{"resource": h.Resource, "node": h.Node, "held": orEmptyList(held), "held_sizes": sizes, "stable_passes": h.StablePasses, "stable_for_seconds": int(h.StableFor.Seconds()), "bound": h.bound(len(h.applied)), "waiting": nil, "rejected": nil, "retired": retired} if w := h.r.waiting; w != nil { diff --git a/modules/keycloak/module.json b/modules/keycloak/module.json index a01997f..c82ae5c 100644 --- a/modules/keycloak/module.json +++ b/modules/keycloak/module.json @@ -71,6 +71,15 @@ "own-secrets": { "admin": "${dir:state}/admin.secret" }, + "data": { + "consumers": { + "oidc-client": { + "class": "rebuildable", + "in": "postgres-database", + "why": "a consumer's client is made again from its declaration, with a new secret" + } + } + }, "resources": [ { "id": "mesh-state", diff --git a/modules/lab/module.json b/modules/lab/module.json index 78c92e9..fe523f3 100644 --- a/modules/lab/module.json +++ b/modules/lab/module.json @@ -5,6 +5,16 @@ "container-runtime", "virtualisation" ], + "data": { + "own": [ + { + "id": "work", + "path": "${dir:work}", + "class": "cache", + "why": "the lab's runs, each thrown away" + } + ] + }, "resources": [ { "id": "state", diff --git a/modules/mailu/module.json b/modules/mailu/module.json index 450a27a..403373d 100644 --- a/modules/mailu/module.json +++ b/modules/mailu/module.json @@ -144,6 +144,89 @@ "why": "the admin API, which this module's own code reaches on loopback from the node's runtime now that it runs outside the mailu network" } ], + "data": { + "own": [ + { + "id": "mail", + "path": "${dir:data-mail}", + "class": "valuable", + "why": "every mailbox" + }, + { + "id": "dkim", + "path": "${dir:data-dkim}", + "class": "valuable", + "why": "the domain's signing keys; a new one means a new DNS record" + }, + { + "id": "data", + "path": "${dir:data-data}", + "class": "valuable", + "why": "the server's own data" + }, + { + "id": "dav", + "path": "${dir:data-dav}", + "class": "valuable", + "why": "calendars and contacts" + }, + { + "id": "webmail", + "path": "${dir:data-webmail}", + "class": "valuable", + "why": "the webmail's own data: its users' settings and address books" + }, + { + "id": "queue", + "path": "${dir:data-mailqueue}", + "class": "valuable", + "why": "mail accepted and not yet delivered, kept nowhere else" + }, + { + "id": "filter", + "path": "${dir:data-filter}", + "class": "rebuildable", + "why": "the spam filter's learned statistics; it learns again" + }, + { + "id": "certs", + "path": "${dir:data-certs}", + "class": "rebuildable", + "why": "certificates, issued again" + }, + { + "id": "automx", + "path": "${dir:data-automx}", + "class": "rebuildable", + "why": "client autoconfiguration, written again" + }, + { + "id": "fetchmail", + "path": "${dir:data-fetchmail}", + "class": "rebuildable", + "why": "which remote messages were fetched; lost, some are fetched twice" + }, + { + "id": "clamav", + "path": "${dir:data-clamav}", + "class": "cache", + "why": "virus signatures, downloaded again" + }, + { + "id": "redis", + "path": "${dir:data-redis}", + "class": "cache", + "why": "the filter's working set" + } + ], + "consumers": { + "smtp": { + "class": "valuable", + "in": "mail", + "why": "a consumer's mailbox holds the only copy of its mail" + } + } + }, "resources": [ { "id": "mesh-state", @@ -562,12 +645,5 @@ "failregex": "^.*(?:imap|pop3|submission|managesieve)-login: .*\\(auth failed, \\d+ attempts(?: in \\d+ secs)?\\):.*rip=(?:,|$)", "jail": "backend = systemd\njournalmatch = CONTAINER_NAME=mailu-front\nport = smtp,submission,submissions,imap,imaps,pop3,pop3s\nmaxretry = 3\nfindtime = 1d\nbantime = 1d" } - ], - "contributions": [ - { - "seat": "node-backup", - "kind": "backup", - "content": "path ${dir:data-mail}\npath ${dir:data-dkim}\npath ${dir:data-data}\npath ${dir:data-dav}\npath ${dir:data-webmail}\n" - } ] } diff --git a/modules/matrix/module.json b/modules/matrix/module.json index 5dd6853..6887b7a 100644 --- a/modules/matrix/module.json +++ b/modules/matrix/module.json @@ -38,6 +38,16 @@ "binds": { "route": "${dir:state}/route.json" }, + "data": { + "own": [ + { + "id": "db", + "path": "${dir:db}", + "class": "valuable", + "why": "every room, message and account" + } + ] + }, "resources": [ { "id": "state", diff --git a/modules/mesh-vault/module.json b/modules/mesh-vault/module.json index 4f6ddeb..87b6e6d 100644 --- a/modules/mesh-vault/module.json +++ b/modules/mesh-vault/module.json @@ -30,6 +30,35 @@ "secret": "${dir:grants}" }, "keeps": "/var/lib/mesh-vault/root", + "data": { + "own": [ + { + "id": "state", + "path": "${dir:state}", + "class": "valuable", + "why": "the vault's own keys" + }, + { + "id": "ledger", + "path": "${dir:ledger}", + "class": "valuable", + "why": "every secret the vault keeps" + }, + { + "id": "root", + "path": "${dir:root}", + "class": "valuable", + "why": "the vault's root material" + } + ], + "consumers": { + "secret": { + "class": "valuable", + "in": "ledger", + "why": "a secret given to a consumer is kept nowhere else in the clear" + } + } + }, "resources": [ { "id": "state", @@ -82,12 +111,5 @@ "name": "mesh-vault", "scope": "mesh" } - ], - "contributions": [ - { - "seat": "node-backup", - "kind": "backup", - "content": "path ${dir:state}\npath ${dir:ledger}\npath ${dir:root}\n" - } ] } diff --git a/modules/minio/module.json b/modules/minio/module.json index d4f95eb..0914536 100644 --- a/modules/minio/module.json +++ b/modules/minio/module.json @@ -66,6 +66,23 @@ "own-secrets": { "root": "${dir:state}/root.secret" }, + "data": { + "own": [ + { + "id": "data", + "path": "${dir:data}", + "class": "valuable", + "why": "every consumer's bucket and its objects" + } + ], + "consumers": { + "s3-bucket": { + "class": "valuable", + "in": "data", + "why": "a consumer's objects are the only copy of what it stored; a consumer that keeps something irreplaceable here says so (kept-by)" + } + } + }, "resources": [ { "id": "state", @@ -155,12 +172,5 @@ } } ] - }, - "contributions": [ - { - "seat": "node-backup", - "kind": "backup", - "content": "path ${dir:data}\n" - } - ] + } } diff --git a/modules/model-usage/module.json b/modules/model-usage/module.json index c627c5c..6dc6075 100644 --- a/modules/model-usage/module.json +++ b/modules/model-usage/module.json @@ -25,6 +25,16 @@ "own-secrets": { "broker": "${dir:mesh-state}/broker" }, + "data": { + "own": [ + { + "id": "state", + "path": "${dir:state}", + "class": "cache", + "why": "the mesh's own files for it; its records are in its database" + } + ] + }, "resources": [ { "id": "mesh-state", diff --git a/modules/mongodb/module.json b/modules/mongodb/module.json index 8915d62..89bdace 100644 --- a/modules/mongodb/module.json +++ b/modules/mongodb/module.json @@ -45,6 +45,33 @@ "own-secrets": { "root": "${dir:state}/root.secret" }, + "data": { + "own": [ + { + "id": "data", + "path": "${dir:data}", + "class": "valuable", + "backup": { + "dump": "docker exec mongodb-server sh -c 'printf \"password: %s\\n\" \"$(cat /run/secrets/root)\" > /tmp/.backup.yaml && mongodump --quiet --config /tmp/.backup.yaml --username root --authenticationDatabase admin --archive; s=$?; rm -f /tmp/.backup.yaml; exit $s' > ${dir:dumps}/all.archive.partial && mv ${dir:dumps}/all.archive.partial ${dir:dumps}/all.archive", + "into": "dumps" + }, + "why": "every consumer's database; copied by the dump, not as live files" + }, + { + "id": "dumps", + "path": "${dir:dumps}", + "class": "rebuildable", + "why": "last night's dump, made again every night" + } + ], + "consumers": { + "mongodb-database": { + "class": "valuable", + "in": "data", + "why": "a consumer's documents are the only copy of what it wrote; a consumer that keeps something irreplaceable here says so (kept-by)" + } + } + }, "resources": [ { "id": "state", @@ -122,12 +149,5 @@ } } ] - }, - "contributions": [ - { - "seat": "node-backup", - "kind": "backup", - "content": "run docker exec mongodb-server sh -c 'printf \"password: %s\\n\" \"$(cat /run/secrets/root)\" > /tmp/.backup.yaml && mongodump --quiet --config /tmp/.backup.yaml --username root --authenticationDatabase admin --archive; s=$?; rm -f /tmp/.backup.yaml; exit $s' > ${dir:dumps}/all.archive.partial && mv ${dir:dumps}/all.archive.partial ${dir:dumps}/all.archive\npath ${dir:dumps}\n" - } - ] + } } diff --git a/modules/mosquitto/module.json b/modules/mosquitto/module.json index 0ffc9c6..d9740da 100644 --- a/modules/mosquitto/module.json +++ b/modules/mosquitto/module.json @@ -53,6 +53,23 @@ "why": "the same broker over MQTT-on-WebSockets, for browser clients" } ], + "data": { + "own": [ + { + "id": "data", + "path": "${dir:data}", + "class": "rebuildable", + "why": "retained messages and sessions; devices publish them again" + } + ], + "consumers": { + "mqtt-topic": { + "class": "rebuildable", + "in": "data", + "why": "retained messages are published again by the devices" + } + } + }, "resources": [ { "id": "mesh-state", diff --git a/modules/mssql/module.json b/modules/mssql/module.json index 09a5b06..3dad4f6 100644 --- a/modules/mssql/module.json +++ b/modules/mssql/module.json @@ -44,6 +44,33 @@ "sa": "${dir:state}/sa.secret", "reader": "${dir:state}/reader.secret" }, + "data": { + "own": [ + { + "id": "data", + "path": "${dir:data}", + "class": "valuable", + "backup": { + "dump": "{ cat ${dir:state}/sa.secret; echo; cat ${dir:state}/backup.sql; } | docker exec -i mssql sh -c 'read -r p; SQLCMDPASSWORD=\"$p\" exec /opt/mssql-tools18/bin/sqlcmd -C -b -S localhost -U sa -i /dev/stdin'", + "into": "dumps" + }, + "why": "every consumer's database; copied by the dump, not as live files" + }, + { + "id": "dumps", + "path": "${dir:dumps}", + "class": "rebuildable", + "why": "last night's dump, made again every night" + } + ], + "consumers": { + "mssql-database": { + "class": "valuable", + "in": "data", + "why": "a consumer's rows are the only copy of what it wrote" + } + } + }, "resources": [ { "id": "state", @@ -130,12 +157,5 @@ } } ] - }, - "contributions": [ - { - "seat": "node-backup", - "kind": "backup", - "content": "run { cat ${dir:state}/sa.secret; echo; cat ${dir:state}/backup.sql; } | docker exec -i mssql sh -c 'read -r p; SQLCMDPASSWORD=\"$p\" exec /opt/mssql-tools18/bin/sqlcmd -C -b -S localhost -U sa -i /dev/stdin'\npath ${dir:dumps}\n" - } - ] + } } diff --git a/modules/n8n/module.json b/modules/n8n/module.json index 609d2e9..7666572 100644 --- a/modules/n8n/module.json +++ b/modules/n8n/module.json @@ -39,6 +39,22 @@ "why": "the n8n editor, its REST API and the webhook endpoints workflows are triggered through; a public name is the route's" } ], + "data": { + "own": [ + { + "id": "data", + "path": "${dir:data}", + "class": "valuable", + "why": "the instance's encryption key, settings and binary data; workflows are in the database" + }, + { + "id": "cache", + "path": "${dir:cache}", + "class": "cache", + "why": "scratch space" + } + ] + }, "resources": [ { "id": "state", diff --git a/modules/nats/module.json b/modules/nats/module.json index 8f8798b..0c42cd1 100644 --- a/modules/nats/module.json +++ b/modules/nats/module.json @@ -41,6 +41,17 @@ "guards": [ 8222 ], + "data": { + "own": [ + { + "id": "jetstream", + "path": "${dir:jetstream-data}", + "class": "valuable", + "active": "1d", + "why": "the bus's streams and key-value buckets: the hand-act log, conditions, every module's state; written all the time" + } + ] + }, "resources": [ { "id": "jetstream-data", diff --git a/modules/nextcloud/module.json b/modules/nextcloud/module.json index fb177b4..4d648e0 100644 --- a/modules/nextcloud/module.json +++ b/modules/nextcloud/module.json @@ -44,6 +44,16 @@ "why": "files and sync, over http; a public name is a route grant later" } ], + "data": { + "own": [ + { + "id": "html", + "path": "${dir:html}", + "class": "valuable", + "why": "the instance's configuration, its secret and installed apps; the files are in the object store" + } + ] + }, "resources": [ { "id": "mesh-state", @@ -117,12 +127,5 @@ } } ] - }, - "contributions": [ - { - "seat": "node-backup", - "kind": "backup", - "content": "path ${dir:html}\n" - } - ] + } } diff --git a/modules/nodered/module.json b/modules/nodered/module.json index 131de90..db2b27e 100644 --- a/modules/nodered/module.json +++ b/modules/nodered/module.json @@ -26,6 +26,16 @@ "why": "the flow editor and the endpoints flows expose" } ], + "data": { + "own": [ + { + "id": "data", + "path": "${dir:data}", + "class": "valuable", + "why": "flows and their credentials" + } + ] + }, "resources": [ { "id": "mesh-state", diff --git a/modules/ollama/module.json b/modules/ollama/module.json index dcb6ce3..12bad0a 100644 --- a/modules/ollama/module.json +++ b/modules/ollama/module.json @@ -25,6 +25,17 @@ "model": "llama3.2" } }, + "data": { + "own": [ + { + "id": "models", + "path": "${dir:state}", + "class": "rebuildable", + "backup": "none", + "why": "models, downloaded again, and too large to copy every night" + } + ] + }, "resources": [ { "id": "state", diff --git a/modules/only-office/module.json b/modules/only-office/module.json index 8763ac6..49f10ef 100644 --- a/modules/only-office/module.json +++ b/modules/only-office/module.json @@ -29,6 +29,52 @@ "why": "the document server over http; its public name is a route grant, and route-proxy reaches it on this published port" } ], + "data": { + "own": [ + { + "id": "data", + "path": "${dir:data}", + "class": "rebuildable", + "why": "documents being edited; the documents themselves are the file store's" + }, + { + "id": "lib", + "path": "${dir:lib}", + "class": "rebuildable", + "why": "the document server's working files" + }, + { + "id": "db", + "path": "${dir:db}", + "class": "rebuildable", + "why": "the document server's own database of editing sessions" + }, + { + "id": "fonts", + "path": "${dir:fonts}", + "class": "rebuildable", + "why": "fonts, installed again" + }, + { + "id": "logs", + "path": "${dir:logs}", + "class": "cache", + "why": "logs" + }, + { + "id": "rabbitmq", + "path": "${dir:rabbitmq}", + "class": "cache", + "why": "its queue's working set" + }, + { + "id": "redis", + "path": "${dir:redis}", + "class": "cache", + "why": "its cache" + } + ] + }, "resources": [ { "id": "state", diff --git a/modules/postgres/cmd/postgres-provider/retire_pg.go b/modules/postgres/cmd/postgres-provider/retire_pg.go index d76a546..969dc4f 100644 --- a/modules/postgres/cmd/postgres-provider/retire_pg.go +++ b/modules/postgres/cmd/postgres-provider/retire_pg.go @@ -98,7 +98,7 @@ const SetAsideStatement = `SELECT datname, pg_database_size(datname) FROM pg_dat // Inventory is what this server holds that the mesh made. func (c *Client) Inventory(ctx context.Context) (Inventory, error) { - inv := Inventory{Active: []string{}, Retired: []Retired{}} + inv := Inventory{Active: []string{}, Retired: []Retired{}, Sizes: map[string]int64{}} r, err := c.Query(ctx, "", InventoryStatement) if err != nil { return inv, err @@ -111,6 +111,9 @@ func (c *Client) Inventory(ctx context.Context) (Inventory, error) { } if login && m.Retired == "" { inv.Active = append(inv.Active, name) + if s := sizeOf(size); s >= 0 { + inv.Sizes[name] = s + } continue } at, _ := time.Parse(time.RFC3339, m.Retired) diff --git a/modules/postgres/cmd/postgres-provider/retirement.go b/modules/postgres/cmd/postgres-provider/retirement.go index f15f181..3c3f578 100644 --- a/modules/postgres/cmd/postgres-provider/retirement.go +++ b/modules/postgres/cmd/postgres-provider/retirement.go @@ -89,6 +89,9 @@ type Retired struct { type Inventory struct { Active []string Retired []Retired + // Sizes is what each active consumer keeps on the backend, in bytes, where the backend can say: what + // the controller tells an empty replacement of a consumer's data by (novox/hq ADR 0233). + Sizes map[string]int64 } // retirement is the loop's memory of the sets it is counting, waiting on and was told to keep. @@ -421,7 +424,13 @@ func (h *Harness) Retirement(ctx context.Context) (map[string]any, error) { retired = append(retired, map[string]any{"consumer": r.Consumer, "node": r.Node, "retired_at": stampOr(r.RetiredAt), "why": r.Why, "size_bytes": r.SizeBytes, "kind": kindOf(r)}) } - out := map[string]any{"resource": h.Resource, "node": h.Node, "held": orEmptyList(held), + sizes := map[string]int64{} + for _, as := range held { + if size, ok := inv.Sizes[as]; ok && size >= 0 { + sizes[as] = size + } + } + out := map[string]any{"resource": h.Resource, "node": h.Node, "held": orEmptyList(held), "held_sizes": sizes, "stable_passes": h.StablePasses, "stable_for_seconds": int(h.StableFor.Seconds()), "bound": h.bound(len(h.applied)), "waiting": nil, "rejected": nil, "retired": retired} if w := h.r.waiting; w != nil { diff --git a/modules/postgres/module.json b/modules/postgres/module.json index 48ff5c6..48ac141 100644 --- a/modules/postgres/module.json +++ b/modules/postgres/module.json @@ -59,6 +59,33 @@ "superuser": "${dir:state}/superuser.secret", "reader": "${dir:state}/reader.secret" }, + "data": { + "own": [ + { + "id": "store", + "path": "${dir:store-data}", + "class": "valuable", + "backup": { + "dump": "docker exec -u postgres postgres sh -c 'cd /var/lib/postgresql/data/dumps && for db in $(psql -Atc \"select datname from pg_database where oid >= 16384 order by 1\"); do pg_dump -Fc -f \"$db.dump.partial\" \"$db\" && mv \"$db.dump.partial\" \"$db.dump\" || exit 1; done'", + "into": "dumps" + }, + "why": "every consumer's database; copied by the dump below, since a running store's files are not a consistent copy" + }, + { + "id": "dumps", + "path": "${dir:dumps}", + "class": "rebuildable", + "why": "last night's dump of the store, made again every night" + } + ], + "consumers": { + "postgres-database": { + "class": "valuable", + "in": "store", + "why": "a consumer's rows are the only copy of what it wrote" + } + } + }, "resources": [ { "id": "state", @@ -128,12 +155,5 @@ } } ] - }, - "contributions": [ - { - "seat": "node-backup", - "kind": "backup", - "content": "run docker exec -u postgres postgres sh -c 'cd /var/lib/postgresql/data/dumps && for db in $(psql -Atc \"select datname from pg_database where oid >= 16384 order by 1\"); do pg_dump -Fc -f \"$db.dump.partial\" \"$db\" && mv \"$db.dump.partial\" \"$db.dump\" || exit 1; done'\npath ${dir:dumps}\n" - } - ] + } } diff --git a/modules/records/module.json b/modules/records/module.json index 02feb5b..2806510 100644 --- a/modules/records/module.json +++ b/modules/records/module.json @@ -18,6 +18,16 @@ "records_status", "records_sync" ], + "data": { + "own": [ + { + "id": "checkout", + "path": "${dir:checkout}", + "class": "rebuildable", + "why": "a clone of the record, cloned again" + } + ] + }, "resources": [ { "id": "mesh-state", diff --git a/modules/redis/module.json b/modules/redis/module.json index 49d23f0..5874aee 100644 --- a/modules/redis/module.json +++ b/modules/redis/module.json @@ -47,6 +47,22 @@ "why": "modules on any machine that were granted a cache" } ], + "data": { + "own": [ + { + "id": "data", + "path": "${dir:data}", + "class": "cache", + "why": "the cache's own snapshot" + } + ], + "consumers": { + "redis-cache": { + "class": "cache", + "why": "a cache: nothing in this mesh requires it, and a consumer that kept data in it would be using a cache as a store" + } + } + }, "resources": [ { "id": "state", diff --git a/modules/restic/cmd/restic-backups/backups.go b/modules/restic/cmd/restic-backups/backups.go index bd3bc51..89b72b8 100644 --- a/modules/restic/cmd/restic-backups/backups.go +++ b/modules/restic/cmd/restic-backups/backups.go @@ -151,6 +151,9 @@ func tagOf(module string) string { return "module=" + module } // Where is where the mesh put this module's things. type Where struct { Declared, Repository, PasswordFile, State string + // Data is the composed file of every data item declared here (novox/hq ADR 0233); empty where the + // module was placed by a definition older than it, and then nothing is measured. + Data string } func whereFromEnv() (Where, error) { @@ -167,6 +170,7 @@ func whereFromEnv() (Where, error) { Repository: get("MESH_BACKUP_REPOSITORY"), PasswordFile: get("MESH_BACKUP_PASSWORD_FILE"), State: get("MESH_BACKUP_STATE"), + Data: os.Getenv("MESH_BACKUP_DATA"), } if len(missing) > 0 { return w, fmt.Errorf("%s not set; the mesh gives them to this module", strings.Join(missing, ", ")) @@ -272,6 +276,16 @@ func (b *Backups) one(ctx context.Context, d Declared) Night { if err != nil { return fail(err) } + n.Snapshot = snapshotOf(out) + n.OK = true + b.recordGood(d.Module, n.At) + b.Say("%s: backed up (%s)", d.Module, n.Snapshot) + return n +} + +// snapshotOf is the short id of the snapshot restic's JSON output says it made; empty when none. +func snapshotOf(out string) string { + id := "" scanner := bufio.NewScanner(strings.NewReader(out)) scanner.Buffer(make([]byte, 1024*1024), 16*1024*1024) for scanner.Scan() { @@ -280,12 +294,10 @@ func (b *Backups) one(ctx context.Context, d Declared) Night { SnapshotID string `json:"snapshot_id"` } if json.Unmarshal(scanner.Bytes(), &m) == nil && m.MessageType == "summary" && len(m.SnapshotID) >= 8 { - n.Snapshot = m.SnapshotID[:8] + id = m.SnapshotID[:8] } } - n.OK = true - b.Say("%s: backed up (%s)", d.Module, n.Snapshot) - return n + return id } // BackUp is a night: every module, or one, then the rotation. It answers each module's outcome. @@ -360,6 +372,9 @@ type ModuleBackups struct { LastNight *Night `json:"lastNight"` RestorePoints int `json:"restorePoints"` Newest *Snapshot `json:"newest,omitempty"` + // Data is every item the module declares, measured, with its newest good backup (novox/hq ADR + // 0233): what the controller's self-check reads. + Data []ItemReport `json:"data"` } // BackedUp is what is backed up here: each module, what it declared, its last night and its restore @@ -369,14 +384,23 @@ func (b *Backups) BackedUp(ctx context.Context, module string) ([]ModuleBackups, if err != nil { return nil, err } + items, err := b.Items() + if err != nil { + return nil, err + } nights := b.Nights() + good := b.Good() + measured := b.Measurements() snaps, _ := b.Snapshots(ctx, module) out := []ModuleBackups{} + listed := map[string]bool{} for _, d := range declared { if module != "" && d.Module != module { continue } - m := ModuleBackups{Module: d.Module, Runs: len(d.Runs), Paths: d.Paths} + listed[d.Module] = true + m := ModuleBackups{Module: d.Module, Runs: len(d.Runs), Paths: d.Paths, + Data: itemReports(items[d.Module], measured[d.Module], d.Paths, good[d.Module])} if n, ok := nights[d.Module]; ok { m.LastNight = &n } @@ -389,6 +413,13 @@ func (b *Backups) BackedUp(ctx context.Context, module string) ([]ModuleBackups, } out = append(out, m) } + // A module declaring data and nothing to back up is still measured, and said. + for name, its := range items { + if listed[name] || (module != "" && name != module) { + continue + } + out = append(out, ModuleBackups{Module: name, Paths: []string{}, Data: itemReports(its, measured[name], nil, time.Time{})}) + } return out, nil } diff --git a/modules/restic/cmd/restic-backups/data.go b/modules/restic/cmd/restic-backups/data.go new file mode 100644 index 0000000..429eba4 --- /dev/null +++ b/modules/restic/cmd/restic-backups/data.go @@ -0,0 +1,512 @@ +// The data the modules on this machine declare, measured (novox/hq ADR 0233). +// +// The mesh composes a second file beside the backup lines: every data item every module on the +// machine declares, one line each, +// +// item +// +// where covered-by is the path whose snapshot keeps it (the item itself, or the directory its dump +// writes into), or `-` when it is not backed up, and protection is backup, redundancy, both, or none. +// This holder measures each item that is not a cache — its size, its newest write, and the redundant +// storage it is on and whether that is healthy (ZFS, md, btrfs) — once an hour, and says it with each +// module's last good backup in `backed-up`. The controller keeps the readings and says when an item shrinks, stops being written, +// goes without a backup, or is replaced by an empty copy of itself; this holder only measures. +// +// And it deletes, when a person has decided: an item the mesh retired — its module gone from this +// machine — is removed by `backup_delete_retired`, and only after a last restore point of it has been +// taken, so the deletion can be undone until a person forgets that restore point. +package main + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "regexp" + "strconv" + "strings" + "sync" + "time" +) + +// Item is one data item a module declares. +type Item struct { + Module string `json:"-"` + ID string `json:"item"` + Class string `json:"class"` + Path string `json:"path"` + CoveredBy string `json:"covered_by,omitempty"` + Protection string `json:"protection,omitempty"` +} + +// Redundancy is the redundant storage an item is on, as read here. +type Redundancy struct { + // Kind is zfs, md or btrfs. + Kind string `json:"kind"` + // Where is the pool, the array device or the filesystem. + Where string `json:"where"` + // Healthy is nil when its state could not be read. + Healthy *bool `json:"healthy"` + Said string `json:"said"` +} + +// Measured is what one measurement found. +type Measured struct { + SizeBytes *int64 `json:"size_bytes,omitempty"` + LastWrite *time.Time `json:"last_write,omitempty"` + MeasuredAt *time.Time `json:"measured_at,omitempty"` + Error string `json:"error,omitempty"` + Redundancy *Redundancy `json:"redundancy,omitempty"` +} + +// ItemReport is one item as `backed-up` says it. +type ItemReport struct { + Item + Measured + LastBackup *time.Time `json:"last_backup,omitempty"` +} + +// The classes the mesh declares; a cache is listed and never measured. +const classCache = "cache" + +var safePath = regexp.MustCompile(`^/[^\s'"\\$` + "`" + `]*$`) + +// parseItems reads the composed data file into each module's items. A line this holder does not read +// is refused, naming the module, as a backup line is. +func parseItems(text string) (map[string][]Item, error) { + out := map[string][]Item{} + module := "" + for _, raw := range strings.Split(text, "\n") { + line := strings.TrimSpace(raw) + if line == "" { + continue + } + if m := moduleHeader.FindStringSubmatch(line); m != nil { + module = m[1] + continue + } + if strings.HasPrefix(line, "#") || module == "" { + continue + } + f := strings.Fields(line) + if (len(f) != 5 && len(f) != 6) || f[0] != "item" || !safePath.MatchString(f[3]) || (f[4] != "-" && !safePath.MatchString(f[4])) { + return nil, fmt.Errorf("%s declares a data line this holder does not read: %s", module, line) + } + it := Item{Module: module, ID: f[1], Class: f[2], Path: f[3]} + if len(f) == 6 { + it.Protection = f[5] + } + if f[4] != "-" { + it.CoveredBy = f[4] + } + out[module] = append(out[module], it) + } + return out, nil +} + +// Items is what the modules on this machine declare; none when the mesh composed no data file — an +// older controller, which composes none. +func (b *Backups) Items() (map[string][]Item, error) { + if b.Where.Data == "" { + return map[string][]Item{}, nil + } + raw, err := os.ReadFile(b.Where.Data) + if errors.Is(err, os.ErrNotExist) { + return map[string][]Item{}, nil + } + if err != nil { + return nil, err + } + return parseItems(string(raw)) +} + +func (b *Backups) measuredFile() string { return filepath.Join(b.Where.State, "measured.json") } + +// Measurements is the newest measurement of each item, by module and item. +func (b *Backups) Measurements() map[string]map[string]Measured { + out := map[string]map[string]Measured{} + if raw, err := os.ReadFile(b.measuredFile()); err == nil { + _ = json.Unmarshal(raw, &out) + } + return out +} + +var measuring sync.Mutex + +// measureCommand sums the files under a path and finds its newest change, in one walk, as root: a +// store's directory is often its own user's alone. One line out: " ". +func measureCommand(path string) string { + return "find '" + path + "' -xdev -printf '%y %s %T@\\n' 2>/dev/null | " + + "awk 'BEGIN{s=0;m=0} {if ($1==\"f\") s+=$2; if ($3>m) m=$3} END {printf \"%d %.0f\\n\", s, m}'" +} + +// measure is one item, measured now. +func (b *Backups) measure(ctx context.Context, it Item) Measured { + at := b.Now().UTC() + m := Measured{MeasuredAt: &at} + if !b.exists(ctx, it.Path) { + m.Error = it.Path + " does not exist" + zero := int64(0) + m.SizeBytes = &zero + return m + } + out, err := b.Run(ctx, "sh", "-c", measureCommand(it.Path)) + if err != nil { + m.Error = err.Error() + return m + } + f := strings.Fields(out) + if len(f) != 2 { + m.Error = "the measurement said " + strings.TrimSpace(out) + return m + } + size, err1 := strconv.ParseInt(f[0], 10, 64) + epoch, err2 := strconv.ParseInt(f[1], 10, 64) + if err1 != nil || err2 != nil { + m.Error = "the measurement said " + strings.TrimSpace(out) + return m + } + m.SizeBytes = &size + if epoch > 0 { + w := time.Unix(epoch, 0).UTC() + m.LastWrite = &w + } + m.Redundancy = b.redundancyOf(ctx, it.Path) + return m +} + +// redundancyOf is the redundant storage a path is on, read as root: a ZFS pool's health and its own +// verdict, an md array's members, a btrfs filesystem's error counters. Nil for storage with no +// redundancy this holder knows how to read — which, for an item said to be protected by redundancy, the +// controller says is no protection at all. +func (b *Backups) redundancyOf(ctx context.Context, path string) *Redundancy { + out, err := b.Run(ctx, "findmnt", "-no", "SOURCE,FSTYPE", "--target", path) + if err != nil { + return nil + } + f := strings.Fields(out) + if len(f) < 2 { + return nil + } + source, fstype := f[0], f[1] + yes, no := true, false + switch { + case fstype == "zfs": + pool, _, _ := strings.Cut(source, "/") + r := &Redundancy{Kind: "zfs", Where: pool} + health, err := b.Run(ctx, "zpool", "list", "-H", "-o", "health", pool) + if err != nil { + r.Said = "zpool list: " + err.Error() + return r + } + status, err := b.Run(ctx, "zpool", "status", "-x", pool) + if err != nil { + r.Said = "zpool status: " + err.Error() + return r + } + health, status = strings.TrimSpace(health), strings.TrimSpace(status) + r.Said = "health " + health + "; " + firstLineOf(status) + if health == "ONLINE" && strings.Contains(status, "is healthy") { + r.Healthy = &yes + } else { + r.Healthy = &no + r.Said = "health " + health + "; " + oneLineOf(status) + } + return r + case strings.HasPrefix(source, "/dev/md"): + device := strings.TrimPrefix(source, "/dev/") + r := &Redundancy{Kind: "md", Where: device} + mdstat, err := b.Run(ctx, "cat", "/proc/mdstat") + if err != nil { + r.Said = err.Error() + return r + } + healthy, said, found := mdHealth(mdstat, device) + if !found { + r.Said = device + " is not in /proc/mdstat" + return r + } + r.Said = said + if healthy { + r.Healthy = &yes + } else { + r.Healthy = &no + } + return r + case fstype == "btrfs": + r := &Redundancy{Kind: "btrfs", Where: source} + stats, err := b.Run(ctx, "btrfs", "device", "stats", "--check", path) + if err != nil { + r.Healthy, r.Said = &no, "device errors: "+oneLineOf(err.Error()) + return r + } + r.Healthy, r.Said = &yes, firstLineOf(stats) + return r + } + return nil +} + +var mdMembers = regexp.MustCompile(`\[([U_]+)\]`) + +// mdHealth reads one array's block of /proc/mdstat: healthy when every member is up and it is not +// recovering. +func mdHealth(mdstat, device string) (bool, string, bool) { + lines := strings.Split(mdstat, "\n") + for i, l := range lines { + if !strings.HasPrefix(l, device+" ") { + continue + } + block := l + for j := i + 1; j < len(lines) && strings.HasPrefix(lines[j], " "); j++ { + block += " " + strings.TrimSpace(lines[j]) + } + members := mdMembers.FindStringSubmatch(block) + healthy := members != nil && !strings.Contains(members[1], "_") && !strings.Contains(block, "recovery") + return healthy, oneLineOf(block), true + } + return false, "", false +} + +func firstLineOf(s string) string { + line, _, _ := strings.Cut(strings.TrimSpace(s), "\n") + return line +} + +func oneLineOf(s string) string { return strings.Join(strings.Fields(s), " ") } + +// MeasureAll measures every item that is not a cache, one at a time, and keeps what it found. +func (b *Backups) MeasureAll(ctx context.Context) error { + measuring.Lock() + defer measuring.Unlock() + items, err := b.Items() + if err != nil { + return err + } + found := map[string]map[string]Measured{} + for module, its := range items { + for _, it := range its { + if it.Class == classCache { + continue + } + if found[module] == nil { + found[module] = map[string]Measured{} + } + found[module][it.ID] = b.measure(ctx, it) + } + } + raw, _ := json.MarshalIndent(found, "", " ") + return os.WriteFile(b.measuredFile(), append(raw, '\n'), 0o600) +} + +func (b *Backups) goodFile() string { return filepath.Join(b.Where.State, "good.json") } + +// Good is each module's newest good night: what a night that failed since does not erase. +func (b *Backups) Good() map[string]time.Time { + out := map[string]time.Time{} + if raw, err := os.ReadFile(b.goodFile()); err == nil { + _ = json.Unmarshal(raw, &out) + } + // A night recorded good before this file existed counts. + for module, n := range b.Nights() { + if n.OK && n.At.After(out[module]) { + out[module] = n.At + } + } + return out +} + +func (b *Backups) recordGood(module string, at time.Time) { + good := b.Good() + good[module] = at + raw, _ := json.MarshalIndent(good, "", " ") + if err := os.WriteFile(b.goodFile(), append(raw, '\n'), 0o600); err != nil { + b.Say("recording %s's good night failed: %v", module, err) + } +} + +// itemReports is every item of one module, with its newest measurement and its newest good backup: the +// module's newest good night, where that night keeps the path that covers the item. +func itemReports(its []Item, measured map[string]Measured, paths []string, good time.Time) []ItemReport { + out := []ItemReport{} + for _, it := range its { + r := ItemReport{Item: it, Measured: measured[it.ID]} + if it.CoveredBy != "" && !good.IsZero() { + for _, p := range paths { + if p == it.CoveredBy { + g := good + r.LastBackup = &g + } + } + } + out = append(out, r) + } + return out +} + +// ---- deleting a retired item ------------------------------------------------------------------- + +// Deletion is how one deletion went, by path. +type Deletion struct { + Module string `json:"module"` + Item string `json:"item"` + Started time.Time `json:"started"` + Running bool `json:"running"` + Done bool `json:"done"` + OK bool `json:"ok"` + Snapshot string `json:"snapshot,omitempty"` + Error string `json:"error,omitempty"` + By string `json:"by,omitempty"` + Why string `json:"why,omitempty"` +} + +func (b *Backups) deletionsFile() string { return filepath.Join(b.Where.State, "deleted.json") } + +var deletionsMu sync.Mutex + +func (b *Backups) deletions() map[string]Deletion { + out := map[string]Deletion{} + if raw, err := os.ReadFile(b.deletionsFile()); err == nil { + _ = json.Unmarshal(raw, &out) + } + return out +} + +func (b *Backups) keepDeletion(path string, d Deletion) { + deletionsMu.Lock() + defer deletionsMu.Unlock() + all := b.deletions() + all[path] = d + raw, _ := json.MarshalIndent(all, "", " ") + if err := os.WriteFile(b.deletionsFile(), append(raw, '\n'), 0o600); err != nil { + b.Say("recording the deletion of %s failed: %v", path, err) + } +} + +// refuseDeleting says why a path may not be deleted: not absolute, too near the root, or declared now +// — by any module's item or backup line on this machine, itself, inside one, or holding one. +func (b *Backups) refuseDeleting(path string) error { + clean := filepath.Clean(path) + if !safePath.MatchString(path) || clean != strings.TrimRight(path, "/") { + return fmt.Errorf("%q is not a path this holder deletes", path) + } + if strings.Count(clean, "/") < 2 { + return fmt.Errorf("%s is too near the root to be a module's data", clean) + } + near := func(declared string) bool { + d := filepath.Clean(declared) + return d == clean || strings.HasPrefix(d, clean+"/") || strings.HasPrefix(clean, d+"/") + } + items, err := b.Items() + if err != nil { + return fmt.Errorf("what is declared here cannot be read, so nothing is deleted: %v", err) + } + for module, its := range items { + for _, it := range its { + if near(it.Path) { + return fmt.Errorf("%s is declared now — %s's %s at %s — so it is not retired; nothing is deleted", + clean, module, it.ID, it.Path) + } + } + } + declared, err := b.Declared() + if err != nil && !errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("what is backed up here cannot be read, so nothing is deleted: %v", err) + } + for _, d := range declared { + for _, p := range d.Paths { + if near(p) { + return fmt.Errorf("%s is backed up now as %s's %s, so it is not retired; nothing is deleted", clean, d.Module, p) + } + } + } + if clean == filepath.Clean(b.Where.Repository) || strings.HasPrefix(b.Where.Repository, clean+"/") || + clean == filepath.Clean(b.Where.State) { + return fmt.Errorf("%s holds this machine's backups; nothing is deleted", clean) + } + return nil +} + +// DeleteRetired starts deleting one retired item: a last restore point of it, tagged as retired, then +// its removal — never the removal without the restore point. Answers at once; DeletedOutcome follows +// it. A path whose deletion already finished answers how it went. +func (b *Backups) DeleteRetired(ctx context.Context, module, item, path, confirm, by, why string) (Deletion, error) { + if module == "" || item == "" || path == "" { + return Deletion{}, errors.New("module, item and path are required") + } + if confirm != item { + return Deletion{}, fmt.Errorf("confirm is the item's name again (%s), to say this is meant", item) + } + if strings.TrimSpace(why) == "" { + return Deletion{}, errors.New("why is required: a deletion is a person's decision, and says why") + } + if d, ok := b.deletions()[path]; ok && (d.Running || (d.Done && d.OK)) { + return d, nil + } + if err := b.refuseDeleting(path); err != nil { + return Deletion{}, err + } + if !b.exists(ctx, path) { + d := Deletion{Module: module, Item: item, Started: b.Now().UTC(), Done: true, OK: true, + Error: path + " was already gone", By: by, Why: why} + b.keepDeletion(path, d) + return d, nil + } + d := Deletion{Module: module, Item: item, Started: b.Now().UTC(), Running: true, By: by, Why: why} + b.keepDeletion(path, d) + go b.deleteNow(context.WithoutCancel(ctx), path, d) + return d, nil +} + +func (b *Backups) deleteNow(ctx context.Context, path string, d Deletion) { + b.mu.Lock() + defer b.mu.Unlock() + finish := func(err error) { + d.Running, d.Done = false, true + if err != nil { + d.Error = err.Error() + b.Say("%s's retired %s at %s was NOT deleted: %v", d.Module, d.Item, path, err) + } else { + d.OK = true + b.Say("%s's retired %s at %s DELETED by %s: %s; its last restore point is %s", d.Module, d.Item, path, + orSomebody(d.By), d.Why, d.Snapshot) + } + b.keepDeletion(path, d) + } + if err := b.ensureRepository(ctx); err != nil { + finish(fmt.Errorf("no restore point could be taken first: %w", err)) + return + } + out, err := b.restic(ctx, "backup", "--json", "--tag", tagOf(d.Module), "--tag", "retired="+d.Item, path) + if err != nil { + finish(fmt.Errorf("the last restore point could not be taken, so nothing was deleted: %w", err)) + return + } + d.Snapshot = snapshotOf(out) + if d.Snapshot == "" { + finish(errors.New("restic took the last restore point and named none, so nothing was deleted")) + return + } + if _, err := b.Run(ctx, "rm", "-rf", "--one-file-system", "--", path); err != nil { + finish(err) + return + } + finish(nil) +} + +// DeletedOutcome is how the deletion of a path went. +func (b *Backups) DeletedOutcome(path string) (Deletion, error) { + d, ok := b.deletions()[path] + if !ok { + return Deletion{}, fmt.Errorf("no deletion of %s was asked of this holder", path) + } + return d, nil +} + +func orSomebody(by string) string { + if by == "" { + return "somebody" + } + return by +} diff --git a/modules/restic/cmd/restic-backups/data_test.go b/modules/restic/cmd/restic-backups/data_test.go new file mode 100644 index 0000000..79582b3 --- /dev/null +++ b/modules/restic/cmd/restic-backups/data_test.go @@ -0,0 +1,241 @@ +package main + +// The data the modules declare, measured, and a retired item deleted only after a last restore point +// (novox/hq ADR 0233) — over a fake restic and a fake machine. + +import ( + "context" + "errors" + "os" + "path/filepath" + "strings" + "sync" + "testing" + "time" +) + +const composedData = "# The data the modules on this machine declare, composed by the mesh. Do not edit.\n" + + "# postgres\nitem store irreplaceable /var/lib/mesh-store /var/lib/mesh-store/dumps\nitem dumps rebuildable /var/lib/mesh-store/dumps -\n" + + "# searxng\nitem valkey cache /var/lib/searxng/valkey-data -\n" + +func withData(t *testing.T, declared, data string) Where { + t.Helper() + w := placed(t, declared) + w.Data = filepath.Join(w.State, "data.conf") + must(t, os.WriteFile(w.Data, []byte(data), 0o600)) + return w +} + +func TestTheComposedDataFileIsReadIntoEachModulesItems(t *testing.T) { + got, err := parseItems(composedData) + must(t, err) + if len(got["postgres"]) != 2 || got["postgres"][0].CoveredBy != "/var/lib/mesh-store/dumps" || got["postgres"][1].CoveredBy != "" || + got["searxng"][0].Class != "cache" { + t.Fatalf("%+v", got) + } + for _, bad := range []string{"# pg\nitem a irreplaceable relative -\n", "# pg\nitem a irreplaceable /x\n", "# pg\nitem a b /x; rm -rf / -\n"} { + if _, err := parseItems(bad); err == nil || !strings.Contains(err.Error(), "pg declares a data line") { + t.Errorf("%q: %v", bad, err) + } + } + // An older controller composes no data file: nothing is measured, nothing fails. + b := &Backups{Where: placed(t, composed), Now: time.Now, Say: quiet} + if items, err := b.Items(); err != nil || len(items) != 0 { + t.Fatalf("%v, %v", items, err) + } +} + +// Every item but a cache is measured — its files' size and its newest change, in one walk as root — and +// `backed-up` says each with the newest good night of the module that keeps the path covering it. +func TestEveryItemButACacheIsMeasuredAndSaidWithItsLastGoodBackup(t *testing.T) { + where := withData(t, composed, composedData) + var mu sync.Mutex + var walked []string + run := func(_ context.Context, name string, args ...string) (string, error) { + mu.Lock() + defer mu.Unlock() + switch { + case name == "test": + return "", nil + case name == "sh" && strings.Contains(args[1], "find '"): + walked = append(walked, args[1]) + if strings.Contains(args[1], "'/var/lib/mesh-store'") { + return "1073741824 1759744800\n", nil + } + return "5000 1759700000\n", nil + case name == "restic": + return "[]", nil + } + return "", nil + } + night := time.Date(2026, 10, 6, 3, 0, 0, 0, time.UTC) + b := &Backups{Where: where, Run: run, Now: func() time.Time { return night.Add(time.Hour) }, Say: quiet} + must(t, b.MeasureAll(context.Background())) + if len(walked) != 2 { + t.Fatalf("walked %d paths, want the two that are not a cache: %v", len(walked), walked) + } + b.recordGood("postgres", night) + got, err := b.BackedUp(context.Background(), "") + must(t, err) + var store, dumps *ItemReport + for _, m := range got { + for i := range m.Data { + switch m.Data[i].ID { + case "store": + store = &m.Data[i] + case "dumps": + dumps = &m.Data[i] + } + } + } + if store == nil || store.SizeBytes == nil || *store.SizeBytes != 1<<30 || store.LastWrite == nil || + store.LastWrite.Unix() != 1759744800 || store.LastBackup == nil || !store.LastBackup.Equal(night) { + t.Fatalf("the store is said as %+v", store) + } + if dumps == nil || dumps.LastBackup != nil { + t.Fatalf("an item not backed up is said backed up: %+v", dumps) + } +} + +// A retired item is deleted only after a last restore point of it, tagged as retired; a restore point +// that fails deletes nothing; and nothing declared now — itself, inside it, or holding it — is deleted. +func TestARetiredItemIsDeletedOnlyAfterALastRestorePoint(t *testing.T) { + where := withData(t, composed, composedData) + var mu sync.Mutex + var calls []string + failBackup := false + run := func(_ context.Context, name string, args ...string) (string, error) { + mu.Lock() + defer mu.Unlock() + line := name + " " + strings.Join(args, " ") + if name == "restic" { + line = "restic " + strings.Join(args[5:], " ") + } + calls = append(calls, line) + switch { + case name == "test": + return "", nil + case strings.HasPrefix(line, "restic backup"): + if failBackup { + return "", errors.New("the repository is locked") + } + return "{\"message_type\":\"summary\",\"snapshot_id\":\"0123456789abcdef\"}\n", nil + } + return "", nil + } + b := &Backups{Where: where, Run: run, Now: time.Now, Say: quiet} + ctx := context.Background() + for _, refused := range []struct{ path, want string }{ + {"/var/lib/mesh-store", "declared now"}, + {"/var/lib/mesh-store/dumps/old", "declared now"}, + {"/var/lib", "declared now"}, + {"/var/lib/mailu/data-mail", "backed up now"}, + {"/srv", "too near the root"}, + {"relative/x", "not a path"}, + } { + if _, err := b.DeleteRetired(ctx, "m", "i", refused.path, "i", "op", "tidy"); err == nil || !strings.Contains(err.Error(), refused.want) { + t.Errorf("%s: %v, want %q", refused.path, err, refused.want) + } + } + if _, err := b.DeleteRetired(ctx, "house", "config", "/var/lib/house/config", "nope", "op", "tidy"); err == nil { + t.Error("a deletion without the item's name again was started") + } + if _, err := b.DeleteRetired(ctx, "house", "config", "/var/lib/house/config", "config", "op", ""); err == nil { + t.Error("a deletion without why was started") + } + + wait := func(path string) Deletion { + t.Helper() + for i := 0; i < 200; i++ { + if d, err := b.DeletedOutcome(path); err == nil && d.Done { + return d + } + time.Sleep(5 * time.Millisecond) + } + t.Fatalf("the deletion of %s never finished", path) + return Deletion{} + } + + mu.Lock() + failBackup = true + mu.Unlock() + _, err := b.DeleteRetired(ctx, "house", "config", "/var/lib/house/config", "config", "op", "moved to the anchor") + must(t, err) + if d := wait("/var/lib/house/config"); d.OK || !strings.Contains(d.Error, "nothing was deleted") { + t.Fatalf("a deletion with no restore point: %+v", d) + } + mu.Lock() + for _, c := range calls { + if strings.HasPrefix(c, "rm ") { + t.Fatal("it deleted without a last restore point") + } + } + mu.Unlock() + + mu.Lock() + failBackup, calls = false, nil + mu.Unlock() + _, err = b.DeleteRetired(ctx, "house", "config", "/var/lib/house/config", "config", "op", "moved to the anchor") + must(t, err) + d := wait("/var/lib/house/config") + if !d.OK || d.Snapshot != "01234567" { + t.Fatalf("%+v", d) + } + var backup, rm = -1, -1 + mu.Lock() + defer mu.Unlock() + for i, c := range calls { + switch { + case c == "restic backup --json --tag module=house --tag retired=config /var/lib/house/config": + backup = i + case c == "rm -rf --one-file-system -- /var/lib/house/config": + rm = i + } + } + if backup < 0 || rm < 0 || rm < backup { + t.Fatalf("ran %v", calls) + } + // Asked again, it answers how it went rather than starting over. + again, err := b.DeleteRetired(ctx, "house", "config", "/var/lib/house/config", "config", "op", "moved") + if err != nil || !again.Done || !again.OK { + t.Fatalf("%+v, %v", again, err) + } +} + +// The redundant storage under an item is read: a ZFS pool healthy, then degraded; an md array with a +// member missing; and plain storage, which is no redundancy at all. +func TestTheRedundantStorageUnderAnItemIsRead(t *testing.T) { + health, statusX := "ONLINE\n", "pool 'storage' is healthy\n" + fs := "storage/media zfs\n" + run := func(_ context.Context, name string, args ...string) (string, error) { + switch name { + case "findmnt": + return fs, nil + case "zpool": + if args[0] == "list" { + return health, nil + } + return statusX, nil + case "cat": + return "Personalities : [raid1]\nmd127 : active raid1 sdb1[1] sda1[0]\n 976630464 blocks super 1.2 [2/1] [U_]\n\nunused devices: \n", nil + } + return "", nil + } + b := &Backups{Run: run, Now: time.Now, Say: quiet} + r := b.redundancyOf(context.Background(), "/storage/media/movies") + if r == nil || r.Kind != "zfs" || r.Where != "storage" || r.Healthy == nil || !*r.Healthy { + t.Fatalf("a healthy pool: %+v", r) + } + health, statusX = "DEGRADED\n", " pool: storage\n state: DEGRADED\nstatus: One or more devices has been removed\n" + if r := b.redundancyOf(context.Background(), "/storage/media/movies"); r.Healthy == nil || *r.Healthy || !strings.Contains(r.Said, "DEGRADED") { + t.Fatalf("a degraded pool: %+v", r) + } + fs = "/dev/md127 ext4\n" + if r := b.redundancyOf(context.Background(), "/srv/x"); r == nil || r.Kind != "md" || r.Healthy == nil || *r.Healthy { + t.Fatalf("an array missing a member: %+v", r) + } + fs = "/dev/nvme0n1p3 ext4\n" + if r := b.redundancyOf(context.Background(), "/var/lib/x"); r != nil { + t.Fatalf("plain storage read as redundant: %+v", r) + } +} diff --git a/modules/restic/cmd/restic-backups/main.go b/modules/restic/cmd/restic-backups/main.go index 9dc24f9..2e31bb9 100644 --- a/modules/restic/cmd/restic-backups/main.go +++ b/modules/restic/cmd/restic-backups/main.go @@ -32,6 +32,7 @@ func main() { } b := &Backups{Where: where, Run: execRunner, Now: time.Now, Say: say} go nights(b) + go measurements(b) if err := stdio.Serve("", tools(b)); err != nil { say("%v", err) os.Exit(1) @@ -56,6 +57,22 @@ func nights(b *Backups) { } } +// measurements measures every declared item once an hour (MESH_BACKUP_MEASURE_EVERY to change it), +// the first a few minutes after start, and after every night (novox/hq ADR 0233). +func measurements(b *Backups) { + every := time.Hour + if d, err := time.ParseDuration(os.Getenv("MESH_BACKUP_MEASURE_EVERY")); err == nil && d >= time.Minute { + every = d + } + time.Sleep(3 * time.Minute) + for { + if err := b.MeasureAll(context.Background()); err != nil { + say("measuring the data declared here failed: %v", err) + } + time.Sleep(every) + } +} + func night(b *Backups) { ctx := context.Background() outcome, err := b.BackUp(ctx, "") @@ -72,6 +89,9 @@ func night(b *Backups) { if len(failed) > 0 { say("the night left %s without a backup", strings.Join(failed, ", ")) } + if err := b.MeasureAll(ctx); err != nil { + say("measuring the data declared here failed: %v", err) + } // Sundays, the repository's own integrity with a sample of the data read back. if time.Now().Weekday() == time.Sunday { if err := b.Check(ctx); err != nil { @@ -133,5 +153,22 @@ func tools(b *Backups) []stdio.Tool { } return b.Restore(context.Background(), module, arg(a, "snapshot"), arg(a, "path")) }), + // Deleting a retired item, when a person decided (novox/hq ADR 0233): the controller's `cleanup + // delete` asks these. Not seat verbs — nobody else calls them. + {Name: "backup_delete_retired", Description: "Delete one item of data the mesh retired on this machine — " + + "its module is gone from here — after taking a last restore point of it, tagged retired, so it can be " + + "restored until that restore point is forgotten. Refuses anything declared now. Answers at once; " + + "backup_deleted says how it went. Use the controller's `cleanup delete`, which records the hand act.", + Input: map[string]any{"module": str("the module it was"), "item": str("the item"), "path": str("where it is"), + "confirm": str("the item's name again"), "why": str("why — required"), "by": str("who decided"), + "via": str("mesh-controller when asked through its verbs")}, + Run: func(a map[string]any) (any, error) { + return b.DeleteRetired(context.Background(), arg(a, "module"), arg(a, "item"), arg(a, "path"), + arg(a, "confirm"), arg(a, "by"), arg(a, "why")) + }}, + {Name: "backup_deleted", Description: "How the deletion of one retired item went: running, or done with " + + "its last restore point, or refused with why.", + Input: map[string]any{"path": str("where it is")}, + Run: func(a map[string]any) (any, error) { return b.DeletedOutcome(arg(a, "path")) }}, } } diff --git a/modules/restic/module.json b/modules/restic/module.json index 2e35e61..9e81a43 100644 --- a/modules/restic/module.json +++ b/modules/restic/module.json @@ -15,6 +15,23 @@ "own-secrets": { "repository": "${dir:state}/repository.secret" }, + "data": { + "own": [ + { + "id": "repository", + "path": "${dir:repository}", + "class": "rebuildable", + "backup": "none", + "why": "the restore points themselves: a copy of data kept elsewhere on this machine, never the only copy of anything; lost, the history goes and nothing live does" + }, + { + "id": "state", + "path": "${dir:state}", + "class": "cache", + "why": "what the last nights and measurements were; the next night writes it again" + } + ] + }, "resources": [ { "id": "state", @@ -34,6 +51,13 @@ "mode": "0600", "content": "# What the modules on this machine back up, composed by the mesh (novox/hq to-be 43). Do not edit.\n${contribution:node-backup:backup}" }, + { + "id": "data-declared", + "type": "file", + "path": "${dir:state}/data.conf", + "mode": "0600", + "content": "# The data the modules on this machine declare, composed by the mesh (novox/hq ADR 0233). Do not edit.\n${contribution:node-backup:data}" + }, { "id": "tool", "type": "package", @@ -61,7 +85,8 @@ "MESH_BACKUP_DECLARED": "${dir:state}/backups.conf", "MESH_BACKUP_REPOSITORY": "${dir:repository}", "MESH_BACKUP_PASSWORD_FILE": "${dir:state}/repository.secret", - "MESH_BACKUP_STATE": "${dir:state}" + "MESH_BACKUP_STATE": "${dir:state}", + "MESH_BACKUP_DATA": "${dir:state}/data.conf" } } ] diff --git a/modules/route-proxy/module.json b/modules/route-proxy/module.json index a58eb5c..38444a4 100644 --- a/modules/route-proxy/module.json +++ b/modules/route-proxy/module.json @@ -43,6 +43,22 @@ "why": "public HTTPS for every name the mesh routes here" } ], + "data": { + "own": [ + { + "id": "acme", + "path": "${dir:acme-cache}", + "class": "rebuildable", + "why": "issued certificates, issued again" + }, + { + "id": "ca", + "path": "${dir:ca-dir}", + "class": "cache", + "why": "the authority's roots, fetched again at every start" + } + ] + }, "resources": [ { "id": "state", diff --git a/modules/searxng/module.json b/modules/searxng/module.json index 69b1adc..647fbd3 100644 --- a/modules/searxng/module.json +++ b/modules/searxng/module.json @@ -19,6 +19,16 @@ "why": "the search pages" } ], + "data": { + "own": [ + { + "id": "valkey", + "path": "${dir:valkey-data}", + "class": "cache", + "why": "the search cache" + } + ] + }, "resources": [ { "id": "mesh-state", diff --git a/modules/ssh-client/module.json b/modules/ssh-client/module.json index 970768b..e6ac8f1 100644 --- a/modules/ssh-client/module.json +++ b/modules/ssh-client/module.json @@ -11,6 +11,16 @@ "ssh_client_known_host", "ssh_client_test" ], + "data": { + "own": [ + { + "id": "ssh", + "path": "${dir:ssh-dir}", + "class": "valuable", + "why": "the operator's keys and known hosts" + } + ] + }, "resources": [ { "id": "ssh-dir", diff --git a/modules/step-ca/module.json b/modules/step-ca/module.json index cbdd57f..159ce47 100644 --- a/modules/step-ca/module.json +++ b/modules/step-ca/module.json @@ -28,6 +28,16 @@ "own-secrets": { "password": "${dir:mesh-state}/password" }, + "data": { + "own": [ + { + "id": "home", + "path": "${dir:home}", + "class": "valuable", + "why": "the mesh's certificate authority: its keys and its database of what it issued" + } + ] + }, "resources": [ { "id": "mesh-state", diff --git a/modules/supabase/module.json b/modules/supabase/module.json index f72e979..0934674 100644 --- a/modules/supabase/module.json +++ b/modules/supabase/module.json @@ -58,6 +58,34 @@ "binds": { "route": "${dir:state}/route.json" }, + "data": { + "own": [ + { + "id": "db", + "path": "${dir:db-data}", + "class": "valuable", + "why": "every table; copied as live files, which may not restore \u2014 a dump is wanted" + }, + { + "id": "storage", + "path": "${dir:storage}", + "class": "valuable", + "why": "uploaded objects" + }, + { + "id": "functions", + "path": "${dir:functions}", + "class": "valuable", + "why": "the edge functions' code" + }, + { + "id": "db-config", + "path": "${dir:db-config}", + "class": "rebuildable", + "why": "the database's configuration, seeded again" + } + ] + }, "resources": [ { "id": "state", diff --git a/modules/umami/module.json b/modules/umami/module.json index 7452f9f..48cbe61 100644 --- a/modules/umami/module.json +++ b/modules/umami/module.json @@ -53,6 +53,15 @@ "why": "one port serves two surfaces — the dashboard and the collection endpoint that the browsers of every tracked site POST to. Both are reached through the proxy, by name, so the port is how the proxy reaches this module and nothing else (novox/hq ADR 0045). It said \"anywhere\" and gave the reason that the collection endpoint must be public, which is true of the name and not of the port: opened, the machine-side port served the dashboard over plain HTTP to the internet, bypassing every rule the proxy applies by path" } ], + "data": { + "consumers": { + "analytics": { + "class": "valuable", + "in": "postgres-database", + "why": "a site's page views are recorded nowhere else" + } + } + }, "resources": [ { "id": "state", diff --git a/modules/unifi/module.json b/modules/unifi/module.json index 76744e5..4c5ddd6 100644 --- a/modules/unifi/module.json +++ b/modules/unifi/module.json @@ -69,6 +69,16 @@ "why": "remote syslog from managed devices" } ], + "data": { + "own": [ + { + "id": "data", + "path": "${dir:data}", + "class": "valuable", + "why": "the network's configuration and every adopted device" + } + ] + }, "resources": [ { "id": "mesh-state", From 7524390cad17d9c47cdd8ad2f9cdf5bdd8672da2 Mon Sep 17 00:00:00 2001 From: jochen Date: Tue, 6 Oct 2026 17:00:23 +0200 Subject: [PATCH 2/2] Bound the holder's measuring; dump the database platform (hq ADR 0233) Walks run at most daily and stop after ten minutes or two million files; datasets are read from their counters and large items from their top level only. The database platform's tables are dumped with pg_dumpall rather than copied as live files. --- modules/distribution/module.json | 1 + modules/ollama/module.json | 1 + modules/restic/cmd/restic-backups/data.go | 148 ++++++++++++++---- .../restic/cmd/restic-backups/data_test.go | 72 ++++++++- modules/restic/cmd/restic-backups/main.go | 7 +- modules/supabase/module.json | 18 ++- 6 files changed, 211 insertions(+), 36 deletions(-) diff --git a/modules/distribution/module.json b/modules/distribution/module.json index 06446f7..e8eb32d 100644 --- a/modules/distribution/module.json +++ b/modules/distribution/module.json @@ -43,6 +43,7 @@ "path": "${dir:registry-data}", "class": "rebuildable", "backup": "none", + "measure": "shallow", "why": "the artifact store: every image is built again from its source, and too large to copy every night (ADR 0214 left it out)" } ] diff --git a/modules/ollama/module.json b/modules/ollama/module.json index 12bad0a..2291c63 100644 --- a/modules/ollama/module.json +++ b/modules/ollama/module.json @@ -32,6 +32,7 @@ "path": "${dir:state}", "class": "rebuildable", "backup": "none", + "measure": "shallow", "why": "models, downloaded again, and too large to copy every night" } ] diff --git a/modules/restic/cmd/restic-backups/data.go b/modules/restic/cmd/restic-backups/data.go index 429eba4..f0d6dd5 100644 --- a/modules/restic/cmd/restic-backups/data.go +++ b/modules/restic/cmd/restic-backups/data.go @@ -3,13 +3,23 @@ // The mesh composes a second file beside the backup lines: every data item every module on the // machine declares, one line each, // -// item +// item // // where covered-by is the path whose snapshot keeps it (the item itself, or the directory its dump -// writes into), or `-` when it is not backed up, and protection is backup, redundancy, both, or none. -// This holder measures each item that is not a cache — its size, its newest write, and the redundant -// storage it is on and whether that is healthy (ZFS, md, btrfs) — once an hour, and says it with each -// module's last good backup in `backed-up`. The controller keeps the readings and says when an item shrinks, stops being written, +// writes into), or `-` when it is not backed up; protection is backup, redundancy, both, or none; and +// measure is how the item is measured. This holder measures each item that is not a cache — its size, +// its newest write, and the redundant storage it is on and whether that is healthy (ZFS, md, btrfs) — +// and says it, with each module's last good backup and the precision of the measurement, in +// `backed-up`. +// +// **Never an unbounded walk of something large.** Three methods, the item's own choice: +// +// - `walk` (the default, for small items): every file summed and the newest change found, in one walk +// as root — at most once a day, and stopped after walkFor or walkFiles, said as "measured partially"; +// - `dataset`: the ZFS dataset holding the path — its `used` from the filesystem's own counters — and +// the newest change among the path's top-level entries; hourly, and nothing is walked; +// - `shallow`: the newest change among the top-level entries only, and no size; hourly. The controller keeps the readings and says when an item shrinks, stops being written, +// // goes without a backup, or is replaced by an empty copy of itself; this holder only measures. // // And it deletes, when a person has decided: an item the mesh retired — its module gone from this @@ -33,6 +43,8 @@ import ( // Item is one data item a module declares. type Item struct { + // Measure is walk, dataset or shallow. + Measure string `json:"measure,omitempty"` Module string `json:"-"` ID string `json:"item"` Class string `json:"class"` @@ -54,9 +66,12 @@ type Redundancy struct { // Measured is what one measurement found. type Measured struct { - SizeBytes *int64 `json:"size_bytes,omitempty"` - LastWrite *time.Time `json:"last_write,omitempty"` - MeasuredAt *time.Time `json:"measured_at,omitempty"` + SizeBytes *int64 `json:"size_bytes,omitempty"` + LastWrite *time.Time `json:"last_write,omitempty"` + MeasuredAt *time.Time `json:"measured_at,omitempty"` + // Precision says what the size is: "exact", "dataset : its whole size", "partial: …" (a lower + // bound, never compared), or "no size: …". + Precision string `json:"precision,omitempty"` Error string `json:"error,omitempty"` Redundancy *Redundancy `json:"redundancy,omitempty"` } @@ -91,13 +106,22 @@ func parseItems(text string) (map[string][]Item, error) { continue } f := strings.Fields(line) - if (len(f) != 5 && len(f) != 6) || f[0] != "item" || !safePath.MatchString(f[3]) || (f[4] != "-" && !safePath.MatchString(f[4])) { + if len(f) < 5 || len(f) > 7 || f[0] != "item" || !safePath.MatchString(f[3]) || (f[4] != "-" && !safePath.MatchString(f[4])) { return nil, fmt.Errorf("%s declares a data line this holder does not read: %s", module, line) } it := Item{Module: module, ID: f[1], Class: f[2], Path: f[3]} - if len(f) == 6 { + if len(f) >= 6 { it.Protection = f[5] } + it.Measure = measureWalk + if len(f) == 7 { + it.Measure = f[6] + } + switch it.Measure { + case measureWalk, measureDataset, measureShallow: + default: + return nil, fmt.Errorf("%s declares a data line this holder does not read: %s", module, line) + } if f[4] != "-" { it.CoveredBy = f[4] } @@ -135,45 +159,105 @@ func (b *Backups) Measurements() map[string]map[string]Measured { var measuring sync.Mutex -// measureCommand sums the files under a path and finds its newest change, in one walk, as root: a -// store's directory is often its own user's alone. One line out: " ". -func measureCommand(path string) string { - return "find '" + path + "' -xdev -printf '%y %s %T@\\n' 2>/dev/null | " + - "awk 'BEGIN{s=0;m=0} {if ($1==\"f\") s+=$2; if ($3>m) m=$3} END {printf \"%d %.0f\\n\", s, m}'" +// The ways an item is measured. +const ( + measureWalk = "walk" + measureDataset = "dataset" + measureShallow = "shallow" +) + +// The bounds on a walk, and how often one runs: a walk is for small items, and one that meets a large +// one stops and says so rather than loading the disks for hours. +var ( + walkFor = 10 * time.Minute + walkFiles = 2_000_000 + walkEvery = 23 * time.Hour +) + +// walkCommand sums the files under a path and finds its newest change, in one walk, as root — bounded +// by time and by files. One line out: " ". +func walkCommand(path string) string { + return fmt.Sprintf("timeout %d find '%s' -xdev -printf '%%y %%s %%T@\\n' 2>/dev/null | head -n %d | "+ + "awk 'BEGIN{s=0;m=0;n=0} {n++; if ($1==\"f\") s+=$2; if ($3>m) m=$3} END {printf \"%%d %%.0f %%d\\n\", s, m, n}'; "+ + "echo \"${PIPESTATUS[0]:-0}\"", int(walkFor.Seconds()), path, walkFiles) } -// measure is one item, measured now. +// topCommand is the newest change among a path and its top-level entries, and how many there are: +// one directory read, bounded. " ". +func topCommand(path string) string { + return "timeout 60 find '" + path + "' -maxdepth 1 -printf '%T@\\n' 2>/dev/null | " + + "awk 'BEGIN{m=0;n=0} {n++; if ($1>m) m=$1} END {printf \"%.0f %d\\n\", m, n-1}'" +} + +func epochTime(s string) *time.Time { + epoch, err := strconv.ParseInt(s, 10, 64) + if err != nil || epoch <= 0 { + return nil + } + t := time.Unix(epoch, 0).UTC() + return &t +} + +// measure is one item, measured now, by its own method. func (b *Backups) measure(ctx context.Context, it Item) Measured { at := b.Now().UTC() m := Measured{MeasuredAt: &at} if !b.exists(ctx, it.Path) { m.Error = it.Path + " does not exist" zero := int64(0) - m.SizeBytes = &zero + m.SizeBytes, m.Precision = &zero, "exact" return m } - out, err := b.Run(ctx, "sh", "-c", measureCommand(it.Path)) + m.Redundancy = b.redundancyOf(ctx, it.Path) + switch it.Measure { + case measureDataset, measureShallow: + out, err := b.Run(ctx, "bash", "-c", topCommand(it.Path)) + if f := strings.Fields(out); err == nil && len(f) == 2 { + m.LastWrite = epochTime(f[0]) + } else if err != nil { + m.Error = err.Error() + } + m.Precision = "no size: the newest change among its top-level entries only" + if it.Measure == measureShallow { + return m + } + out, err = b.Run(ctx, "zfs", "list", "-Hp", "-o", "name,used", it.Path) + f := strings.Fields(out) + if err != nil || len(f) != 2 { + m.Precision = "no size: it is on no ZFS dataset to read one from; the newest change among its top-level entries only" + return m + } + used, err := strconv.ParseInt(f[1], 10, 64) + if err != nil { + return m + } + m.SizeBytes = &used + m.Precision = "dataset " + f[0] + ": its whole size, from the filesystem's counters; the newest change among the top-level entries" + return m + } + out, err := b.Run(ctx, "bash", "-c", walkCommand(it.Path)) if err != nil { m.Error = err.Error() return m } - f := strings.Fields(out) - if len(f) != 2 { + lines := strings.Split(strings.TrimSpace(out), "\n") + f := strings.Fields(lines[0]) + if len(f) != 3 || len(lines) < 2 { m.Error = "the measurement said " + strings.TrimSpace(out) return m } size, err1 := strconv.ParseInt(f[0], 10, 64) - epoch, err2 := strconv.ParseInt(f[1], 10, 64) + files, err2 := strconv.Atoi(f[2]) if err1 != nil || err2 != nil { m.Error = "the measurement said " + strings.TrimSpace(out) return m } - m.SizeBytes = &size - if epoch > 0 { - w := time.Unix(epoch, 0).UTC() - m.LastWrite = &w + m.SizeBytes, m.LastWrite = &size, epochTime(f[1]) + m.Precision = "exact" + if status := strings.TrimSpace(lines[len(lines)-1]); status == "124" || files >= walkFiles { + m.Precision = fmt.Sprintf("partial: measured partially — stopped after %s or %d files; the size is a lower bound, "+ + "and this item wants measure: dataset or shallow", walkFor, walkFiles) } - m.Redundancy = b.redundancyOf(ctx, it.Path) return m } @@ -276,14 +360,17 @@ func firstLineOf(s string) string { func oneLineOf(s string) string { return strings.Join(strings.Fields(s), " ") } -// MeasureAll measures every item that is not a cache, one at a time, and keeps what it found. -func (b *Backups) MeasureAll(ctx context.Context) error { +// MeasureAll measures every item that is not a cache, one at a time, and keeps what it found. An item +// measured by a walk is walked only when walks is true or its last walk is older than walkEvery: the +// hourly round reads counters, and a walk runs at most once a day. +func (b *Backups) MeasureAll(ctx context.Context, walks bool) error { measuring.Lock() defer measuring.Unlock() items, err := b.Items() if err != nil { return err } + before := b.Measurements() found := map[string]map[string]Measured{} for module, its := range items { for _, it := range its { @@ -293,6 +380,11 @@ func (b *Backups) MeasureAll(ctx context.Context) error { if found[module] == nil { found[module] = map[string]Measured{} } + if was, ok := before[module][it.ID]; ok && it.Measure == measureWalk && !walks && was.MeasuredAt != nil && + b.Now().Sub(*was.MeasuredAt) < walkEvery { + found[module][it.ID] = was + continue + } found[module][it.ID] = b.measure(ctx, it) } } diff --git a/modules/restic/cmd/restic-backups/data_test.go b/modules/restic/cmd/restic-backups/data_test.go index 79582b3..0834ad6 100644 --- a/modules/restic/cmd/restic-backups/data_test.go +++ b/modules/restic/cmd/restic-backups/data_test.go @@ -6,6 +6,7 @@ package main import ( "context" "errors" + "fmt" "os" "path/filepath" "strings" @@ -57,12 +58,12 @@ func TestEveryItemButACacheIsMeasuredAndSaidWithItsLastGoodBackup(t *testing.T) switch { case name == "test": return "", nil - case name == "sh" && strings.Contains(args[1], "find '"): + case name == "bash" && strings.Contains(args[1], "find '"): walked = append(walked, args[1]) if strings.Contains(args[1], "'/var/lib/mesh-store'") { - return "1073741824 1759744800\n", nil + return "1073741824 1759744800 4000\n0\n", nil } - return "5000 1759700000\n", nil + return "5000 1759700000 3\n0\n", nil case name == "restic": return "[]", nil } @@ -70,7 +71,7 @@ func TestEveryItemButACacheIsMeasuredAndSaidWithItsLastGoodBackup(t *testing.T) } night := time.Date(2026, 10, 6, 3, 0, 0, 0, time.UTC) b := &Backups{Where: where, Run: run, Now: func() time.Time { return night.Add(time.Hour) }, Say: quiet} - must(t, b.MeasureAll(context.Background())) + must(t, b.MeasureAll(context.Background(), true)) if len(walked) != 2 { t.Fatalf("walked %d paths, want the two that are not a cache: %v", len(walked), walked) } @@ -239,3 +240,66 @@ func TestTheRedundantStorageUnderAnItemIsRead(t *testing.T) { t.Fatalf("plain storage read as redundant: %+v", r) } } + +// Large items are never walked: a dataset is measured from the filesystem's counters and its top-level +// entries, a shallow item from its top-level entries only; a walk that meets more than its bound stops +// and says so; and a walk runs at most once a day — the hourly round keeps the last one. +func TestNothingLargeIsWalkedAndAWalkIsBoundedAndDaily(t *testing.T) { + data := "# media\nitem movies irreplaceable /storage/media/movies - redundancy dataset\n" + + "item meta rebuildable /mnt/plex/config /mnt/plex/config backup shallow\n" + + "item big valuable /srv/big /srv/big backup\n" + where := withData(t, composed, data) + var mu sync.Mutex + var walks []string + truncated := true + run := func(_ context.Context, name string, args ...string) (string, error) { + mu.Lock() + defer mu.Unlock() + line := name + " " + strings.Join(args, " ") + switch { + case name == "test": + return "", nil + case name == "zfs" && args[len(args)-1] == "/storage/media/movies": + return "storage/media\t97067690722560\n", nil + case name == "zfs": + return "", errors.New("not a ZFS dataset") + case name == "bash" && strings.Contains(line, "-maxdepth 1"): + return "1759744800 12673\n", nil + case name == "bash": + walks = append(walks, line) + if strings.Contains(line, "-xdev") && !strings.Contains(line, "'/srv/big'") { + t.Errorf("walked something declared not to be walked: %s", line) + } + if truncated { + return fmt.Sprintf("123 1759700000 %d\n141\n", walkFiles), nil + } + return "999 1759700000 10\n0\n", nil + } + return "", nil + } + now := time.Date(2026, 10, 6, 12, 0, 0, 0, time.UTC) + b := &Backups{Where: where, Run: run, Now: func() time.Time { return now }, Say: quiet} + must(t, b.MeasureAll(context.Background(), false)) + got := b.Measurements()["media"] + if m := got["movies"]; m.SizeBytes == nil || *m.SizeBytes != 97067690722560 || !strings.HasPrefix(m.Precision, "dataset storage/media") || m.LastWrite == nil { + t.Fatalf("the library from its dataset: %+v", m) + } + if m := got["meta"]; m.SizeBytes != nil || !strings.HasPrefix(m.Precision, "no size") || m.LastWrite == nil { + t.Fatalf("a shallow item: %+v", m) + } + if m := got["big"]; !strings.HasPrefix(m.Precision, "partial") { + t.Fatalf("a walk stopped at its bound: %+v", m) + } + // An hour later the hourly round reads counters again and walks nothing. + now = now.Add(time.Hour) + truncated = false + must(t, b.MeasureAll(context.Background(), false)) + if len(walks) != 1 { + t.Fatalf("walked %d times in two rounds an hour apart: %v", len(walks), walks) + } + // After the night, it walks. + must(t, b.MeasureAll(context.Background(), true)) + if len(walks) != 2 || b.Measurements()["media"]["big"].Precision != "exact" { + t.Fatalf("%d walks, %+v", len(walks), b.Measurements()["media"]["big"]) + } +} diff --git a/modules/restic/cmd/restic-backups/main.go b/modules/restic/cmd/restic-backups/main.go index 2e31bb9..1e53e18 100644 --- a/modules/restic/cmd/restic-backups/main.go +++ b/modules/restic/cmd/restic-backups/main.go @@ -58,7 +58,8 @@ func nights(b *Backups) { } // measurements measures every declared item once an hour (MESH_BACKUP_MEASURE_EVERY to change it), -// the first a few minutes after start, and after every night (novox/hq ADR 0233). +// the first a few minutes after start — counters hourly, a walk at most daily — and every walk after +// each night (novox/hq ADR 0233). func measurements(b *Backups) { every := time.Hour if d, err := time.ParseDuration(os.Getenv("MESH_BACKUP_MEASURE_EVERY")); err == nil && d >= time.Minute { @@ -66,7 +67,7 @@ func measurements(b *Backups) { } time.Sleep(3 * time.Minute) for { - if err := b.MeasureAll(context.Background()); err != nil { + if err := b.MeasureAll(context.Background(), false); err != nil { say("measuring the data declared here failed: %v", err) } time.Sleep(every) @@ -89,7 +90,7 @@ func night(b *Backups) { if len(failed) > 0 { say("the night left %s without a backup", strings.Join(failed, ", ")) } - if err := b.MeasureAll(ctx); err != nil { + if err := b.MeasureAll(ctx, true); err != nil { say("measuring the data declared here failed: %v", err) } // Sundays, the repository's own integrity with a sample of the data read back. diff --git a/modules/supabase/module.json b/modules/supabase/module.json index 0934674..0fcb70d 100644 --- a/modules/supabase/module.json +++ b/modules/supabase/module.json @@ -64,7 +64,17 @@ "id": "db", "path": "${dir:db-data}", "class": "valuable", - "why": "every table; copied as live files, which may not restore \u2014 a dump is wanted" + "backup": { + "dump": "docker exec supabase-db pg_dumpall -h 127.0.0.1 -U supabase_admin > ${dir:dumps}/all.sql.partial && mv ${dir:dumps}/all.sql.partial ${dir:dumps}/all.sql", + "into": "dumps" + }, + "why": "every table; copied by pg_dumpall inside the database's container (local connections are trusted there), since a running store's files are not a consistent copy" + }, + { + "id": "dumps", + "path": "${dir:dumps}", + "class": "rebuildable", + "why": "last night's dump of the database, made again every night" }, { "id": "storage", @@ -111,6 +121,12 @@ "mode": "0700", "owner": "105:106" }, + { + "id": "dumps", + "type": "directory", + "path": "${dir:state}/dumps", + "mode": "0700" + }, { "id": "db-config", "type": "directory",