From 3c0496d1d350ffd7f33dc605a361da4622d118e9 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 21 Oct 2026 13:54:33 +0200 Subject: [PATCH 1/9] n8n runs its upstream image with uuid mounted and the media group joined, in place of a Dockerfile (issue 1639, ADR 0480 clauses 1, 3, 7, 8) --- modules/n8n/Dockerfile | 20 -------------------- modules/n8n/README.md | 7 ++++++- modules/n8n/module.json | 40 ++++++++++++++++++++++++++++++++-------- 3 files changed, 38 insertions(+), 29 deletions(-) delete mode 100644 modules/n8n/Dockerfile diff --git a/modules/n8n/Dockerfile b/modules/n8n/Dockerfile deleted file mode 100644 index 7d15a2bf..00000000 --- a/modules/n8n/Dockerfile +++ /dev/null @@ -1,20 +0,0 @@ -# n8n with what its workflows reach for beyond the upstream image. -# -# The base is named, not pinned here (novox/hq issue 044): module.json's `build.on` declares N8N_BASE -# as the upstream image by digest, and the mesh hands the build its own copy (ADR 0097). -ARG N8N_BASE -FROM ${N8N_BASE} - -USER root -# - `media` (GID 2000), with `node` in it: the shared media library is group-writable by the -# operator's media group, and a workflow files downloads into it. A container resource cannot add -# a supplementary group, so the image's own /etc/group carries it. 2000 is the operator's media -# group today; novox/hq 153 proposes reading it from the accessed data (${access::gid}). -# - uuid, pinned to the version the workflows were written against: Code nodes require() it -# (NODE_FUNCTION_ALLOW_EXTERNAL=*), and a Code node can only require what is installed. -RUN apk add --no-cache shadow \ - && groupadd -g 2000 media \ - && usermod -aG media node \ - && npm install -g uuid@14.0.1 - -USER node diff --git a/modules/n8n/README.md b/modules/n8n/README.md index c911c864..751b5f70 100644 --- a/modules/n8n/README.md +++ b/modules/n8n/README.md @@ -1,6 +1,10 @@ # n8n -The `n8n` module of the mesh catalogue: runs the container image `redis`; runs the container image `selenium/standalone-chrome`. +The `n8n` module of the mesh catalogue: runs the upstream image `n8nio/n8n` by digest with the `uuid` package mounted into it (ADR 0480 clauses 1, 3, 7 and 8); runs the container image `redis`; runs the container image `selenium/standalone-chrome`. + +- The `server` container no longer comes from a Dockerfile. Its program is the upstream image's, bit for bit the one the module built on before (digest `4846eb2f...`; the running container reports release 1.71.3). +- `uuid` 14.0.1, which Code nodes `require`, is fetched once by its pinned sha256 from the package registry and mounted read-only at `/usr/local/lib/node_modules/uuid` (where `npm install -g` put it in the image the module built). +- The server joins the media library's group as a supplementary group, `groups` from the config key `media-gid`; the key has no default, so a machine that does not say it refuses n8n. ## Files it does not know @@ -8,4 +12,5 @@ Each directory or archive this module declares says how it treats a file it did - `${dir:state}`: everything in it is the program's; the mesh moves nothing out (ADR 0422 §2). - `${dir:data}`: everything in it is the program's; the mesh moves nothing out (ADR 0422 §2). +- `${dir:state}/uuid`: everything in it is the program's; the mesh moves nothing out (ADR 0422 §2). - `${dir:cache}`: everything in it is the program's; the mesh moves nothing out (ADR 0422 §2). diff --git a/modules/n8n/module.json b/modules/n8n/module.json index 975247fa..5ee86819 100644 --- a/modules/n8n/module.json +++ b/modules/n8n/module.json @@ -30,6 +30,14 @@ "mode": "read-write" } ], + "config": { + "media-gid": { + "type": "number", + "description": "The number of the group that owns the media library on this machine, which the server joins as a supplementary group so a workflow can file into the library (novox/hq ADR 0480 clause 7). Set per machine; until the container takes it from the media library itself (issue 1618) nothing writes a default, so a machine that does not say it refuses n8n by name.", + "minimum": 1, + "trusted": true + } + }, "listens": [ { "name": "web", @@ -92,6 +100,13 @@ "mode": "0600", "template": "files/server.env" }, + { + "id": "uuid", + "type": "archive", + "path": "${dir:state}/uuid", + "artifact": "uuid", + "unknown": "program" + }, { "id": "net", "type": "network", @@ -119,7 +134,11 @@ "volumes": [ "${dir:data}:/home/node/.n8n", "${dir:state}/n8n-database.secret:/run/secrets/database:ro", + "${dir:state}/uuid:/usr/local/lib/node_modules/uuid:ro", "${access:media}:/media-library" + ], + "groups": [ + "${config:media-gid}" ] }, { @@ -151,17 +170,22 @@ } ], "build": { - "on": [ - { - "arg": "N8N_BASE", - "image": "n8nio/n8n@sha256:4846eb2f4b874ab04cde7fc1e249d2ddaec66e9aea64439beb2972cfea88e3c0" - } - ], "artifacts": [ { "name": "server", - "kind": "image", - "from": "Dockerfile" + "kind": "upstream", + "from": "n8nio/n8n@sha256:4846eb2f4b874ab04cde7fc1e249d2ddaec66e9aea64439beb2972cfea88e3c0" + }, + { + "name": "uuid", + "kind": "archive", + "url": "https://registry.npmjs.org/uuid/-/uuid-14.0.1.tgz", + "sha256": "e062c57ed120ea135478f305da04802bf69eab23fbc0ca2bae6989b6855390c4", + "version": "14.0.1", + "within": "package", + "names-on-purpose": { + "registry.npmjs.org": "the npm registry publishes the uuid package there; the build seat fetches it once, by its pinned sha256 (novox/hq ADR 0316 §2, ADR 0480 §8)" + } }, { "name": "files", From 0ec446ab755d1b68bfd8e4829faa34f71ccd47cc Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 21 Oct 2026 14:02:46 +0200 Subject: [PATCH 2/9] n8n's media-gid is a required config value: a machine that does not say it refuses n8n (issue 1639) --- modules/n8n/module.json | 1 + 1 file changed, 1 insertion(+) diff --git a/modules/n8n/module.json b/modules/n8n/module.json index 5ee86819..04bec8ff 100644 --- a/modules/n8n/module.json +++ b/modules/n8n/module.json @@ -35,6 +35,7 @@ "type": "number", "description": "The number of the group that owns the media library on this machine, which the server joins as a supplementary group so a workflow can file into the library (novox/hq ADR 0480 clause 7). Set per machine; until the container takes it from the media library itself (issue 1618) nothing writes a default, so a machine that does not say it refuses n8n by name.", "minimum": 1, + "required": true, "trusted": true } }, From b4e90e5fce5464727aa3e51a83a1847b0b16d816 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 22 Oct 2026 09:55:03 +0200 Subject: [PATCH 3/9] Run the bus from its pinned upstream image with the snapshot program and the entrypoint mounted, and write and rehearse the way there and back (issues 1640, 1641, 1642, ADR 0480 clauses 1-3, 5, 6, 11) --- checks/images/go.mod | 3 + checks/images/main.go | 80 +++++ checks/images/main_test.go | 51 +++ merge-check.sh | 2 + modules/nats/Dockerfile | 47 --- modules/nats/README.md | 61 +++- .../mesh-nats-snapshot}/archive.go | 0 .../mesh-nats-snapshot}/connect.go | 0 .../nats/cmd/mesh-nats-snapshot/image_test.go | 309 ++++++++++++++++++ .../mesh-nats-snapshot}/live_test.go | 2 +- .../mesh-nats-snapshot}/main.go | 6 +- .../mesh-nats-snapshot}/restore.go | 0 .../mesh-nats-snapshot}/snapshot.go | 0 .../mesh-nats-snapshot}/templates_test.go | 2 +- modules/nats/cmd/nats-tools/manifest_test.go | 110 +++++++ modules/nats/cmd/nats-tools/server_test.go | 37 ++- modules/nats/go.mod | 2 +- modules/nats/module.json | 57 +++- modules/nats/{ => program}/entrypoint.sh | 0 modules/nats/snapshot/go.mod | 15 - modules/nats/snapshot/go.sum | 12 - modules/nats/snapshot/image_test.go | 152 --------- 22 files changed, 682 insertions(+), 266 deletions(-) create mode 100644 checks/images/go.mod create mode 100644 checks/images/main.go create mode 100644 checks/images/main_test.go delete mode 100644 modules/nats/Dockerfile rename modules/nats/{snapshot => cmd/mesh-nats-snapshot}/archive.go (100%) rename modules/nats/{snapshot => cmd/mesh-nats-snapshot}/connect.go (100%) create mode 100644 modules/nats/cmd/mesh-nats-snapshot/image_test.go rename modules/nats/{snapshot => cmd/mesh-nats-snapshot}/live_test.go (99%) rename modules/nats/{snapshot => cmd/mesh-nats-snapshot}/main.go (95%) rename modules/nats/{snapshot => cmd/mesh-nats-snapshot}/restore.go (100%) rename modules/nats/{snapshot => cmd/mesh-nats-snapshot}/snapshot.go (100%) rename modules/nats/{snapshot => cmd/mesh-nats-snapshot}/templates_test.go (98%) create mode 100644 modules/nats/cmd/nats-tools/manifest_test.go rename modules/nats/{ => program}/entrypoint.sh (100%) delete mode 100644 modules/nats/snapshot/go.mod delete mode 100644 modules/nats/snapshot/go.sum delete mode 100644 modules/nats/snapshot/image_test.go diff --git a/checks/images/go.mod b/checks/images/go.mod new file mode 100644 index 00000000..1137f5f0 --- /dev/null +++ b/checks/images/go.mod @@ -0,0 +1,3 @@ +module git.novox.be/novox/mesh-catalog/checks/images + +go 1.22 diff --git a/checks/images/main.go b/checks/images/main.go new file mode 100644 index 00000000..ddd408a1 --- /dev/null +++ b/checks/images/main.go @@ -0,0 +1,80 @@ +// Command images refuses any manifest but build-agent's declaring an artifact of kind `image`, an image built from a +// Dockerfile (novox/hq ADR 0452 clause 1, ADR 0480 clause 11 as ADR 0487 clause 13 changed it). From R4d a module +// runs a pinned upstream image with its programs mounted in; build-agent is the one builder that still builds an +// image until R4g, when the catalogue refuses the kind outright (issue 1619) and this check goes with it. +// +// go run . +// +// Exits 1 on a finding. +package main + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "sort" +) + +// Exempt is the one module still allowed to declare the kind, until R4g. +var Exempt = map[string]bool{"build-agent": true} + +// Finding is one refusal: Module's artifact Artifact is an image built from a Dockerfile. +type Finding struct{ Module, Artifact string } + +// Check reads every manifest under root/modules and answers what it refuses. +func Check(root string) ([]Finding, error) { + manifests, err := filepath.Glob(filepath.Join(root, "modules", "*", "module.json")) + if err != nil { + return nil, err + } + sort.Strings(manifests) + var out []Finding + for _, path := range manifests { + raw, err := os.ReadFile(path) + if err != nil { + return nil, err + } + var m struct { + Module string `json:"module"` + Build struct { + Artifacts []struct { + Name string `json:"name"` + Kind string `json:"kind"` + } `json:"artifacts"` + } `json:"build"` + } + if err := json.Unmarshal(raw, &m); err != nil { + return nil, fmt.Errorf("%s: %w", path, err) + } + if Exempt[m.Module] { + continue + } + for _, a := range m.Build.Artifacts { + if a.Kind == "image" { + out = append(out, Finding{Module: m.Module, Artifact: a.Name}) + } + } + } + return out, nil +} + +func main() { + root := "." + if len(os.Args) > 1 { + root = os.Args[1] + } + found, err := Check(root) + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + for _, f := range found { + fmt.Printf("%s declares the artifact %s as an image built from a Dockerfile: run a pinned upstream image with the "+ + "module's programs mounted in (novox/hq ADR 0452 clause 1, ADR 0480 clause 11)\n", f.Module, f.Artifact) + } + if len(found) > 0 { + os.Exit(1) + } + fmt.Println("images: no module but build-agent builds an image from a Dockerfile") +} diff --git a/checks/images/main_test.go b/checks/images/main_test.go new file mode 100644 index 00000000..677c6009 --- /dev/null +++ b/checks/images/main_test.go @@ -0,0 +1,51 @@ +package main + +import ( + "os" + "path/filepath" + "reflect" + "testing" +) + +func catalogue(t *testing.T, manifests map[string]string) string { + t.Helper() + root := t.TempDir() + for name, body := range manifests { + dir := filepath.Join(root, "modules", name) + if err := os.MkdirAll(dir, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "module.json"), []byte(body), 0o644); err != nil { + t.Fatal(err) + } + } + return root +} + +// A fixture manifest with an image artifact is refused, the builder's is not, and an upstream image is no image. +func TestOnlyBuildAgentMayDeclareAnImageBuiltFromADockerfile(t *testing.T) { + root := catalogue(t, map[string]string{ + "old": `{"module":"old","build":{"artifacts":[{"name":"server","kind":"image","from":"Dockerfile"}]}}`, + "build-agent": `{"module":"build-agent","build":{"artifacts":[{"name":"agent","kind":"image","from":"Dockerfile"}]}}`, + "new": `{"module":"new","build":{"artifacts":[{"name":"server","kind":"upstream","from":"alpine@sha256:00"},{"name":"p","kind":"bundle"}]}}`, + "none": `{"module":"none"}`, + }) + got, err := Check(root) + if err != nil { + t.Fatal(err) + } + if want := []Finding{{Module: "old", Artifact: "server"}}; !reflect.DeepEqual(got, want) { + t.Errorf("found %v, want %v", got, want) + } +} + +// The catalogue this check lives in passes it. +func TestTheCatalogueDeclaresNoImage(t *testing.T) { + got, err := Check("../..") + if err != nil { + t.Fatal(err) + } + if len(got) > 0 { + t.Errorf("the catalogue declares %v", got) + } +} diff --git a/merge-check.sh b/merge-check.sh index d3aacbf6..d3b7d97c 100755 --- a/merge-check.sh +++ b/merge-check.sh @@ -55,6 +55,8 @@ set -eu (cd checks/words && go test -count=1 . && go run . ../..) # The toolchains (novox/hq ADR 0477 clause 4): the package go only in modules/go, nodejs and npm only in modules/nodejs. (cd checks/toolchains && go test -count=1 . && go run . ../..) +# The image (novox/hq ADR 0480 clause 11): no manifest but build-agent's declares an image built from a Dockerfile. +(cd checks/images && go test -count=1 . && go run . ../..) # The agent's home (novox/hq ADR 0266): no archive, and no file written into, under it. (cd checks/agenthome && go test -count=1 . && go run . ../..) # The mail-store seat (novox/hq ADR 0378 §3): only mesh-mail may claim it. diff --git a/modules/nats/Dockerfile b/modules/nats/Dockerfile deleted file mode 100644 index 9f57db4d..00000000 --- a/modules/nats/Dockerfile +++ /dev/null @@ -1,47 +0,0 @@ -# nats's server image: the upstream server, plus an entrypoint that reloads it in place when the -# mesh rewrites its configuration. See entrypoint.sh for why that belongs here and not in the host. -# -# **Pinned to the multi-architecture index digest, not a platform's.** `docker manifest inspect` -# reports a platform manifest per architecture and the index that lists them; pinning a platform's -# digest builds on this workstation and fails on any node of another architecture, with an error -# that names a manifest rather than the mistake. This is the index — `docker pull` reports the same -# one, and `RepoDigests` confirms it. -# -# **The release the digest is, said here because a digest does not say it:** -# -# upstream: nats 2.11.17-alpine -# -# Kept equal to the server version cmd/nats-tools tests against (its go.mod), and a test there fails -# when they differ: that test is what says the server delivers every message to a consumer with -# several filters. 2.10.29 did not — it moved such a consumer past a message now and then without -# handing it over, and the controller never heard of a merge (novox/hq issue 266). -# -# Unlike every other module's Dockerfile, this builds no TypeScript and uses no mesh base image: -# the module's code is the server, which upstream already built. There is no BUILD_BASE here on -# purpose — the server is not compiled; only the snapshot program below is. The upstream image is -# declared in the manifest under build.on and arrives as NATS_BASE, like every other base the mesh -# copies into its own store before a build (novox/hq ADR 0097); the digest above is the index one -# for the reason given. So does GO_BASE, the toolchain the snapshot program is built with. -# -# **One thing is compiled: the bus's snapshot program** (novox/hq ADR 0235). The machine's backup -# holder runs the module's declared dump — `docker exec` into this container — and the program asks -# the server for each stream through the snapshot API, writing a tar on stdout. It is here, beside -# the server, for two reasons: the dump runs where the server is without mounting anything into it, -# and a restore needs nats-server itself — the very binary this image already carries — to fill a new -# store. Its own Go module (snapshot/go.mod), so this build fetches only public modules. -ARG NATS_BASE -ARG GO_BASE -FROM ${GO_BASE} AS snapshot -WORKDIR /src -COPY snapshot/go.mod snapshot/go.sum ./ -RUN go mod download -COPY snapshot/*.go ./ -RUN CGO_ENABLED=0 go build -trimpath -ldflags '-s -w' -o /mesh-nats-snapshot . - -FROM ${NATS_BASE} - -COPY entrypoint.sh /usr/local/bin/mesh-nats-entrypoint -RUN chmod 0755 /usr/local/bin/mesh-nats-entrypoint -COPY --from=snapshot /mesh-nats-snapshot /usr/local/bin/mesh-nats-snapshot - -ENTRYPOINT ["/usr/local/bin/mesh-nats-entrypoint"] diff --git a/modules/nats/README.md b/modules/nats/README.md index c9d3c046..465ff5c0 100644 --- a/modules/nats/README.md +++ b/modules/nats/README.md @@ -2,7 +2,12 @@ The bus server (novox/hq design 25), the configuration the module owns (`nats.conf`: ports, TLS, JetStream's store) and the user list the controller composes (`accounts.conf`), reloaded in place by -the entrypoint. Its tools (`cmd/nats-tools`) are served by the node's runtime and only read. +the entrypoint. The server is the upstream nats image, pinned by digest in the manifest's `server` +artifact (`nats:-alpine@sha256:`: the digest names the image, the tag says the release, +and `cmd/nats-tools`' release test keeps the tag equal to the server its tests run). The module's own two +programs are mounted into it read-only (novox/hq ADR 0480): the snapshot program (a Go bundle of +`cmd/mesh-nats-snapshot`) at `/opt/mesh`, and the entrypoint (`program/entrypoint.sh`, an archive) at +`/opt/mesh-entrypoint`, run through `sh`. Both archive resources are in the container's `restart-on`. Its tools (`cmd/nats-tools`) are served by the node's runtime and only read. ## Its data, and how it is backed up @@ -14,9 +19,9 @@ events, every module's state. while the server writes them may not restore. The `jetstream` data item is protected by a dump instead (novox/hq ADR 0235, to-be 43): each night the machine's backup holder runs - docker exec -i mesh-broker-nats mesh-nats-snapshot snapshot < > /bus.tar + docker exec -i mesh-broker-nats /opt/mesh/mesh-nats-snapshot snapshot < > /bus.tar -and backs up the `snapshots` directory. `mesh-nats-snapshot` (`snapshot/`, built into this image) +and backs up the `snapshots` directory. `mesh-nats-snapshot` (`cmd/mesh-nats-snapshot`, mounted into the server's container) asks the server for each stream through JetStream's snapshot API, one at a time, acknowledging each chunk so the server's flow control keeps moving; the server goes on taking writes and no stream is paused or reconfigured. It writes one tar on stdout: @@ -47,14 +52,15 @@ live one**, which a person swaps in. 1. **Take the night back, beside the live data.** `node-backup.restore` for `nats` on the bus's machine (optionally naming a restore point) restores the snapshots directory as `.restored-/`, holding that night's `bus.tar`. -2. **Check it.** `docker exec -i mesh-broker-nats mesh-nats-snapshot verify < /bus.tar` — +2. **Check it.** `docker exec -i mesh-broker-nats /opt/mesh/mesh-nats-snapshot verify < /bus.tar` — every file against the manifest's checksum, every archive read to its end. -3. **Build the new store**, with the bus's own image (the server that fills it is the same release - the bus runs): +3. **Build the new store**, with the bus's own image and the mounted program (the server that fills + it is the same release the bus runs): image=$(docker inspect -f '{{.Config.Image}}' mesh-broker-nats) docker run --rm -v :/in:ro -v :/out \ - --entrypoint mesh-nats-snapshot "$image" restore --into /out --from /in/bus.tar + -v /var/lib/nats-module/snapshot:/opt/mesh:ro \ + --entrypoint /opt/mesh/mesh-nats-snapshot "$image" restore --into /out --from /in/bus.tar It starts a server of its own on the container's loopback with the bus's one account (`MESH`), restores every stream, holds each to the manifest, stops it, and leaves @@ -75,16 +81,45 @@ A single stream can be restored into any server that does not hold it with `mesh-nats-snapshot restore --server --credential `; the archive is also what the nats CLI reads (`nats stream restore ` on an unpacked `streams//`). +## Upgrading the bus, and going back + +The bus is replaced only as a planned step a person starts (novox/hq ADR 0236, ADR 0480 clause 6): the +merge of a change to this module builds the bus and sends nothing. The steps, in this order: + +1. **Nothing else runs.** No delivery plan is open (`mesh-controller.plans`). +2. **Snapshot with the dump the bus runs now**, through the machine's backup holder + (`/node-backup.now` for `nats` on the bus's machine), and verify the `bus.tar` it wrote + (`docker exec -i mesh-broker-nats /opt/mesh/mesh-nats-snapshot verify`, or the old path + `mesh-nats-snapshot verify` on the declaration before ADR 0480). Note where it is: that is `snapshot-taken`. +3. **Ask the operator for the moment** on the proven channel, naming what it raises: bus maintenance is + open while the step runs; every bus connection is dropped for seconds and reconnects; conditions of + other modules may flash and clear. Go on only on their yes. +4. **Start it:** `mesh-controller.bus` with `upgrade` true, `why`, `reversible` true and `snapshot-taken`. + `reversible` is true because the server binary (the same digest), the data directory and the + configuration do not change; `TestTheStoreSurvivesTheNewDeclarationTheOldAndTheNewAgain` shows it: + the new declaration, then the old one, then the new again run on one store, each reading what the + one before wrote. +5. **After:** `nats_server`, `nats_streams` (every stream and bucket as before) and the controller's + `doctor`; the step ends done when the controller has checked every stream, every consumer and a + round trip (ADR 0236 section 40). Then wait for the next nightly dump and verify it: the upgrade is + finished only once that dump verified. +6. **Back:** a bus upgrade to the previous build puts the old declaration back on the same store; if the + store itself is hurt, restore the snapshot as described above. + ## Tests -- `snapshot/`: `MESH_TEST_DOCKER=1 go test -race ./...` — a throwaway `nats:2.11` filled like the +- `cmd/mesh-nats-snapshot`: `MESH_TEST_DOCKER=1 go test -race ./...` — a throwaway `nats:2.11` filled like the mesh's bus (events with a gap deleted, a work queue part worked, buckets with history, deletes and a purge, an object, durable consumers, a memory stream) and written to throughout the snapshot; the snapshot restored into a fresh server and into a new store served by a third, each compared message - by message; a damaged archive refused; and the snapshot user refused every write. - `MESH_TEST_NATS_IMAGE=` also runs the manifest's declared dump, exactly, against the - module's image and configuration. -- `cmd/nats-tools`: `go test ./...`. + by message; a damaged archive refused; and the snapshot user refused every write. The same run starts + the manifest's pinned upstream image with the program and the entrypoint mounted as the manifest + declares them: the declared dump, exactly, writes a `bus.tar` that verifies; a change of the user list + reloads the server and a connection open meanwhile survives; and the new declaration, the old one and + the new again run on one store. +- `cmd/nats-tools`: `go test ./...` — the tools, the release test (the manifest's tag equals the server + tested) and the manifest's shape (programs mounted read-only, named in `restart-on`, no image built from + a Dockerfile). ## Files it does not know @@ -94,3 +129,5 @@ Each directory or archive this module declares says how it treats a file it did - `/var/lib/nats-module/conf`: everything in it is the program's; the mesh moves nothing out (ADR 0422 §2). - `${dir:snapshots}`: everything in it is the program's; the mesh moves nothing out (ADR 0422 §2). - `${dir:mesh-state}`: everything in it is the program's; the mesh moves nothing out (ADR 0422 §2). +- `/var/lib/nats-module/snapshot`: the unpacked snapshot program; everything in it is the program's; the mesh moves nothing out (ADR 0422 §2). +- `/var/lib/nats-module/entrypoint`: the unpacked entrypoint; everything in it is the program's; the mesh moves nothing out (ADR 0422 §2). diff --git a/modules/nats/snapshot/archive.go b/modules/nats/cmd/mesh-nats-snapshot/archive.go similarity index 100% rename from modules/nats/snapshot/archive.go rename to modules/nats/cmd/mesh-nats-snapshot/archive.go diff --git a/modules/nats/snapshot/connect.go b/modules/nats/cmd/mesh-nats-snapshot/connect.go similarity index 100% rename from modules/nats/snapshot/connect.go rename to modules/nats/cmd/mesh-nats-snapshot/connect.go diff --git a/modules/nats/cmd/mesh-nats-snapshot/image_test.go b/modules/nats/cmd/mesh-nats-snapshot/image_test.go new file mode 100644 index 00000000..68ba997d --- /dev/null +++ b/modules/nats/cmd/mesh-nats-snapshot/image_test.go @@ -0,0 +1,309 @@ +package main + +// The declared dump and the mounted entrypoint, exactly as the manifest declares them, against the upstream image +// the manifest pins: the mesh's snapshot program and entrypoint mounted at /opt (novox/hq ADR 0480 clauses 1, 2, 5). +// +// MESH_TEST_DOCKER=1 go test -run 'TestTheDeclaredDump|TestTheEntrypoint' . +// +// The image is pulled by the manifest's reference, so the server these run is the one the bus runs. + +import ( + "encoding/json" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/nats-io/nats.go" +) + +// manifestOf is the nats module's own definition, beside this program. +func manifestOf(t *testing.T) map[string]any { + t.Helper() + raw, err := os.ReadFile("../../module.json") + if err != nil { + t.Fatal(err) + } + var m map[string]any + if err := json.Unmarshal(raw, &m); err != nil { + t.Fatal(err) + } + return m +} + +func declaredDump(t *testing.T, m map[string]any) string { + t.Helper() + for _, item := range m["data"].(map[string]any)["own"].([]any) { + it := item.(map[string]any) + if it["id"] != "jetstream" { + continue + } + backup, _ := it["backup"].(map[string]any) + if backup == nil || backup["into"] != "snapshots" { + t.Fatalf("the bus's streams are not protected by a dump into its snapshots: %v", it["backup"]) + } + return backup["dump"].(string) + } + t.Fatal("the nats module declares no jetstream data item") + return "" +} + +// serverConf is the module's own server settings, from the template its resource names. +func serverConf(t *testing.T, m map[string]any) string { + t.Helper() + for _, r := range m["resources"].([]any) { + res := r.(map[string]any) + if res["id"] == "server-conf" { + return templateText(t, res) + } + } + t.Fatal("the nats module declares no server configuration") + return "" +} + +// busImage is the upstream image the manifest pins, as a reference docker pulls (name:tag@digest). +func busImage(t *testing.T, m map[string]any) string { + t.Helper() + for _, a := range m["build"].(map[string]any)["artifacts"].([]any) { + art := a.(map[string]any) + if art["name"] == "server" { + if art["kind"] != "upstream" { + t.Fatalf("the server artifact is a %v, not an upstream image", art["kind"]) + } + return art["from"].(string) + } + } + t.Fatal("the nats module declares no server artifact") + return "" +} + +// programs builds the snapshot program for alpine (static, as the mesh's build does) and lays out the two +// directories the manifest mounts: the program at /opt/mesh and the entrypoint at /opt/mesh-entrypoint. +func programs(t *testing.T) (program, entry string) { + t.Helper() + program, entry = t.TempDir(), t.TempDir() + build := exec.Command("go", "build", "-trimpath", "-o", filepath.Join(program, "mesh-nats-snapshot"), ".") + build.Env = append(os.Environ(), "CGO_ENABLED=0", "GOOS=linux") + if out, err := build.CombinedOutput(); err != nil { + t.Fatalf("building the program: %v\n%s", err, out) + } + script, err := os.ReadFile(filepath.Join(moduleDir, "program", "entrypoint.sh")) + if err != nil { + t.Fatal(err) + } + // Mode 0644: the manifest runs the script through sh, so no execute bit is relied on. + if err := os.WriteFile(filepath.Join(entry, "entrypoint.sh"), script, 0o644); err != nil { + t.Fatal(err) + } + return program, entry +} + +// runBus starts the upstream image the way the manifest does: its volumes, its args, the module's configuration +// and the given user list. The poll is a second so a test need not wait out the production five. +func runBus(t *testing.T, m map[string]any, name, accounts string) (port, pin, conf string) { + t.Helper() + if out, _ := exec.Command("docker", "ps", "-a", "-q", "--filter", "name=^"+name+"$").Output(); len(strings.TrimSpace(string(out))) > 0 { + t.Skipf("a container called %s already runs here; this test will not touch it", name) + } + conf, tlsDir, data := t.TempDir(), t.TempDir(), t.TempDir() + pin = selfSigned(t, tlsDir) + _ = os.WriteFile(filepath.Join(conf, "nats.conf"), []byte(serverConf(t, m)), 0o644) + _ = os.WriteFile(filepath.Join(conf, "accounts.conf"), []byte(accounts), 0o644) + program, entry := programs(t) + docker(t, "run", "-d", "--rm", "--name", name, "--user", owner(), "-p", "127.0.0.1::4222", + "-e", "MESH_NATS_CONF_POLL_SECONDS=1", + "-v", data+":/data", "-v", conf+":/etc/nats:ro", "-v", tlsDir+":/tls:ro", + "-v", program+":/opt/mesh:ro", "-v", entry+":/opt/mesh-entrypoint:ro", + busImage(t, m), "sh", "/opt/mesh-entrypoint/entrypoint.sh") + t.Cleanup(func() { _ = exec.Command("docker", "rm", "-f", name).Run() }) + _, port, _ = strings.Cut(docker(t, "port", name, "4222/tcp"), ":") + return port, pin, conf +} + +// The dump is the one the manifest declares, run by `sh -c` as the holder runs it, with the module's +// directories filled in; the server is the module's image with the module's own configuration and a +// user list holding the bus's own module with the controller's grants for it. +func TestTheDeclaredDumpRunsInTheBussImage(t *testing.T) { + needDocker(t) + const name = "mesh-broker-nats" // the container the dump names + m := manifestOf(t) + dump := declaredDump(t, m) + + state, snapshots := t.TempDir(), t.TempDir() + accounts := liveBusConf() + accounts = accounts[strings.Index(accounts, "accounts {"):] + port, pin, _ := runBus(t, m, name, accounts) + + admin := dial(t, "tls://127.0.0.1:"+port, Credential{Fingerprint: pin, User: "admin", Password: "admin"}, nil) + js, _ := admin.JetStream() + kv, err := js.CreateKeyValue(&nats.KeyValueConfig{Bucket: "mesh-controller_hand-acts"}) + if err != nil { + t.Fatal(err) + } + for _, k := range []string{"a", "b", "c"} { + if _, err := kv.Put(k, []byte("act "+k)); err != nil { + t.Fatal(err) + } + } + + run := func(cred Credential) (string, error) { + raw, _ := json.Marshal(cred) + if err := os.WriteFile(filepath.Join(state, "broker"), raw, 0o600); err != nil { + t.Fatal(err) + } + command := strings.NewReplacer("${dir:mesh-state}", state, "${dir:snapshots}", snapshots).Replace(dump) + out, err := exec.Command("sh", "-c", command).CombinedOutput() + return string(out), err + } + + began := time.Now() + out, err := run(Credential{URL: "nats://bus.example:4222", Fingerprint: pin, User: snapshotUser, Password: "snap"}) + if err != nil { + t.Fatalf("the declared dump failed: %v\n%s", err, out) + } + t.Logf("the declared dump took %s:\n%s", time.Since(began).Round(time.Millisecond), out) + tarPath := filepath.Join(snapshots, "bus.tar") + f, err := os.Open(tarPath) + if err != nil { + t.Fatalf("the dump left no bus.tar: %v", err) + } + a, err := ReadArchive(f, "") + f.Close() + if err != nil { + t.Fatal(err) + } + defer a.Close() + if err := a.Verify(); err != nil { + t.Fatal(err) + } + if len(a.Manifest.Streams) != 1 || a.Manifest.Streams[0].Name != "KV_mesh-controller_hand-acts" || a.Manifest.Streams[0].Messages != 3 { + t.Fatalf("the snapshot holds %+v", a.Manifest.Streams) + } + before, _ := os.ReadFile(tarPath) + + // A night that cannot snapshot fails, says why, and leaves last night's snapshot as it was. + out, err = run(Credential{Fingerprint: pin, User: snapshotUser, Password: "wrong"}) + if err == nil { + t.Fatalf("a dump the bus refused succeeded:\n%s", out) + } + if !strings.Contains(out, "Authorization Violation") && !strings.Contains(strings.ToLower(out), "authorization") { + t.Errorf("the failure does not say the bus refused the credential: %s", out) + } + after, _ := os.ReadFile(tarPath) + if string(after) != string(before) { + t.Error("a failed night replaced the last good snapshot") + } + if _, err := os.Stat(tarPath + ".partial"); !os.IsNotExist(err) { + t.Error("a failed night left its partial file behind") + } +} + +// **A change of the user list reloads the server and keeps its connections** (ADR 0480 clause 5, ADR 0236 §37). +// The mounted entrypoint watches accounts.conf and signals the server; a client connected meanwhile is not +// dropped, and a user added by the change can then connect. +func TestTheEntrypointReloadsTheServerWithoutDroppingAConnection(t *testing.T) { + needDocker(t) + m := manifestOf(t) + accounts := plainConf() + accounts = accounts[strings.Index(accounts, "accounts {"):] + port, pin, conf := runBus(t, m, fmt.Sprintf("mesh-reload-test-%d", os.Getpid()), accounts) + + admin := dial(t, "tls://127.0.0.1:"+port, Credential{Fingerprint: pin, User: "admin", Password: "admin"}, nil) + if err := admin.Flush(); err != nil { + t.Fatal(err) + } + + changed := strings.Replace(accounts, `{ user: "admin", password: "admin" }`, + `{ user: "admin", password: "admin" }, { user: "late", password: "late" }`, 1) + if changed == accounts { + t.Fatal("the test's user list did not change") + } + // Written as the mesh writes it: replaced, not edited in place, so the watcher sees whole content. + tmp := filepath.Join(conf, "accounts.conf.new") + if err := os.WriteFile(tmp, []byte(changed), 0o644); err != nil { + t.Fatal(err) + } + if err := os.Rename(tmp, filepath.Join(conf, "accounts.conf")); err != nil { + t.Fatal(err) + } + + // dial retries until the server takes the user: the reload happens within the poll interval. + late := dial(t, "tls://127.0.0.1:"+port, Credential{Fingerprint: pin, User: "late", Password: "late"}, nil) + _ = late + if !admin.IsConnected() { + t.Fatal("the connection open before the reload was dropped by it") + } + if err := admin.Flush(); err != nil { + t.Fatalf("the connection open before the reload no longer works: %v", err) + } + if n := admin.Stats().Reconnects; n != 0 { + t.Fatalf("the connection open before the reload reconnected %d times", n) + } +} + +// **The bus can go back** (ADR 0480 clause 6): the new declaration and then the old one run on the same store, both +// well, which is what makes `reversible` true for the bus upgrade. The server binary (the upstream image's digest), +// the data directory and the configuration are the same in both; only where the entrypoint and the program come +// from differs. The old declaration is run as the image it replaced ran it: the same script as the image's own +// entrypoint, the program on the image's path. Then the new one again, so a way there, back and there again is +// shown, each time with the stream's messages as the one before left them. +func TestTheStoreSurvivesTheNewDeclarationTheOldAndTheNewAgain(t *testing.T) { + needDocker(t) + m := manifestOf(t) + accounts := liveBusConf() + accounts = accounts[strings.Index(accounts, "accounts {"):] + conf, tlsDir, data := t.TempDir(), t.TempDir(), t.TempDir() + pin := selfSigned(t, tlsDir) + _ = os.WriteFile(filepath.Join(conf, "nats.conf"), []byte(serverConf(t, m)), 0o644) + _ = os.WriteFile(filepath.Join(conf, "accounts.conf"), []byte(accounts), 0o644) + program, entry := programs(t) + script := filepath.Join(entry, "entrypoint.sh") + if err := os.Chmod(script, 0o755); err != nil { + t.Fatal(err) + } + common := []string{"-d", "--rm", "--user", owner(), "-p", "127.0.0.1::4222", "-e", "MESH_NATS_CONF_POLL_SECONDS=1", + "-v", data + ":/data", "-v", conf + ":/etc/nats:ro", "-v", tlsDir + ":/tls:ro"} + declarations := map[string][]string{ + "new": {"-v", program + ":/opt/mesh:ro", "-v", entry + ":/opt/mesh-entrypoint:ro", busImage(t, m), "sh", "/opt/mesh-entrypoint/entrypoint.sh"}, + "old": {"-v", script + ":/usr/local/bin/mesh-nats-entrypoint:ro", + "-v", filepath.Join(program, "mesh-nats-snapshot") + ":/usr/local/bin/mesh-nats-snapshot:ro", + "--entrypoint", "/usr/local/bin/mesh-nats-entrypoint", busImage(t, m)}, + } + var kv nats.KeyValue + puts := 0 + step := func(which string) { + name := fmt.Sprintf("mesh-rehearsal-%s-%d", which, os.Getpid()) + docker(t, append(append([]string{"run", "--name", name}, common...), declarations[which]...)...) + t.Cleanup(func() { _ = exec.Command("docker", "rm", "-f", name).Run() }) + _, port, _ := strings.Cut(docker(t, "port", name, "4222/tcp"), ":") + c := dial(t, "tls://127.0.0.1:"+port, Credential{Fingerprint: pin, User: "admin", Password: "admin"}, nil) + js, _ := c.JetStream() + var err error + if puts == 0 { + kv, err = js.CreateKeyValue(&nats.KeyValueConfig{Bucket: "mesh-rehearsal"}) + } else { + kv, err = js.KeyValue("mesh-rehearsal") + } + if err != nil { + t.Fatalf("%s declaration: the store's bucket: %v", which, err) + } + for i := 1; i <= puts; i++ { + e, err := kv.Get(fmt.Sprintf("k%d", i)) + if err != nil || string(e.Value()) != fmt.Sprintf("v%d", i) { + t.Fatalf("%s declaration: key k%d reads %v, %v", which, i, e, err) + } + } + puts++ + if _, err := kv.Put(fmt.Sprintf("k%d", puts), []byte(fmt.Sprintf("v%d", puts))); err != nil { + t.Fatal(err) + } + c.Close() + docker(t, "stop", "-t", "10", name) + } + step("new") + step("old") + step("new") +} diff --git a/modules/nats/snapshot/live_test.go b/modules/nats/cmd/mesh-nats-snapshot/live_test.go similarity index 99% rename from modules/nats/snapshot/live_test.go rename to modules/nats/cmd/mesh-nats-snapshot/live_test.go index 520cd82a..d6c6a4df 100644 --- a/modules/nats/snapshot/live_test.go +++ b/modules/nats/cmd/mesh-nats-snapshot/live_test.go @@ -35,7 +35,7 @@ import ( "github.com/nats-io/nats.go" ) -// image is the bus's release (the Dockerfile's `# upstream:` line names the same one). +// image is the bus's release (the manifest's server artifact names the same release in its tag). const image = "nats:2.11-alpine" // snapshotUser is the bus's own module on a machine, as the controller names it. diff --git a/modules/nats/snapshot/main.go b/modules/nats/cmd/mesh-nats-snapshot/main.go similarity index 95% rename from modules/nats/snapshot/main.go rename to modules/nats/cmd/mesh-nats-snapshot/main.go index 81c670b2..bbfffb7d 100644 --- a/modules/nats/snapshot/main.go +++ b/modules/nats/cmd/mesh-nats-snapshot/main.go @@ -9,9 +9,9 @@ // transfer with flow control that the server answers from a consistent view of the stream while // it goes on taking messages. This program asks for it, one stream at a time, and nothing else. // -// It lives in the bus's image because that is where it runs: the machine's backup holder runs the -// nats module's declared dump — `docker exec` into the server's container — and the archive comes -// out on stdout, so nothing is mounted into the server for it and the credential arrives on stdin. +// It is mounted into the bus's container (at /opt/mesh) because that is where it runs: the machine's +// backup holder runs the nats module's declared dump — `docker exec` into the server's container — and +// the archive comes out on stdout, so the credential arrives on stdin. // // mesh-nats-snapshot snapshot [--server URL] [--credential FILE|-] > bus.tar // mesh-nats-snapshot verify [--from FILE] diff --git a/modules/nats/snapshot/restore.go b/modules/nats/cmd/mesh-nats-snapshot/restore.go similarity index 100% rename from modules/nats/snapshot/restore.go rename to modules/nats/cmd/mesh-nats-snapshot/restore.go diff --git a/modules/nats/snapshot/snapshot.go b/modules/nats/cmd/mesh-nats-snapshot/snapshot.go similarity index 100% rename from modules/nats/snapshot/snapshot.go rename to modules/nats/cmd/mesh-nats-snapshot/snapshot.go diff --git a/modules/nats/snapshot/templates_test.go b/modules/nats/cmd/mesh-nats-snapshot/templates_test.go similarity index 98% rename from modules/nats/snapshot/templates_test.go rename to modules/nats/cmd/mesh-nats-snapshot/templates_test.go index fbd1490a..7d0718b5 100644 --- a/modules/nats/snapshot/templates_test.go +++ b/modules/nats/cmd/mesh-nats-snapshot/templates_test.go @@ -8,7 +8,7 @@ import ( ) // The module's directory, where its manifest and its files/ are. -const moduleDir = ".." +const moduleDir = "../.." // templateText is the text a file resource places on the machine: its template's, in the module's files/ // (novox/hq ADR 0349 §2). A resource that names no template, or a template that is not there, fails the test. diff --git a/modules/nats/cmd/nats-tools/manifest_test.go b/modules/nats/cmd/nats-tools/manifest_test.go new file mode 100644 index 00000000..5420fca2 --- /dev/null +++ b/modules/nats/cmd/nats-tools/manifest_test.go @@ -0,0 +1,110 @@ +package main + +import ( + "encoding/json" + "os" + "strings" + "testing" +) + +// The shape ADR 0480 clauses 1, 2, 3 and 5 give the bus's manifest: the upstream image by digest, the mesh's two +// programs mounted read-only under /opt, unpacked by archive resources the container names under restart-on +// (the node-engine does not look inside a mounted directory), and the container the same one it was. +func TestTheBussManifestMountsItsProgramsIntoTheUpstreamImage(t *testing.T) { + raw, err := os.ReadFile("../../module.json") + if err != nil { + t.Fatal(err) + } + var m struct { + Data struct { + Own []struct { + ID string `json:"id"` + Backup struct { + Dump string `json:"dump"` + } `json:"backup"` + } `json:"own"` + } `json:"data"` + Resources []map[string]any `json:"resources"` + Build struct { + On any `json:"on"` + Artifacts []struct { + Name, Kind, From, Language, System, Binary string + } `json:"artifacts"` + } `json:"build"` + } + if err := json.Unmarshal(raw, &m); err != nil { + t.Fatal(err) + } + if m.Build.On != nil { + t.Error("build.on is declared: nothing is built on a base image any more") + } + artifacts := map[string]struct{ Name, Kind, From, Language, System, Binary string }{} + for _, a := range m.Build.Artifacts { + artifacts[a.Name] = a + if a.Kind == "image" { + t.Errorf("artifact %s is an image built from a Dockerfile (ADR 0452)", a.Name) + } + } + if a := artifacts["server"]; a.Kind != "upstream" || !strings.Contains(a.From, "@sha256:") { + t.Errorf("server is %+v: an upstream image pinned by digest", a) + } + if a := artifacts["snapshot"]; a.Kind != "bundle" || a.Language != "go" || a.System != "alpine" || + a.From != "cmd/mesh-nats-snapshot" || a.Binary != "mesh-nats-snapshot" { + t.Errorf("snapshot is %+v: a Go bundle of the module's own package, stamped for alpine", a) + } + if a := artifacts["entrypoint"]; a.Kind != "archive" || a.From != "program" { + t.Errorf("entrypoint is %+v: an archive of program/", a) + } + if _, err := os.Stat("../../program/entrypoint.sh"); err != nil { + t.Error(err) + } + if _, err := os.Stat("../../Dockerfile"); err == nil { + t.Error("the Dockerfile is still there") + } + + byID := map[string]map[string]any{} + for _, r := range m.Resources { + byID[r["id"].(string)] = r + } + server := byID["server"] + if server["name"] != "mesh-broker-nats" || server["artifact"] != "server" { + t.Errorf("the container is %v on artifact %v: it keeps its name, which the dump and the jails use", server["name"], server["artifact"]) + } + volumes, _ := server["volumes"].([]any) + restart, _ := server["restart-on"].([]any) + for _, p := range []struct{ resource, artifact, mount string }{ + {"program-snapshot", "snapshot", ":/opt/mesh:ro"}, + {"program-entrypoint", "entrypoint", ":/opt/mesh-entrypoint:ro"}, + } { + r := byID[p.resource] + if r == nil || r["type"] != "archive" || r["artifact"] != p.artifact { + t.Errorf("%s is %v: an archive resource unpacking %s", p.resource, r, p.artifact) + continue + } + mounted := false + for _, v := range volumes { + if v == r["path"].(string)+p.mount { + mounted = true + } + } + if !mounted { + t.Errorf("%s is not mounted read-only as %s", p.resource, p.mount) + } + named := false + for _, v := range restart { + named = named || v == p.resource + } + if !named { + t.Errorf("%s is not in the container's restart-on: a new program would leave the old one running", p.resource) + } + } + args, _ := server["args"].([]any) + if len(args) != 2 || args[0] != "sh" || args[1] != "/opt/mesh-entrypoint/entrypoint.sh" { + t.Errorf("args are %v: the entrypoint run through sh, so no execute bit is relied on", args) + } + for _, d := range m.Data.Own { + if d.ID == "jetstream" && !strings.Contains(d.Backup.Dump, "docker exec -i mesh-broker-nats /opt/mesh/mesh-nats-snapshot snapshot") { + t.Errorf("the dump is %q: it calls the mounted program", d.Backup.Dump) + } + } +} diff --git a/modules/nats/cmd/nats-tools/server_test.go b/modules/nats/cmd/nats-tools/server_test.go index 57ab02ee..acff8f8f 100644 --- a/modules/nats/cmd/nats-tools/server_test.go +++ b/modules/nats/cmd/nats-tools/server_test.go @@ -1,6 +1,7 @@ package main import ( + "encoding/json" "fmt" "io" "math/rand" @@ -15,21 +16,43 @@ import ( "github.com/nats-io/nats.go" ) -// The server these tests run is the server the image runs. The image is pinned by a digest, which -// names no release, so the Dockerfile says which release it is, and this keeps the two equal: a -// test of the server below is only worth something about the server the mesh actually runs. +// The server these tests run is the server the mesh runs. The manifest's `server` artifact is the upstream +// image, written `name:tag@sha256:` (novox/hq ADR 0480 clause 3): the digest names the image and the +// tag says the release it is. This keeps the tag and the server compiled in here equal: a test of the server +// below is only worth something about the server the mesh actually runs. func TestTheImageIsTheServerTestedHere(t *testing.T) { - raw, err := os.ReadFile("../../Dockerfile") + raw, err := os.ReadFile("../../module.json") if err != nil { t.Fatal(err) } - said := regexp.MustCompile(`(?m)^# upstream: nats (\d+\.\d+\.\d+)-alpine$`).FindSubmatch(raw) + var m struct { + Build struct { + Artifacts []struct { + Name string `json:"name"` + Kind string `json:"kind"` + From string `json:"from"` + } `json:"artifacts"` + } `json:"build"` + } + if err := json.Unmarshal(raw, &m); err != nil { + t.Fatal(err) + } + from := "" + for _, a := range m.Build.Artifacts { + if a.Name == "server" { + if a.Kind != "upstream" { + t.Fatalf("the server artifact is a %q, not an upstream image", a.Kind) + } + from = a.From + } + } + said := regexp.MustCompile(`^nats:(\d+\.\d+\.\d+)-alpine@sha256:[0-9a-f]{64}$`).FindStringSubmatch(from) if said == nil { - t.Fatal("the Dockerfile does not say which nats release its digest is (`# upstream: nats X.Y.Z-alpine`)") + t.Fatalf("the server artifact is %q: it must read nats:X.Y.Z-alpine@sha256:, the tag saying the release", from) } // The server's own word for its release, compiled into it: a test binary's build information may list // no dependencies (the build seat's did not, so this read "these tests run no nats-server"). - if server.VERSION != string(said[1]) { + if server.VERSION != said[1] { t.Fatalf("the image runs nats %s and these tests run %s: move them together", said[1], server.VERSION) } } diff --git a/modules/nats/go.mod b/modules/nats/go.mod index ac9888b8..9f7e38c0 100644 --- a/modules/nats/go.mod +++ b/modules/nats/go.mod @@ -4,6 +4,7 @@ go 1.25.0 require ( git.novox.be/novox/mesh-sdk/go v0.1.7 + github.com/klauspost/compress v1.18.5 github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.51.0 ) @@ -11,7 +12,6 @@ require ( require ( github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op // indirect github.com/google/go-tpm v0.9.8 // indirect - github.com/klauspost/compress v1.18.5 // indirect github.com/minio/highwayhash v1.0.4 // indirect github.com/nats-io/jwt/v2 v2.8.1 // indirect github.com/nats-io/nkeys v0.4.15 // indirect diff --git a/modules/nats/module.json b/modules/nats/module.json index 3ad66bc3..b3bf5771 100644 --- a/modules/nats/module.json +++ b/modules/nats/module.json @@ -57,7 +57,7 @@ "class": "valuable", "active": "1d", "backup": { - "dump": "docker exec -i mesh-broker-nats mesh-nats-snapshot snapshot < ${dir:mesh-state}/broker > ${dir:snapshots}/bus.tar.partial && mv ${dir:snapshots}/bus.tar.partial ${dir:snapshots}/bus.tar || { rm -f ${dir:snapshots}/bus.tar.partial; exit 1; }", + "dump": "docker exec -i mesh-broker-nats /opt/mesh/mesh-nats-snapshot snapshot < ${dir:mesh-state}/broker > ${dir:snapshots}/bus.tar.partial && mv ${dir:snapshots}/bus.tar.partial ${dir:snapshots}/bus.tar || { rm -f ${dir:snapshots}/bus.tar.partial; exit 1; }", "into": "snapshots" }, "why": "the bus's streams and key-value buckets: the hand-act log, conditions, every module's state; written all the time, so copied by the server's own snapshot of each stream (novox/hq ADR 0235), never as live files" @@ -105,6 +105,20 @@ "template": "files/nats.conf", "mode": "0644" }, + { + "id": "program-snapshot", + "type": "archive", + "path": "/var/lib/nats-module/snapshot", + "artifact": "snapshot", + "unknown": "program" + }, + { + "id": "program-entrypoint", + "type": "archive", + "path": "/var/lib/nats-module/entrypoint", + "artifact": "entrypoint", + "unknown": "program" + }, { "id": "server", "type": "container", @@ -116,9 +130,19 @@ "volumes": [ "/var/lib/mesh-broker-nats:/data", "/var/lib/nats-module/conf:/etc/nats:ro", - "${access:tls}:/tls:ro" + "${access:tls}:/tls:ro", + "/var/lib/nats-module/snapshot:/opt/mesh:ro", + "/var/lib/nats-module/entrypoint:/opt/mesh-entrypoint:ro" ], - "artifact": "server" + "args": [ + "sh", + "/opt/mesh-entrypoint/entrypoint.sh" + ], + "artifact": "server", + "restart-on": [ + "program-snapshot", + "program-entrypoint" + ] } ], "accesses": [ @@ -129,21 +153,24 @@ } ], "build": { - "on": [ - { - "arg": "NATS_BASE", - "image": "nats@sha256:e4bf19f15fd3218814a4e3c9e0064e1334bd8aa20d5984b9f1a0afd084f8cc00" - }, - { - "arg": "GO_BASE", - "image": "golang@sha256:8ac98ca534ac3f51e1f420a1dd2c15e74c75cfa0f23f3ad27eb5d7236c349a0c" - } - ], "artifacts": [ { "name": "server", - "kind": "image", - "from": "Dockerfile" + "kind": "upstream", + "from": "nats:2.11.17-alpine@sha256:e4bf19f15fd3218814a4e3c9e0064e1334bd8aa20d5984b9f1a0afd084f8cc00" + }, + { + "name": "snapshot", + "kind": "bundle", + "language": "go", + "system": "alpine", + "from": "cmd/mesh-nats-snapshot", + "binary": "mesh-nats-snapshot" + }, + { + "name": "entrypoint", + "kind": "archive", + "from": "program" }, { "name": "tools", diff --git a/modules/nats/entrypoint.sh b/modules/nats/program/entrypoint.sh similarity index 100% rename from modules/nats/entrypoint.sh rename to modules/nats/program/entrypoint.sh diff --git a/modules/nats/snapshot/go.mod b/modules/nats/snapshot/go.mod deleted file mode 100644 index b089934b..00000000 --- a/modules/nats/snapshot/go.mod +++ /dev/null @@ -1,15 +0,0 @@ -module nats-snapshot - -go 1.25.0 - -require ( - github.com/klauspost/compress v1.18.5 - github.com/nats-io/nats.go v1.51.0 -) - -require ( - github.com/nats-io/nkeys v0.4.15 // indirect - github.com/nats-io/nuid v1.0.1 // indirect - golang.org/x/crypto v0.49.0 // indirect - golang.org/x/sys v0.42.0 // indirect -) diff --git a/modules/nats/snapshot/go.sum b/modules/nats/snapshot/go.sum deleted file mode 100644 index 9050eeb6..00000000 --- a/modules/nats/snapshot/go.sum +++ /dev/null @@ -1,12 +0,0 @@ -github.com/klauspost/compress v1.18.5 h1:/h1gH5Ce+VWNLSWqPzOVn6XBO+vJbCNGvjoaGBFW2IE= -github.com/klauspost/compress v1.18.5/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= -github.com/nats-io/nats.go v1.51.0 h1:ByW84XTz6W03GSSsygsZcA+xgKK8vPGaa/FCAAEHnAI= -github.com/nats-io/nats.go v1.51.0/go.mod h1:26HypzazeOkyO3/mqd1zZd53STJN0EjCYF9Uy2ZOBno= -github.com/nats-io/nkeys v0.4.15 h1:JACV5jRVO9V856KOapQ7x+EY8Jo3qw1vJt/9Jpwzkk4= -github.com/nats-io/nkeys v0.4.15/go.mod h1:CpMchTXC9fxA5zrMo4KpySxNjiDVvr8ANOSZdiNfUrs= -github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= -github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= -golang.org/x/crypto v0.49.0 h1:+Ng2ULVvLHnJ/ZFEq4KdcDd/cfjrrjjNSXNzxg0Y4U4= -golang.org/x/crypto v0.49.0/go.mod h1:ErX4dUh2UM+CFYiXZRTcMpEcN8b/1gxEuv3nODoYtCA= -golang.org/x/sys v0.42.0 h1:omrd2nAlyT5ESRdCLYdm3+fMfNFE/+Rf4bDIQImRJeo= -golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= diff --git a/modules/nats/snapshot/image_test.go b/modules/nats/snapshot/image_test.go deleted file mode 100644 index 8e9600a2..00000000 --- a/modules/nats/snapshot/image_test.go +++ /dev/null @@ -1,152 +0,0 @@ -package main - -// The declared dump, exactly as the backup holder runs it, against the bus's own image. -// -// docker build --build-arg NATS_BASE= --build-arg GO_BASE= -t mesh-nats:check .. -// MESH_TEST_NATS_IMAGE=mesh-nats:check go test -run TestTheDeclaredDump ./... - -import ( - "encoding/json" - "os" - "os/exec" - "path/filepath" - "strings" - "testing" - "time" - - "github.com/nats-io/nats.go" -) - -// manifestOf is the nats module's own definition, beside this program. -func manifestOf(t *testing.T) map[string]any { - t.Helper() - raw, err := os.ReadFile("../module.json") - if err != nil { - t.Fatal(err) - } - var m map[string]any - if err := json.Unmarshal(raw, &m); err != nil { - t.Fatal(err) - } - return m -} - -func declaredDump(t *testing.T, m map[string]any) string { - t.Helper() - for _, item := range m["data"].(map[string]any)["own"].([]any) { - it := item.(map[string]any) - if it["id"] != "jetstream" { - continue - } - backup, _ := it["backup"].(map[string]any) - if backup == nil || backup["into"] != "snapshots" { - t.Fatalf("the bus's streams are not protected by a dump into its snapshots: %v", it["backup"]) - } - return backup["dump"].(string) - } - t.Fatal("the nats module declares no jetstream data item") - return "" -} - -func serverConf(t *testing.T, m map[string]any) string { - t.Helper() - for _, r := range m["resources"].([]any) { - res := r.(map[string]any) - if res["id"] == "server-conf" { - return res["content"].(string) - } - } - t.Fatal("the nats module declares no server configuration") - return "" -} - -// The dump is the one the manifest declares, run by `sh -c` as the holder runs it, with the module's -// directories filled in; the server is the module's image with the module's own configuration and a -// user list holding the bus's own module with the controller's grants for it. -func TestTheDeclaredDumpRunsInTheBussImage(t *testing.T) { - image := os.Getenv("MESH_TEST_NATS_IMAGE") - if image == "" { - t.Skip("MESH_TEST_NATS_IMAGE unset: the bus's image, built from ../Dockerfile") - } - const name = "mesh-broker-nats" // the container the dump names - if out, _ := exec.Command("docker", "ps", "-a", "-q", "--filter", "name=^"+name+"$").Output(); len(strings.TrimSpace(string(out))) > 0 { - t.Skipf("a container called %s already runs here; this test will not touch it", name) - } - m := manifestOf(t) - dump := declaredDump(t, m) - - conf, tlsDir, data, state, snapshots := t.TempDir(), t.TempDir(), t.TempDir(), t.TempDir(), t.TempDir() - pin := selfSigned(t, tlsDir) - _ = os.WriteFile(filepath.Join(conf, "nats.conf"), []byte(serverConf(t, m)), 0o644) - accounts := liveBusConf() - accounts = accounts[strings.Index(accounts, "accounts {"):] - _ = os.WriteFile(filepath.Join(conf, "accounts.conf"), []byte(accounts), 0o644) - - docker(t, "run", "-d", "--rm", "--name", name, "--user", owner(), "-p", "127.0.0.1::4222", - "-v", data+":/data", "-v", conf+":/etc/nats:ro", "-v", tlsDir+":/tls:ro", image) - t.Cleanup(func() { _ = exec.Command("docker", "rm", "-f", name).Run() }) - _, port, _ := strings.Cut(docker(t, "port", name, "4222/tcp"), ":") - - admin := dial(t, "tls://127.0.0.1:"+port, Credential{Fingerprint: pin, User: "admin", Password: "admin"}, nil) - js, _ := admin.JetStream() - kv, err := js.CreateKeyValue(&nats.KeyValueConfig{Bucket: "mesh-controller_hand-acts"}) - if err != nil { - t.Fatal(err) - } - for _, k := range []string{"a", "b", "c"} { - if _, err := kv.Put(k, []byte("act "+k)); err != nil { - t.Fatal(err) - } - } - - run := func(cred Credential) (string, error) { - raw, _ := json.Marshal(cred) - if err := os.WriteFile(filepath.Join(state, "broker"), raw, 0o600); err != nil { - t.Fatal(err) - } - command := strings.NewReplacer("${dir:mesh-state}", state, "${dir:snapshots}", snapshots).Replace(dump) - out, err := exec.Command("sh", "-c", command).CombinedOutput() - return string(out), err - } - - began := time.Now() - out, err := run(Credential{URL: "nats://bus.example:4222", Fingerprint: pin, User: snapshotUser, Password: "snap"}) - if err != nil { - t.Fatalf("the declared dump failed: %v\n%s", err, out) - } - t.Logf("the declared dump took %s:\n%s", time.Since(began).Round(time.Millisecond), out) - tarPath := filepath.Join(snapshots, "bus.tar") - f, err := os.Open(tarPath) - if err != nil { - t.Fatalf("the dump left no bus.tar: %v", err) - } - a, err := ReadArchive(f, "") - f.Close() - if err != nil { - t.Fatal(err) - } - defer a.Close() - if err := a.Verify(); err != nil { - t.Fatal(err) - } - if len(a.Manifest.Streams) != 1 || a.Manifest.Streams[0].Name != "KV_mesh-controller_hand-acts" || a.Manifest.Streams[0].Messages != 3 { - t.Fatalf("the snapshot holds %+v", a.Manifest.Streams) - } - before, _ := os.ReadFile(tarPath) - - // A night that cannot snapshot fails, says why, and leaves last night's snapshot as it was. - out, err = run(Credential{Fingerprint: pin, User: snapshotUser, Password: "wrong"}) - if err == nil { - t.Fatalf("a dump the bus refused succeeded:\n%s", out) - } - if !strings.Contains(out, "Authorization Violation") && !strings.Contains(strings.ToLower(out), "authorization") { - t.Errorf("the failure does not say the bus refused the credential: %s", out) - } - after, _ := os.ReadFile(tarPath) - if string(after) != string(before) { - t.Error("a failed night replaced the last good snapshot") - } - if _, err := os.Stat(tarPath + ".partial"); !os.IsNotExist(err) { - t.Error("a failed night left its partial file behind") - } -} From e7ca330e1fbfa7bc716eee7b17f1705e7fb675b7 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 22 Oct 2026 10:00:15 +0200 Subject: [PATCH 4/9] Rehearse the bus upgrade on a copy of the last snapshot with a tool of the nats module, and run the docker tests only as integration tests (issue 1642, ADR 0480 clause 6) --- modules/nats/README.md | 20 +- .../nats/cmd/mesh-nats-snapshot/image_test.go | 4 +- .../nats/cmd/mesh-nats-snapshot/live_test.go | 8 +- modules/nats/cmd/nats-tools/main.go | 31 +- modules/nats/cmd/nats-tools/rehearse.go | 359 ++++++++++++++++++ .../nats-tools/rehearse_integration_test.go | 136 +++++++ modules/nats/cmd/nats-tools/rehearse_test.go | 262 +++++++++++++ modules/nats/module.json | 8 +- 8 files changed, 814 insertions(+), 14 deletions(-) create mode 100644 modules/nats/cmd/nats-tools/rehearse.go create mode 100644 modules/nats/cmd/nats-tools/rehearse_integration_test.go create mode 100644 modules/nats/cmd/nats-tools/rehearse_test.go diff --git a/modules/nats/README.md b/modules/nats/README.md index 465ff5c0..803db15e 100644 --- a/modules/nats/README.md +++ b/modules/nats/README.md @@ -7,7 +7,7 @@ artifact (`nats:-alpine@sha256:`: the digest names the image, t and `cmd/nats-tools`' release test keeps the tag equal to the server its tests run). The module's own two programs are mounted into it read-only (novox/hq ADR 0480): the snapshot program (a Go bundle of `cmd/mesh-nats-snapshot`) at `/opt/mesh`, and the entrypoint (`program/entrypoint.sh`, an archive) at -`/opt/mesh-entrypoint`, run through `sh`. Both archive resources are in the container's `restart-on`. Its tools (`cmd/nats-tools`) are served by the node's runtime and only read. +`/opt/mesh-entrypoint`, run through `sh`. Both archive resources are in the container's `restart-on`. Its tools (`cmd/nats-tools`) are served by the node's runtime and only read, but for `nats_rehearse`, which starts containers of its own on a copy of the last snapshot and nothing else. ## Its data, and how it is backed up @@ -87,28 +87,34 @@ The bus is replaced only as a planned step a person starts (novox/hq ADR 0236, A merge of a change to this module builds the bus and sends nothing. The steps, in this order: 1. **Nothing else runs.** No delivery plan is open (`mesh-controller.plans`). -2. **Snapshot with the dump the bus runs now**, through the machine's backup holder +2. **Rehearse on a copy.** `/nats.nats_rehearse` on the bus's machine, given the new build's image and + the directories holding its unpacked program and entrypoint (and the old image, which defaults to the + running bus's): it verifies the latest `bus.tar`, restores it into a throwaway directory, starts the new + declaration on it on a loopback port and checks every stream is served, stops it, starts the old + declaration on the same copy and checks again, then removes everything and answers each step. It touches + nothing of the running bus, and refuses while a bus upgrade is open. Go on only when it answers `ok`. +3. **Snapshot with the dump the bus runs now**, through the machine's backup holder (`/node-backup.now` for `nats` on the bus's machine), and verify the `bus.tar` it wrote (`docker exec -i mesh-broker-nats /opt/mesh/mesh-nats-snapshot verify`, or the old path `mesh-nats-snapshot verify` on the declaration before ADR 0480). Note where it is: that is `snapshot-taken`. -3. **Ask the operator for the moment** on the proven channel, naming what it raises: bus maintenance is +4. **Ask the operator for the moment** on the proven channel, naming what it raises: bus maintenance is open while the step runs; every bus connection is dropped for seconds and reconnects; conditions of other modules may flash and clear. Go on only on their yes. -4. **Start it:** `mesh-controller.bus` with `upgrade` true, `why`, `reversible` true and `snapshot-taken`. +5. **Start it:** `mesh-controller.bus` with `upgrade` true, `why`, `reversible` true and `snapshot-taken`. `reversible` is true because the server binary (the same digest), the data directory and the configuration do not change; `TestTheStoreSurvivesTheNewDeclarationTheOldAndTheNewAgain` shows it: the new declaration, then the old one, then the new again run on one store, each reading what the one before wrote. -5. **After:** `nats_server`, `nats_streams` (every stream and bucket as before) and the controller's +6. **After:** `nats_server`, `nats_streams` (every stream and bucket as before) and the controller's `doctor`; the step ends done when the controller has checked every stream, every consumer and a round trip (ADR 0236 section 40). Then wait for the next nightly dump and verify it: the upgrade is finished only once that dump verified. -6. **Back:** a bus upgrade to the previous build puts the old declaration back on the same store; if the +7. **Back:** a bus upgrade to the previous build puts the old declaration back on the same store; if the store itself is hurt, restore the snapshot as described above. ## Tests -- `cmd/mesh-nats-snapshot`: `MESH_TEST_DOCKER=1 go test -race ./...` — a throwaway `nats:2.11` filled like the +- `cmd/mesh-nats-snapshot`: `go test -tags integration -race ./...`, run by `mesh-controller.try` (they start throwaway containers; nobody runs them by hand) — a throwaway `nats:2.11` filled like the mesh's bus (events with a gap deleted, a work queue part worked, buckets with history, deletes and a purge, an object, durable consumers, a memory stream) and written to throughout the snapshot; the snapshot restored into a fresh server and into a new store served by a third, each compared message diff --git a/modules/nats/cmd/mesh-nats-snapshot/image_test.go b/modules/nats/cmd/mesh-nats-snapshot/image_test.go index 68ba997d..17818617 100644 --- a/modules/nats/cmd/mesh-nats-snapshot/image_test.go +++ b/modules/nats/cmd/mesh-nats-snapshot/image_test.go @@ -1,9 +1,11 @@ +//go:build integration + package main // The declared dump and the mounted entrypoint, exactly as the manifest declares them, against the upstream image // the manifest pins: the mesh's snapshot program and entrypoint mounted at /opt (novox/hq ADR 0480 clauses 1, 2, 5). // -// MESH_TEST_DOCKER=1 go test -run 'TestTheDeclaredDump|TestTheEntrypoint' . +// go test -tags integration -run 'TestTheDeclaredDump|TestTheEntrypoint|TestTheStoreSurvives' . (through mesh-controller.try) // // The image is pulled by the manifest's reference, so the server these run is the one the bus runs. diff --git a/modules/nats/cmd/mesh-nats-snapshot/live_test.go b/modules/nats/cmd/mesh-nats-snapshot/live_test.go index d6c6a4df..95e8ea7d 100644 --- a/modules/nats/cmd/mesh-nats-snapshot/live_test.go +++ b/modules/nats/cmd/mesh-nats-snapshot/live_test.go @@ -1,8 +1,10 @@ +//go:build integration + package main // The snapshot and the way back, against real servers: the bus's own release in throwaway containers. // -// MESH_TEST_DOCKER=1 go test -race ./... +// go test -tags integration -race ./... (run through mesh-controller.try, never by hand on a workstation) // // Every container and file it makes is removed when it ends. MESH_TEST_SNAPSHOT_MESSAGES sets how // many events the source holds (default 6000), to time a snapshot at the live bus's size. @@ -57,8 +59,8 @@ var snapshotGrants = struct{ publish, subscribe []string }{ func needDocker(t *testing.T) { t.Helper() - if os.Getenv("MESH_TEST_DOCKER") != "1" { - t.Skip("MESH_TEST_DOCKER is not 1: these tests start throwaway nats containers") + if _, err := exec.LookPath("docker"); err != nil { + t.Fatalf("these tests start throwaway nats containers and need docker on the machine that runs them: %v", err) } } diff --git a/modules/nats/cmd/nats-tools/main.go b/modules/nats/cmd/nats-tools/main.go index f2e0f6d2..79927266 100644 --- a/modules/nats/cmd/nats-tools/main.go +++ b/modules/nats/cmd/nats-tools/main.go @@ -1,6 +1,7 @@ // nats-tools: the mesh bus's own tools (novox/hq design 25), a Go bundle the node's runtime launches on the // machine that runs the bus. Everything here reads — the server's monitoring API and the user list the -// controller composed — and nothing changes the bus. stdout is the MCP channel; this says nothing else. +// controller composed — and nothing changes the bus; the one tool that starts anything, nats_rehearse, starts +// only containers of its own on a copy of a snapshot. stdout is the MCP channel; this says nothing else. package main import ( @@ -22,6 +23,13 @@ func num(description string) map[string]any { func arg(a map[string]any, k string) string { s, _ := a[k].(string); return s } +func first(a, b string) string { + if a != "" { + return a + } + return b +} + func limitOf(a map[string]any, fallback int) int { if v, ok := a["limit"].(float64); ok && v > 0 { return int(v) @@ -140,6 +148,27 @@ func tools(monitor Monitor, users func(context.Context) ([]byte, error), written varz, verr := get("/varz") return InForce(answer, at, werr, varz, verr), nil }}, + {Name: "nats_rehearse", + Description: "Rehearse the bus upgrade on a copy (novox/hq ADR 0480 clause 6): verify the latest snapshot (the dump's bus.tar), restore it into a throwaway directory, start the NEW declaration (the upstream image with the program and entrypoint mounted) on it on a loopback port and check it serves every stream, stop it, start the OLD declaration on the same copy and check again, and remove everything. Answers each step. It touches nothing of the running bus (container, directory, ports) and refuses while a bus upgrade is open or when that cannot be told.", + Input: map[string]any{ + "snapshot": str("the snapshot's path on this machine (default: the nats module's bus.tar)"), + "new_image": str("the new declaration's image, name:tag@sha256:digest"), + "new_program": str("the directory on this machine holding mesh-nats-snapshot, mounted at /opt/mesh (default: the module's unpacked program)"), + "new_entrypoint": str("the directory on this machine holding entrypoint.sh, mounted at /opt/mesh-entrypoint (default: the module's unpacked entrypoint)"), + "old_image": str("the old declaration's image (default: the image of the running bus's container, read only)"), + }, + Run: func(a map[string]any) (any, error) { + args := RehearseArgs{ + Snapshot: first(arg(a, "snapshot"), env("MESH_NATS_SNAPSHOT", "")), + NewImage: arg(a, "new_image"), + NewProgram: first(arg(a, "new_program"), env("MESH_NATS_PROGRAM_DIR", "")), + NewEntrypoint: first(arg(a, "new_entrypoint"), env("MESH_NATS_ENTRYPOINT_DIR", "")), + OldImage: arg(a, "old_image"), + } + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Minute) + defer cancel() + return LiveRehearsal().Run(ctx, args) + }}, } } diff --git a/modules/nats/cmd/nats-tools/rehearse.go b/modules/nats/cmd/nats-tools/rehearse.go new file mode 100644 index 00000000..88fedd0d --- /dev/null +++ b/modules/nats/cmd/nats-tools/rehearse.go @@ -0,0 +1,359 @@ +package main + +// nats_rehearse: the bus upgrade rehearsed on a copy, before the real step (novox/hq ADR 0480 clause 6, ADR 0236). +// +// It takes the latest snapshot the dump wrote (bus.tar), verifies it, restores it into a throwaway directory, starts +// the NEW declaration's container on that copy on a loopback port and checks that it serves every stream the snapshot +// holds, stops it, starts the OLD declaration on the same copy and checks again, and removes everything it made. Each +// step is answered. That is what makes `reversible` true for the bus upgrade: the new declaration and then the old +// one run on one store, both well. +// +// **It touches only its own copy.** Every container it starts is named mesh-rehearsal- and publishes only a random +// port on the loopback; its directory is a new temporary one; the running bus's container, directory and ports are +// never named, started, stopped, mounted or published. The one thing it reads of the running bus is the image +// reference of its container (`docker inspect`), as the default for the old declaration. It refuses to run while a bus +// upgrade is open, and when it cannot tell, so a rehearsal never loads the machine in the middle of the step. + +import ( + "context" + "crypto/rand" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "time" + + stdio "git.novox.be/novox/mesh-sdk/go" + "github.com/nats-io/nats.go" +) + +// StreamWant is one stream the snapshot holds and the copy must serve with at least its messages. +type StreamWant struct { + Name string `json:"name"` + Messages uint64 `json:"messages"` +} + +// Step is one answered step of the rehearsal. +type Step struct { + Step string `json:"step"` + OK bool `json:"ok"` + Detail string `json:"detail,omitempty"` +} + +// Declaration is a way to run the bus: an image, what is mounted into it and the arguments it is given. +type Declaration struct { + Name string + Image string + Mounts []string // host:container[:ro], besides the copy's own + Args []string +} + +// Rehearsal is the rehearsal and what it reaches outside itself; the tests replace the reaches. +type Rehearsal struct { + // Docker runs the docker command line and answers its standard output. + Docker func(ctx context.Context, args ...string) (string, error) + // Serves connects to the copy and checks it serves the streams. + Serves func(ctx context.Context, port, user, password string, want []StreamWant) (string, error) + // UpgradeOpen says whether a bus upgrade is open, and why not if it cannot say. + UpgradeOpen func(ctx context.Context) (bool, string, error) + // TempDir makes the throwaway directory; RemoveAll removes it. + TempDir func() (string, error) + RemoveAll func(string) error + // ID is a fresh word for the names; UID the user the containers run as, so what they write can be removed. + ID func() string + UID string + // BusContainer is the running bus's container, which is never touched. + BusContainer string +} + +// RehearseArgs are the call's arguments. +type RehearseArgs struct { + Snapshot string // host path of bus.tar + NewImage string // the new declaration's image + NewProgram string // host directory holding mesh-nats-snapshot, mounted at /opt/mesh + NewEntrypoint string // host directory holding entrypoint.sh, mounted at /opt/mesh-entrypoint + OldImage string // the old declaration's image; the running bus's when empty +} + +const ( + rehearsalUser = "rehearsal" + rehearsalConf = "listen: \"0.0.0.0:4222\"\nhttp: 0.0.0.0:8222\njetstream { store_dir: \"/data\" }\ninclude \"accounts.conf\"\n" + rehearsalAccounts = "accounts { MESH { jetstream: enabled, users: [ { user: \"%s\", password: \"%s\" } ] } }\n" +) + +// Run is the rehearsal. It answers every step; a failed step stops it, everything is removed, and the error says which. +func (r Rehearsal) Run(ctx context.Context, a RehearseArgs) (map[string]any, error) { + var steps []Step + done := func(name string, err error, detail string) error { + s := Step{Step: name, OK: err == nil, Detail: detail} + if err != nil { + s.Detail = err.Error() + } + steps = append(steps, s) + return err + } + fail := func(err error) (map[string]any, error) { + return map[string]any{"ok": false, "steps": steps}, fmt.Errorf("the rehearsal failed at %q: %v (steps: %s)", + steps[len(steps)-1].Step, err, spoken(steps)) + } + + if a.Snapshot == "" || a.NewImage == "" || a.NewProgram == "" || a.NewEntrypoint == "" { + return nil, errors.New("snapshot, new_image, new_program and new_entrypoint are needed") + } + for _, p := range []string{a.Snapshot, a.NewProgram, a.NewEntrypoint} { + if !filepath.IsAbs(p) || strings.ContainsAny(p, ":,\n") { + return nil, fmt.Errorf("%q is not an absolute path without a colon, comma or newline", p) + } + } + open, why, err := r.UpgradeOpen(ctx) + if err != nil { + _ = done("no bus upgrade is open", err, "") + return fail(fmt.Errorf("it cannot tell whether a bus upgrade is open, and refuses until it can: %w", err)) + } + if open { + _ = done("no bus upgrade is open", errors.New("a bus upgrade is open: "+why), "") + return fail(errors.New("a bus upgrade is open: " + why)) + } + _ = done("no bus upgrade is open", nil, "") + + oldImage := a.OldImage + if oldImage == "" { + out, err := r.Docker(ctx, "inspect", "-f", "{{.Config.Image}}", r.BusContainer) + oldImage = strings.TrimSpace(out) + if err != nil || oldImage == "" { + _ = done("the old declaration's image", fmt.Errorf("the running bus's image could not be read and old_image is not given: %v", err), "") + return fail(errors.New("no old image")) + } + } + _ = done("the old declaration's image", nil, oldImage) + + dir, err := r.TempDir() + if err != nil { + _ = done("a throwaway directory", err, "") + return fail(err) + } + id := r.ID() + var started []string + defer func() { + for _, n := range started { + _, _ = r.Docker(context.Background(), "rm", "-f", n) + } + _ = r.RemoveAll(dir) + }() + conf, store := filepath.Join(dir, "conf"), filepath.Join(dir, "store") + for _, d := range []string{conf, store} { + if err := os.MkdirAll(d, 0o700); err != nil { + _ = done("a throwaway directory", err, "") + return fail(err) + } + } + password := id + id + // nats.conf is what the mesh's entrypoint reads; nats-server.conf is what the upstream image runs with no arguments, + // which is how an old declaration without an entrypoint of its own starts. + for _, f := range []string{"nats.conf", "nats-server.conf"} { + if err := os.WriteFile(filepath.Join(conf, f), []byte(rehearsalConf), 0o600); err != nil { + _ = done("a throwaway directory", err, "") + return fail(err) + } + } + if err := os.WriteFile(filepath.Join(conf, "accounts.conf"), []byte(fmt.Sprintf(rehearsalAccounts, rehearsalUser, password)), 0o600); err != nil { + _ = done("a throwaway directory", err, "") + return fail(err) + } + _ = done("a throwaway directory", nil, dir) + + snapDir, snapFile := filepath.Dir(a.Snapshot), filepath.Base(a.Snapshot) + // The snapshot's directory is the module's, readable by the machine's root only; what reads it runs as root with + // the directory mounted read-only, and writes nothing there. What writes (the copy) is handed to this user. + program := []string{"-v", a.NewProgram + ":/opt/mesh:ro", "-v", snapDir + ":/in:ro"} + tool := func(extra ...string) []string { + return append(append([]string{"run", "--rm"}, program...), extra...) + } + + // 1. The snapshot is whole. + out, err := r.Docker(ctx, tool("--entrypoint", "/opt/mesh/mesh-nats-snapshot", a.NewImage, "verify", "--from", "/in/"+snapFile)...) + if err := done("the snapshot verifies", err, strings.TrimSpace(out)); err != nil { + return fail(err) + } + // 2. What it holds. + out, err = r.Docker(ctx, "run", "--rm", "-v", snapDir+":/in:ro", "--entrypoint", "tar", a.NewImage, "-xOf", "/in/"+snapFile, "manifest.json") + var want []StreamWant + if err == nil { + var m struct { + Streams []StreamWant `json:"streams"` + } + if err = json.Unmarshal([]byte(out), &m); err == nil { + want = m.Streams + if len(want) == 0 { + err = errors.New("the snapshot holds no stream") + } + } + } + if err := done("the snapshot's streams are read", err, fmt.Sprintf("%d stream(s)", len(want))); err != nil { + return fail(err) + } + // 3. A new store from it, beside nothing: the throwaway directory. + out, err = r.Docker(ctx, "run", "--rm", "-v", a.NewProgram+":/opt/mesh:ro", "-v", snapDir+":/in:ro", "-v", store+":/out", + "--entrypoint", "/opt/mesh/mesh-nats-snapshot", a.NewImage, "restore", "--into", "/out/jetstream-copy", "--from", "/in/"+snapFile) + if err := done("the snapshot is restored into the copy", err, lastLine(out)); err != nil { + return fail(err) + } + // The restore ran as root (it reads the snapshot's directory); the copy is handed to this user, and its + // jetstream directory moved to where the server looks. + _, err = r.Docker(ctx, "run", "--rm", "-v", store+":/out", "--entrypoint", "sh", a.NewImage, "-c", + "mv /out/jetstream-copy/jetstream /out/jetstream && rmdir /out/jetstream-copy && chown -R "+r.UID+" /out") + if err := done("the copy is handed to the rehearsal's user", err, ""); err != nil { + return fail(err) + } + + // 4. The new declaration, then the old one, on the same copy. + decls := []Declaration{ + {Name: "new", Image: a.NewImage, + Mounts: []string{a.NewProgram + ":/opt/mesh:ro", a.NewEntrypoint + ":/opt/mesh-entrypoint:ro"}, + Args: []string{"sh", "/opt/mesh-entrypoint/entrypoint.sh"}}, + {Name: "old", Image: oldImage}, + } + for _, d := range decls { + name := "mesh-rehearsal-" + d.Name + "-" + id + if name == r.BusContainer { + return nil, errors.New("a rehearsal never names the running bus's container") + } + args := []string{"run", "-d", "--name", name, "--user", r.UID, "-p", "127.0.0.1::4222", "-e", "MESH_NATS_CONF_POLL_SECONDS=1", + "-v", store + ":/data", "-v", conf + ":/etc/nats:ro"} + for _, m := range d.Mounts { + args = append(args, "-v", m) + } + args = append(append(args, d.Image), d.Args...) + started = append(started, name) + if _, err := r.Docker(ctx, args...); err != nil { + _ = done("the "+d.Name+" declaration starts on the copy", err, "") + return fail(err) + } + mapped, err := r.Docker(ctx, "port", name, "4222/tcp") + _, port, ok := strings.Cut(strings.TrimSpace(strings.SplitN(mapped, "\n", 2)[0]), ":") + if err != nil || !ok { + _ = done("the "+d.Name+" declaration starts on the copy", fmt.Errorf("its port is not known: %v %q", err, mapped), "") + return fail(errors.New("no port")) + } + detail, err := r.Serves(ctx, port, rehearsalUser, password, want) + if err := done("the "+d.Name+" declaration serves every stream of the copy", err, detail); err != nil { + logs, _ := r.Docker(context.Background(), "logs", "--tail", "20", name) + steps[len(steps)-1].Detail += " | its last log lines: " + strings.TrimSpace(logs) + return fail(err) + } + if _, err := r.Docker(ctx, "stop", "-t", "10", name); err != nil { + _ = done("the "+d.Name+" declaration stops", err, "") + return fail(err) + } + _ = done("the "+d.Name+" declaration stops", nil, "") + } + _ = done("everything made is removed", nil, "containers "+strings.Join(started, ", ")+" and "+dir) + return map[string]any{"ok": true, "reversible": true, "steps": steps, + "says": "the new declaration and then the old one ran healthy on one copy of the snapshot's store"}, nil +} + +func lastLine(s string) string { + lines := strings.Split(strings.TrimSpace(s), "\n") + return lines[len(lines)-1] +} + +func spoken(steps []Step) string { + var parts []string + for _, s := range steps { + mark := "ok" + if !s.OK { + mark = "FAILED" + } + parts = append(parts, s.Step+": "+mark) + } + return strings.Join(parts, "; ") +} + +// LiveRehearsal is the rehearsal as this machine reaches what it needs. +func LiveRehearsal() Rehearsal { + return Rehearsal{ + Docker: func(ctx context.Context, args ...string) (string, error) { + cmd := exec.CommandContext(ctx, "docker", args...) + var stderr strings.Builder + cmd.Stderr = &stderr + out, err := cmd.Output() + if err != nil { + return string(out), fmt.Errorf("docker %s: %v: %s", args[0], err, strings.TrimSpace(stderr.String())) + } + return string(out), nil + }, + Serves: servesStreams, + UpgradeOpen: func(context.Context) (bool, string, error) { + // The controller's own word, through the runtime: `bus` says the last step and whether it still runs. + raw, err := stdio.Ask("mesh-controller.bus", map[string]any{}) + if err != nil { + return false, "", err + } + return upgradeRunning(string(raw)) + }, + TempDir: func() (string, error) { + return os.MkdirTemp(env("MESH_NATS_REHEARSAL_DIR", os.TempDir()), "mesh-nats-rehearsal-*") + }, + RemoveAll: os.RemoveAll, + ID: func() string { b := make([]byte, 4); _, _ = rand.Read(b); return hex.EncodeToString(b) }, + UID: fmt.Sprintf("%d:%d", os.Getuid(), os.Getgid()), + BusContainer: env("MESH_NATS_CONTAINER", "mesh-broker-nats"), + } +} + +// upgradeRunning reads the controller's answer for the last bus step: running, or ended. An answer it cannot read is +// an error, so the rehearsal refuses rather than guesses. +func upgradeRunning(answer string) (bool, string, error) { + if strings.Contains(answer, "running since") { + i := strings.Index(answer, "last step") + if i < 0 { + i = 0 + } + return true, strings.TrimSpace(answer[i:min(len(answer), i+200)]), nil + } + if strings.Contains(answer, "the bus is ") || strings.Contains(answer, "last step") || strings.Contains(answer, "bus upgrade") { + return false, "", nil + } + return false, "", fmt.Errorf("the controller's answer for the bus is not one this tool reads: %.120q", answer) +} + +// servesStreams connects to the copy and checks each stream is there with at least its messages. +func servesStreams(ctx context.Context, port, user, password string, want []StreamWant) (string, error) { + var nc *nats.Conn + var err error + deadline := time.Now().Add(60 * time.Second) + for { + nc, err = nats.Connect("nats://127.0.0.1:"+port, nats.UserInfo(user, password), nats.Timeout(2*time.Second)) + if err == nil { + break + } + if time.Now().After(deadline) || ctx.Err() != nil { + return "", fmt.Errorf("the copy did not come up: %w", err) + } + time.Sleep(500 * time.Millisecond) + } + defer nc.Close() + js, err := nc.JetStream() + if err != nil { + return "", err + } + var missing []string + for _, w := range want { + info, err := js.StreamInfo(w.Name) + if err != nil { + missing = append(missing, w.Name+" ("+err.Error()+")") + continue + } + if info.State.Msgs < w.Messages { + missing = append(missing, fmt.Sprintf("%s holds %d of %d messages", w.Name, info.State.Msgs, w.Messages)) + } + } + if len(missing) > 0 { + return "", errors.New("the copy lacks: " + strings.Join(missing, "; ")) + } + return fmt.Sprintf("%d stream(s), each with at least the messages the snapshot holds", len(want)), nil +} diff --git a/modules/nats/cmd/nats-tools/rehearse_integration_test.go b/modules/nats/cmd/nats-tools/rehearse_integration_test.go new file mode 100644 index 00000000..f818cfca --- /dev/null +++ b/modules/nats/cmd/nats-tools/rehearse_integration_test.go @@ -0,0 +1,136 @@ +//go:build integration + +package main + +// The rehearsal against real containers: a source server filled and snapshotted by the real program, then the +// rehearsal run on that bus.tar with the manifest's pinned image, the real program and the real entrypoint. Run through +// mesh-controller.try (the merge check runs it with -tags integration); never by hand on a workstation. + +import ( + "context" + "encoding/json" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/nats-io/nats.go" +) + +func sh(t *testing.T, name string, args ...string) string { + t.Helper() + out, err := exec.Command(name, args...).CombinedOutput() + if err != nil { + t.Fatalf("%s %s: %v\n%s", name, strings.Join(args, " "), err, out) + } + return strings.TrimSpace(string(out)) +} + +func TestARehearsalRunsTheNewAndTheOldDeclarationOnACopyOfARealSnapshot(t *testing.T) { + if _, err := exec.LookPath("docker"); err != nil { + t.Fatalf("this test needs docker on the machine that runs it: %v", err) + } + raw, err := os.ReadFile("../../module.json") + if err != nil { + t.Fatal(err) + } + var m struct { + Build struct { + Artifacts []struct{ Name, From string } `json:"artifacts"` + } `json:"build"` + } + if err := json.Unmarshal(raw, &m); err != nil { + t.Fatal(err) + } + image := "" + for _, a := range m.Build.Artifacts { + if a.Name == "server" { + image = a.From + } + } + work := t.TempDir() + program, entry, snaps := filepath.Join(work, "program"), filepath.Join(work, "entry"), filepath.Join(work, "snaps") + for _, d := range []string{program, entry, snaps, filepath.Join(work, "srcdata")} { + if err := os.MkdirAll(d, 0o777); err != nil { + t.Fatal(err) + } + _ = os.Chmod(d, 0o777) + } + build := exec.Command("go", "build", "-trimpath", "-o", filepath.Join(program, "mesh-nats-snapshot"), "../mesh-nats-snapshot") + build.Env = append(os.Environ(), "CGO_ENABLED=0", "GOOS=linux") + if out, err := build.CombinedOutput(); err != nil { + t.Fatalf("building the snapshot program: %v\n%s", err, out) + } + script, err := os.ReadFile("../../program/entrypoint.sh") + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(entry, "entrypoint.sh"), script, 0o644); err != nil { + t.Fatal(err) + } + + // The source: a plain server with the one account, filled, and snapshotted by the real program. + conf := filepath.Join(work, "srcconf") + if err := os.MkdirAll(conf, 0o755); err != nil { + t.Fatal(err) + } + _ = os.WriteFile(filepath.Join(conf, "nats.conf"), []byte(rehearsalConf), 0o644) + _ = os.WriteFile(filepath.Join(conf, "accounts.conf"), []byte(fmt.Sprintf(rehearsalAccounts, "admin", "admin")), 0o644) + src := fmt.Sprintf("mesh-rehearsal-source-%d", os.Getpid()) + uid := fmt.Sprintf("%d:%d", os.Getuid(), os.Getgid()) + sh(t, "docker", "run", "-d", "--rm", "--name", src, "--user", uid, "-p", "127.0.0.1::4222", + "-v", filepath.Join(work, "srcdata")+":/data", "-v", conf+":/etc/nats:ro", image, "-c", "/etc/nats/nats.conf") + t.Cleanup(func() { _ = exec.Command("docker", "rm", "-f", src).Run() }) + _, port, _ := strings.Cut(sh(t, "docker", "port", src, "4222/tcp"), ":") + var nc *nats.Conn + for deadline := time.Now().Add(30 * time.Second); ; time.Sleep(300 * time.Millisecond) { + nc, err = nats.Connect("nats://127.0.0.1:"+port, nats.UserInfo("admin", "admin")) + if err == nil { + break + } + if time.Now().After(deadline) { + t.Fatal(err) + } + } + defer nc.Close() + js, _ := nc.JetStream() + kv, err := js.CreateKeyValue(&nats.KeyValueConfig{Bucket: "mesh-rehearsal"}) + if err != nil { + t.Fatal(err) + } + for i := 0; i < 5; i++ { + if _, err := kv.Put(fmt.Sprintf("k%d", i), []byte("v")); err != nil { + t.Fatal(err) + } + } + cred, _ := json.Marshal(map[string]string{"url": "nats://127.0.0.1:" + port, "user": "admin", "password": "admin"}) + cmd := exec.Command(filepath.Join(program, "mesh-nats-snapshot"), "snapshot", "--server", "nats://127.0.0.1:"+port, "--credential", "-") + cmd.Stdin = strings.NewReader(string(cred)) + tarball, err := cmd.Output() + if err != nil { + t.Fatalf("the snapshot: %v", err) + } + if err := os.WriteFile(filepath.Join(snaps, "bus.tar"), tarball, 0o644); err != nil { + t.Fatal(err) + } + + r := LiveRehearsal() + r.UpgradeOpen = func(context.Context) (bool, string, error) { return false, "", nil } + r.BusContainer = "mesh-broker-nats-not-here" + ctx, cancel := context.WithTimeout(context.Background(), 8*time.Minute) + defer cancel() + got, err := r.Run(ctx, RehearseArgs{Snapshot: filepath.Join(snaps, "bus.tar"), NewImage: image, + NewProgram: program, NewEntrypoint: entry, OldImage: image}) + if err != nil { + t.Fatalf("%v\n%+v", err, got) + } + if got["ok"] != true { + t.Fatalf("answer %+v", got) + } + if left := sh(t, "docker", "ps", "-a", "-q", "--filter", "name=mesh-rehearsal-new", "--filter", "name=mesh-rehearsal-old"); left != "" { + t.Errorf("containers are left: %s", left) + } +} diff --git a/modules/nats/cmd/nats-tools/rehearse_test.go b/modules/nats/cmd/nats-tools/rehearse_test.go new file mode 100644 index 00000000..7301571a --- /dev/null +++ b/modules/nats/cmd/nats-tools/rehearse_test.go @@ -0,0 +1,262 @@ +package main + +import ( + "context" + "errors" + "os" + "strings" + "testing" +) + +// fake is a scripted docker: every call is recorded; a call whose first argument and one later word match a key of +// fail answers that error. +type fake struct { + calls []string + outputs map[string]string // first word of the call -> output + fail map[string]error + removed []string +} + +func (f *fake) docker(_ context.Context, args ...string) (string, error) { + line := strings.Join(args, " ") + f.calls = append(f.calls, line) + for key, err := range f.fail { + if strings.Contains(line, key) { + return "", err + } + } + switch args[0] { + case "inspect": + return "nats:old@sha256:0\n", nil + case "port": + return "127.0.0.1:41000\n", nil + } + for _, a := range args { + if a == "tar" { + return `{"streams":[{"name":"KV_a","messages":3},{"name":"EVENTS","messages":10}]}`, nil + } + } + return f.outputs[args[0]], nil +} + +func rehearsal(t *testing.T, f *fake) (Rehearsal, *[]string) { + t.Helper() + dir := t.TempDir() + var serves []string + return Rehearsal{ + Docker: f.docker, + Serves: func(_ context.Context, port, user, password string, want []StreamWant) (string, error) { + serves = append(serves, port+" "+user) + if len(want) != 2 || want[0].Name != "KV_a" || want[0].Messages != 3 { + t.Errorf("the streams it was asked to find: %+v", want) + } + return "ok", nil + }, + UpgradeOpen: func(context.Context) (bool, string, error) { return false, "", nil }, + TempDir: func() (string, error) { return dir, nil }, + RemoveAll: func(p string) error { f.removed = append(f.removed, p); return os.RemoveAll(p) }, + ID: func() string { return "abcd" }, + UID: "1000:1000", + BusContainer: "mesh-broker-nats", + }, &serves +} + +var args = RehearseArgs{Snapshot: "/var/lib/snap/bus.tar", NewImage: "nats:2.11.17-alpine@sha256:aa", NewProgram: "/var/lib/p", NewEntrypoint: "/var/lib/e"} + +func TestARehearsalRunsTheNewDeclarationThenTheOldOnOneCopyAndRemovesEverything(t *testing.T) { + f := &fake{} + r, serves := rehearsal(t, f) + got, err := r.Run(context.Background(), args) + if err != nil { + t.Fatal(err) + } + if got["ok"] != true || got["reversible"] != true { + t.Fatalf("answer %v", got) + } + var names []string + for _, s := range got["steps"].([]Step) { + if !s.OK { + t.Errorf("step %q failed: %s", s.Step, s.Detail) + } + names = append(names, s.Step) + } + for _, want := range []string{"no bus upgrade is open", "the snapshot verifies", "the snapshot is restored into the copy", + "the new declaration serves every stream of the copy", "the old declaration serves every stream of the copy", "everything made is removed"} { + found := false + for _, n := range names { + found = found || n == want + } + if !found { + t.Errorf("no step %q in %v", want, names) + } + } + if len(*serves) != 2 { + t.Errorf("the copy was checked %d times, not twice (new, then old)", len(*serves)) + } + // New before old, on the same store; the old one is the running bus's image. + var runs []string + for _, c := range f.calls { + if strings.HasPrefix(c, "run -d") { + runs = append(runs, c) + } + } + if len(runs) != 2 || !strings.Contains(runs[0], "mesh-rehearsal-new-abcd") || !strings.Contains(runs[1], "mesh-rehearsal-old-abcd") { + t.Fatalf("containers started: %v", runs) + } + if !strings.Contains(runs[0], "/opt/mesh-entrypoint") || !strings.HasSuffix(runs[0], "sh /opt/mesh-entrypoint/entrypoint.sh") { + t.Errorf("the new declaration is not the mounted entrypoint: %s", runs[0]) + } + if !strings.HasSuffix(runs[1], "nats:old@sha256:0") { + t.Errorf("the old declaration is not the running bus's image: %s", runs[1]) + } + store := func(s string) string { return s[strings.Index(s, "-v ")+3 : strings.Index(s, ":/data")] } + if store(runs[0]) != store(runs[1]) { + t.Errorf("the two declarations ran on different stores: %s / %s", store(runs[0]), store(runs[1])) + } + // Everything is removed: both containers, and the directory. + for _, n := range []string{"mesh-rehearsal-new-abcd", "mesh-rehearsal-old-abcd"} { + if !contains(f.calls, "rm -f "+n) { + t.Errorf("%s was not removed", n) + } + } + if len(f.removed) != 1 { + t.Errorf("the throwaway directory was removed %d times", len(f.removed)) + } +} + +// Nothing the rehearsal runs names the running bus's container, directory or ports. +func TestARehearsalNeverTouchesTheRunningBus(t *testing.T) { + f := &fake{} + r, _ := rehearsal(t, f) + if _, err := r.Run(context.Background(), args); err != nil { + t.Fatal(err) + } + for _, c := range f.calls { + if strings.Contains(c, "mesh-broker-nats") && !strings.HasPrefix(c, "inspect -f {{.Config.Image}} mesh-broker-nats") { + t.Errorf("a call names the running bus: %s", c) + } + for _, bad := range []string{"/var/lib/mesh-broker-nats", "4222:4222", "8222", "exec "} { + if strings.Contains(c, bad) { + t.Errorf("a call reaches %q of the running bus: %s", bad, c) + } + } + if strings.HasPrefix(c, "run -d") && !strings.Contains(c, "-p 127.0.0.1::4222") { + t.Errorf("a container publishes more than a random loopback port: %s", c) + } + } +} + +func TestARehearsalRefusesWhileABusUpgradeIsOpen(t *testing.T) { + f := &fake{} + r, _ := rehearsal(t, f) + r.UpgradeOpen = func(context.Context) (bool, string, error) { return true, "step 3 running since 12:00", nil } + got, err := r.Run(context.Background(), args) + if err == nil || !strings.Contains(err.Error(), "a bus upgrade is open") { + t.Fatalf("err %v", err) + } + if got["ok"] != false { + t.Errorf("answer %v", got) + } + if len(f.calls) != 0 { + t.Errorf("it ran docker while an upgrade is open: %v", f.calls) + } +} + +func TestARehearsalRefusesWhenItCannotTellWhetherAnUpgradeIsOpen(t *testing.T) { + f := &fake{} + r, _ := rehearsal(t, f) + r.UpgradeOpen = func(context.Context) (bool, string, error) { return false, "", errors.New("no answer") } + if _, err := r.Run(context.Background(), args); err == nil || len(f.calls) != 0 { + t.Fatalf("err %v, calls %v", err, f.calls) + } +} + +func TestAFailedDeclarationStopsTheRehearsalSaysWhichAndStillRemovesEverything(t *testing.T) { + f := &fake{} + r, _ := rehearsal(t, f) + r.Serves = func(_ context.Context, port, _, _ string, _ []StreamWant) (string, error) { + return "", errors.New("KV_a holds 1 of 3 messages") + } + got, err := r.Run(context.Background(), args) + if err == nil || !strings.Contains(err.Error(), "the new declaration serves every stream of the copy") || !strings.Contains(err.Error(), "KV_a holds 1 of 3") { + t.Fatalf("err %v", err) + } + steps := got["steps"].([]Step) + if steps[len(steps)-1].OK { + t.Error("the last step is not the failed one") + } + for _, c := range f.calls { + if strings.Contains(c, "mesh-rehearsal-old") && strings.HasPrefix(c, "run") { + t.Error("the old declaration ran after the new one failed") + } + } + if !contains(f.calls, "rm -f mesh-rehearsal-new-abcd") || len(f.removed) != 1 { + t.Errorf("not cleaned up: %v %v", f.calls, f.removed) + } +} + +func TestABadSnapshotStopsTheRehearsalBeforeAnythingStarts(t *testing.T) { + f := &fake{fail: map[string]error{"verify": errors.New("archive does not match its manifest")}} + r, _ := rehearsal(t, f) + if _, err := r.Run(context.Background(), args); err == nil || !strings.Contains(err.Error(), "the snapshot verifies") { + t.Fatalf("err %v", err) + } + for _, c := range f.calls { + if strings.HasPrefix(c, "run -d") { + t.Error("a container started on an unverified snapshot") + } + } +} + +func TestTheOldImageNeedsToBeKnown(t *testing.T) { + f := &fake{fail: map[string]error{"inspect": errors.New("no such container")}} + r, _ := rehearsal(t, f) + if _, err := r.Run(context.Background(), args); err == nil || !strings.Contains(err.Error(), "no old image") { + t.Fatalf("err %v", err) + } + a := args + a.OldImage = "nats:given" + f2 := &fake{fail: map[string]error{"inspect": errors.New("no such container")}} + r2, _ := rehearsal(t, f2) + if _, err := r2.Run(context.Background(), a); err != nil { + t.Fatal(err) + } +} + +func TestPathsAreCheckedBeforeAnythingRuns(t *testing.T) { + f := &fake{} + r, _ := rehearsal(t, f) + for _, bad := range []RehearseArgs{{}, {Snapshot: "bus.tar", NewImage: "i", NewProgram: "/p", NewEntrypoint: "/e"}, {Snapshot: "/a:b/bus.tar", NewImage: "i", NewProgram: "/p", NewEntrypoint: "/e"}} { + if _, err := r.Run(context.Background(), bad); err == nil { + t.Errorf("%+v was accepted", bad) + } + } + if len(f.calls) != 0 { + t.Errorf("docker ran: %v", f.calls) + } +} + +func TestTheControllersWordForTheBusIsReadOrRefused(t *testing.T) { + for answer, want := range map[string]bool{ + `the bus is nats, built abc, on novox; never rolled out`: false, + "the bus is nats\nlast step 4: a → b on novox by x (y): running since 2026-10-22 12:00; snapshot s": true, + "last step 4: a → b on novox by x (y): done at 2026-10-22 12:00; snapshot s": false, + } { + open, _, err := upgradeRunning(answer) + if err != nil || open != want { + t.Errorf("%q: open %v, err %v", answer, open, err) + } + } + if _, _, err := upgradeRunning("{}"); err == nil { + t.Error("an answer it cannot read was taken for a closed upgrade") + } +} + +func contains(calls []string, want string) bool { + for _, c := range calls { + if c == want { + return true + } + } + return false +} diff --git a/modules/nats/module.json b/modules/nats/module.json index b3bf5771..ca172c2f 100644 --- a/modules/nats/module.json +++ b/modules/nats/module.json @@ -44,7 +44,8 @@ "nats_backlog", "nats_buckets", "nats_users", - "nats_user_can" + "nats_user_can", + "nats_rehearse" ], "guards": [ 8222 @@ -185,7 +186,10 @@ "env": { "MESH_NATS_MONITOR": "http://127.0.0.1:8222", "MESH_NATS_CONTAINER": "mesh-broker-nats", - "MESH_NATS_USERS_FILE": "/etc/nats/accounts.conf" + "MESH_NATS_USERS_FILE": "/etc/nats/accounts.conf", + "MESH_NATS_SNAPSHOT": "${dir:snapshots}/bus.tar", + "MESH_NATS_PROGRAM_DIR": "/var/lib/nats-module/snapshot", + "MESH_NATS_ENTRYPOINT_DIR": "/var/lib/nats-module/entrypoint" } }, { From 77b89ff18205c26b57883574ca0cb7a380f1210c Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 22 Oct 2026 10:01:23 +0200 Subject: [PATCH 5/9] Take the rehearsal tool out of the nats module: a new nats build is held until the bus upgrade itself, so it could only run after the step it rehearses (ADR 0236 section 36) --- modules/nats/README.md | 18 +- modules/nats/cmd/nats-tools/main.go | 31 +- modules/nats/cmd/nats-tools/rehearse.go | 359 ------------------ .../nats-tools/rehearse_integration_test.go | 136 ------- modules/nats/cmd/nats-tools/rehearse_test.go | 262 ------------- modules/nats/module.json | 8 +- 6 files changed, 9 insertions(+), 805 deletions(-) delete mode 100644 modules/nats/cmd/nats-tools/rehearse.go delete mode 100644 modules/nats/cmd/nats-tools/rehearse_integration_test.go delete mode 100644 modules/nats/cmd/nats-tools/rehearse_test.go diff --git a/modules/nats/README.md b/modules/nats/README.md index 803db15e..5a5fe68e 100644 --- a/modules/nats/README.md +++ b/modules/nats/README.md @@ -7,7 +7,7 @@ artifact (`nats:-alpine@sha256:`: the digest names the image, t and `cmd/nats-tools`' release test keeps the tag equal to the server its tests run). The module's own two programs are mounted into it read-only (novox/hq ADR 0480): the snapshot program (a Go bundle of `cmd/mesh-nats-snapshot`) at `/opt/mesh`, and the entrypoint (`program/entrypoint.sh`, an archive) at -`/opt/mesh-entrypoint`, run through `sh`. Both archive resources are in the container's `restart-on`. Its tools (`cmd/nats-tools`) are served by the node's runtime and only read, but for `nats_rehearse`, which starts containers of its own on a copy of the last snapshot and nothing else. +`/opt/mesh-entrypoint`, run through `sh`. Both archive resources are in the container's `restart-on`. Its tools (`cmd/nats-tools`) are served by the node's runtime and only read. ## Its data, and how it is backed up @@ -87,29 +87,23 @@ The bus is replaced only as a planned step a person starts (novox/hq ADR 0236, A merge of a change to this module builds the bus and sends nothing. The steps, in this order: 1. **Nothing else runs.** No delivery plan is open (`mesh-controller.plans`). -2. **Rehearse on a copy.** `/nats.nats_rehearse` on the bus's machine, given the new build's image and - the directories holding its unpacked program and entrypoint (and the old image, which defaults to the - running bus's): it verifies the latest `bus.tar`, restores it into a throwaway directory, starts the new - declaration on it on a loopback port and checks every stream is served, stops it, starts the old - declaration on the same copy and checks again, then removes everything and answers each step. It touches - nothing of the running bus, and refuses while a bus upgrade is open. Go on only when it answers `ok`. -3. **Snapshot with the dump the bus runs now**, through the machine's backup holder +2. **Snapshot with the dump the bus runs now**, through the machine's backup holder (`/node-backup.now` for `nats` on the bus's machine), and verify the `bus.tar` it wrote (`docker exec -i mesh-broker-nats /opt/mesh/mesh-nats-snapshot verify`, or the old path `mesh-nats-snapshot verify` on the declaration before ADR 0480). Note where it is: that is `snapshot-taken`. -4. **Ask the operator for the moment** on the proven channel, naming what it raises: bus maintenance is +3. **Ask the operator for the moment** on the proven channel, naming what it raises: bus maintenance is open while the step runs; every bus connection is dropped for seconds and reconnects; conditions of other modules may flash and clear. Go on only on their yes. -5. **Start it:** `mesh-controller.bus` with `upgrade` true, `why`, `reversible` true and `snapshot-taken`. +4. **Start it:** `mesh-controller.bus` with `upgrade` true, `why`, `reversible` true and `snapshot-taken`. `reversible` is true because the server binary (the same digest), the data directory and the configuration do not change; `TestTheStoreSurvivesTheNewDeclarationTheOldAndTheNewAgain` shows it: the new declaration, then the old one, then the new again run on one store, each reading what the one before wrote. -6. **After:** `nats_server`, `nats_streams` (every stream and bucket as before) and the controller's +5. **After:** `nats_server`, `nats_streams` (every stream and bucket as before) and the controller's `doctor`; the step ends done when the controller has checked every stream, every consumer and a round trip (ADR 0236 section 40). Then wait for the next nightly dump and verify it: the upgrade is finished only once that dump verified. -7. **Back:** a bus upgrade to the previous build puts the old declaration back on the same store; if the +6. **Back:** a bus upgrade to the previous build puts the old declaration back on the same store; if the store itself is hurt, restore the snapshot as described above. ## Tests diff --git a/modules/nats/cmd/nats-tools/main.go b/modules/nats/cmd/nats-tools/main.go index 79927266..f2e0f6d2 100644 --- a/modules/nats/cmd/nats-tools/main.go +++ b/modules/nats/cmd/nats-tools/main.go @@ -1,7 +1,6 @@ // nats-tools: the mesh bus's own tools (novox/hq design 25), a Go bundle the node's runtime launches on the // machine that runs the bus. Everything here reads — the server's monitoring API and the user list the -// controller composed — and nothing changes the bus; the one tool that starts anything, nats_rehearse, starts -// only containers of its own on a copy of a snapshot. stdout is the MCP channel; this says nothing else. +// controller composed — and nothing changes the bus. stdout is the MCP channel; this says nothing else. package main import ( @@ -23,13 +22,6 @@ func num(description string) map[string]any { func arg(a map[string]any, k string) string { s, _ := a[k].(string); return s } -func first(a, b string) string { - if a != "" { - return a - } - return b -} - func limitOf(a map[string]any, fallback int) int { if v, ok := a["limit"].(float64); ok && v > 0 { return int(v) @@ -148,27 +140,6 @@ func tools(monitor Monitor, users func(context.Context) ([]byte, error), written varz, verr := get("/varz") return InForce(answer, at, werr, varz, verr), nil }}, - {Name: "nats_rehearse", - Description: "Rehearse the bus upgrade on a copy (novox/hq ADR 0480 clause 6): verify the latest snapshot (the dump's bus.tar), restore it into a throwaway directory, start the NEW declaration (the upstream image with the program and entrypoint mounted) on it on a loopback port and check it serves every stream, stop it, start the OLD declaration on the same copy and check again, and remove everything. Answers each step. It touches nothing of the running bus (container, directory, ports) and refuses while a bus upgrade is open or when that cannot be told.", - Input: map[string]any{ - "snapshot": str("the snapshot's path on this machine (default: the nats module's bus.tar)"), - "new_image": str("the new declaration's image, name:tag@sha256:digest"), - "new_program": str("the directory on this machine holding mesh-nats-snapshot, mounted at /opt/mesh (default: the module's unpacked program)"), - "new_entrypoint": str("the directory on this machine holding entrypoint.sh, mounted at /opt/mesh-entrypoint (default: the module's unpacked entrypoint)"), - "old_image": str("the old declaration's image (default: the image of the running bus's container, read only)"), - }, - Run: func(a map[string]any) (any, error) { - args := RehearseArgs{ - Snapshot: first(arg(a, "snapshot"), env("MESH_NATS_SNAPSHOT", "")), - NewImage: arg(a, "new_image"), - NewProgram: first(arg(a, "new_program"), env("MESH_NATS_PROGRAM_DIR", "")), - NewEntrypoint: first(arg(a, "new_entrypoint"), env("MESH_NATS_ENTRYPOINT_DIR", "")), - OldImage: arg(a, "old_image"), - } - ctx, cancel := context.WithTimeout(context.Background(), 20*time.Minute) - defer cancel() - return LiveRehearsal().Run(ctx, args) - }}, } } diff --git a/modules/nats/cmd/nats-tools/rehearse.go b/modules/nats/cmd/nats-tools/rehearse.go deleted file mode 100644 index 88fedd0d..00000000 --- a/modules/nats/cmd/nats-tools/rehearse.go +++ /dev/null @@ -1,359 +0,0 @@ -package main - -// nats_rehearse: the bus upgrade rehearsed on a copy, before the real step (novox/hq ADR 0480 clause 6, ADR 0236). -// -// It takes the latest snapshot the dump wrote (bus.tar), verifies it, restores it into a throwaway directory, starts -// the NEW declaration's container on that copy on a loopback port and checks that it serves every stream the snapshot -// holds, stops it, starts the OLD declaration on the same copy and checks again, and removes everything it made. Each -// step is answered. That is what makes `reversible` true for the bus upgrade: the new declaration and then the old -// one run on one store, both well. -// -// **It touches only its own copy.** Every container it starts is named mesh-rehearsal- and publishes only a random -// port on the loopback; its directory is a new temporary one; the running bus's container, directory and ports are -// never named, started, stopped, mounted or published. The one thing it reads of the running bus is the image -// reference of its container (`docker inspect`), as the default for the old declaration. It refuses to run while a bus -// upgrade is open, and when it cannot tell, so a rehearsal never loads the machine in the middle of the step. - -import ( - "context" - "crypto/rand" - "encoding/hex" - "encoding/json" - "errors" - "fmt" - "os" - "os/exec" - "path/filepath" - "strings" - "time" - - stdio "git.novox.be/novox/mesh-sdk/go" - "github.com/nats-io/nats.go" -) - -// StreamWant is one stream the snapshot holds and the copy must serve with at least its messages. -type StreamWant struct { - Name string `json:"name"` - Messages uint64 `json:"messages"` -} - -// Step is one answered step of the rehearsal. -type Step struct { - Step string `json:"step"` - OK bool `json:"ok"` - Detail string `json:"detail,omitempty"` -} - -// Declaration is a way to run the bus: an image, what is mounted into it and the arguments it is given. -type Declaration struct { - Name string - Image string - Mounts []string // host:container[:ro], besides the copy's own - Args []string -} - -// Rehearsal is the rehearsal and what it reaches outside itself; the tests replace the reaches. -type Rehearsal struct { - // Docker runs the docker command line and answers its standard output. - Docker func(ctx context.Context, args ...string) (string, error) - // Serves connects to the copy and checks it serves the streams. - Serves func(ctx context.Context, port, user, password string, want []StreamWant) (string, error) - // UpgradeOpen says whether a bus upgrade is open, and why not if it cannot say. - UpgradeOpen func(ctx context.Context) (bool, string, error) - // TempDir makes the throwaway directory; RemoveAll removes it. - TempDir func() (string, error) - RemoveAll func(string) error - // ID is a fresh word for the names; UID the user the containers run as, so what they write can be removed. - ID func() string - UID string - // BusContainer is the running bus's container, which is never touched. - BusContainer string -} - -// RehearseArgs are the call's arguments. -type RehearseArgs struct { - Snapshot string // host path of bus.tar - NewImage string // the new declaration's image - NewProgram string // host directory holding mesh-nats-snapshot, mounted at /opt/mesh - NewEntrypoint string // host directory holding entrypoint.sh, mounted at /opt/mesh-entrypoint - OldImage string // the old declaration's image; the running bus's when empty -} - -const ( - rehearsalUser = "rehearsal" - rehearsalConf = "listen: \"0.0.0.0:4222\"\nhttp: 0.0.0.0:8222\njetstream { store_dir: \"/data\" }\ninclude \"accounts.conf\"\n" - rehearsalAccounts = "accounts { MESH { jetstream: enabled, users: [ { user: \"%s\", password: \"%s\" } ] } }\n" -) - -// Run is the rehearsal. It answers every step; a failed step stops it, everything is removed, and the error says which. -func (r Rehearsal) Run(ctx context.Context, a RehearseArgs) (map[string]any, error) { - var steps []Step - done := func(name string, err error, detail string) error { - s := Step{Step: name, OK: err == nil, Detail: detail} - if err != nil { - s.Detail = err.Error() - } - steps = append(steps, s) - return err - } - fail := func(err error) (map[string]any, error) { - return map[string]any{"ok": false, "steps": steps}, fmt.Errorf("the rehearsal failed at %q: %v (steps: %s)", - steps[len(steps)-1].Step, err, spoken(steps)) - } - - if a.Snapshot == "" || a.NewImage == "" || a.NewProgram == "" || a.NewEntrypoint == "" { - return nil, errors.New("snapshot, new_image, new_program and new_entrypoint are needed") - } - for _, p := range []string{a.Snapshot, a.NewProgram, a.NewEntrypoint} { - if !filepath.IsAbs(p) || strings.ContainsAny(p, ":,\n") { - return nil, fmt.Errorf("%q is not an absolute path without a colon, comma or newline", p) - } - } - open, why, err := r.UpgradeOpen(ctx) - if err != nil { - _ = done("no bus upgrade is open", err, "") - return fail(fmt.Errorf("it cannot tell whether a bus upgrade is open, and refuses until it can: %w", err)) - } - if open { - _ = done("no bus upgrade is open", errors.New("a bus upgrade is open: "+why), "") - return fail(errors.New("a bus upgrade is open: " + why)) - } - _ = done("no bus upgrade is open", nil, "") - - oldImage := a.OldImage - if oldImage == "" { - out, err := r.Docker(ctx, "inspect", "-f", "{{.Config.Image}}", r.BusContainer) - oldImage = strings.TrimSpace(out) - if err != nil || oldImage == "" { - _ = done("the old declaration's image", fmt.Errorf("the running bus's image could not be read and old_image is not given: %v", err), "") - return fail(errors.New("no old image")) - } - } - _ = done("the old declaration's image", nil, oldImage) - - dir, err := r.TempDir() - if err != nil { - _ = done("a throwaway directory", err, "") - return fail(err) - } - id := r.ID() - var started []string - defer func() { - for _, n := range started { - _, _ = r.Docker(context.Background(), "rm", "-f", n) - } - _ = r.RemoveAll(dir) - }() - conf, store := filepath.Join(dir, "conf"), filepath.Join(dir, "store") - for _, d := range []string{conf, store} { - if err := os.MkdirAll(d, 0o700); err != nil { - _ = done("a throwaway directory", err, "") - return fail(err) - } - } - password := id + id - // nats.conf is what the mesh's entrypoint reads; nats-server.conf is what the upstream image runs with no arguments, - // which is how an old declaration without an entrypoint of its own starts. - for _, f := range []string{"nats.conf", "nats-server.conf"} { - if err := os.WriteFile(filepath.Join(conf, f), []byte(rehearsalConf), 0o600); err != nil { - _ = done("a throwaway directory", err, "") - return fail(err) - } - } - if err := os.WriteFile(filepath.Join(conf, "accounts.conf"), []byte(fmt.Sprintf(rehearsalAccounts, rehearsalUser, password)), 0o600); err != nil { - _ = done("a throwaway directory", err, "") - return fail(err) - } - _ = done("a throwaway directory", nil, dir) - - snapDir, snapFile := filepath.Dir(a.Snapshot), filepath.Base(a.Snapshot) - // The snapshot's directory is the module's, readable by the machine's root only; what reads it runs as root with - // the directory mounted read-only, and writes nothing there. What writes (the copy) is handed to this user. - program := []string{"-v", a.NewProgram + ":/opt/mesh:ro", "-v", snapDir + ":/in:ro"} - tool := func(extra ...string) []string { - return append(append([]string{"run", "--rm"}, program...), extra...) - } - - // 1. The snapshot is whole. - out, err := r.Docker(ctx, tool("--entrypoint", "/opt/mesh/mesh-nats-snapshot", a.NewImage, "verify", "--from", "/in/"+snapFile)...) - if err := done("the snapshot verifies", err, strings.TrimSpace(out)); err != nil { - return fail(err) - } - // 2. What it holds. - out, err = r.Docker(ctx, "run", "--rm", "-v", snapDir+":/in:ro", "--entrypoint", "tar", a.NewImage, "-xOf", "/in/"+snapFile, "manifest.json") - var want []StreamWant - if err == nil { - var m struct { - Streams []StreamWant `json:"streams"` - } - if err = json.Unmarshal([]byte(out), &m); err == nil { - want = m.Streams - if len(want) == 0 { - err = errors.New("the snapshot holds no stream") - } - } - } - if err := done("the snapshot's streams are read", err, fmt.Sprintf("%d stream(s)", len(want))); err != nil { - return fail(err) - } - // 3. A new store from it, beside nothing: the throwaway directory. - out, err = r.Docker(ctx, "run", "--rm", "-v", a.NewProgram+":/opt/mesh:ro", "-v", snapDir+":/in:ro", "-v", store+":/out", - "--entrypoint", "/opt/mesh/mesh-nats-snapshot", a.NewImage, "restore", "--into", "/out/jetstream-copy", "--from", "/in/"+snapFile) - if err := done("the snapshot is restored into the copy", err, lastLine(out)); err != nil { - return fail(err) - } - // The restore ran as root (it reads the snapshot's directory); the copy is handed to this user, and its - // jetstream directory moved to where the server looks. - _, err = r.Docker(ctx, "run", "--rm", "-v", store+":/out", "--entrypoint", "sh", a.NewImage, "-c", - "mv /out/jetstream-copy/jetstream /out/jetstream && rmdir /out/jetstream-copy && chown -R "+r.UID+" /out") - if err := done("the copy is handed to the rehearsal's user", err, ""); err != nil { - return fail(err) - } - - // 4. The new declaration, then the old one, on the same copy. - decls := []Declaration{ - {Name: "new", Image: a.NewImage, - Mounts: []string{a.NewProgram + ":/opt/mesh:ro", a.NewEntrypoint + ":/opt/mesh-entrypoint:ro"}, - Args: []string{"sh", "/opt/mesh-entrypoint/entrypoint.sh"}}, - {Name: "old", Image: oldImage}, - } - for _, d := range decls { - name := "mesh-rehearsal-" + d.Name + "-" + id - if name == r.BusContainer { - return nil, errors.New("a rehearsal never names the running bus's container") - } - args := []string{"run", "-d", "--name", name, "--user", r.UID, "-p", "127.0.0.1::4222", "-e", "MESH_NATS_CONF_POLL_SECONDS=1", - "-v", store + ":/data", "-v", conf + ":/etc/nats:ro"} - for _, m := range d.Mounts { - args = append(args, "-v", m) - } - args = append(append(args, d.Image), d.Args...) - started = append(started, name) - if _, err := r.Docker(ctx, args...); err != nil { - _ = done("the "+d.Name+" declaration starts on the copy", err, "") - return fail(err) - } - mapped, err := r.Docker(ctx, "port", name, "4222/tcp") - _, port, ok := strings.Cut(strings.TrimSpace(strings.SplitN(mapped, "\n", 2)[0]), ":") - if err != nil || !ok { - _ = done("the "+d.Name+" declaration starts on the copy", fmt.Errorf("its port is not known: %v %q", err, mapped), "") - return fail(errors.New("no port")) - } - detail, err := r.Serves(ctx, port, rehearsalUser, password, want) - if err := done("the "+d.Name+" declaration serves every stream of the copy", err, detail); err != nil { - logs, _ := r.Docker(context.Background(), "logs", "--tail", "20", name) - steps[len(steps)-1].Detail += " | its last log lines: " + strings.TrimSpace(logs) - return fail(err) - } - if _, err := r.Docker(ctx, "stop", "-t", "10", name); err != nil { - _ = done("the "+d.Name+" declaration stops", err, "") - return fail(err) - } - _ = done("the "+d.Name+" declaration stops", nil, "") - } - _ = done("everything made is removed", nil, "containers "+strings.Join(started, ", ")+" and "+dir) - return map[string]any{"ok": true, "reversible": true, "steps": steps, - "says": "the new declaration and then the old one ran healthy on one copy of the snapshot's store"}, nil -} - -func lastLine(s string) string { - lines := strings.Split(strings.TrimSpace(s), "\n") - return lines[len(lines)-1] -} - -func spoken(steps []Step) string { - var parts []string - for _, s := range steps { - mark := "ok" - if !s.OK { - mark = "FAILED" - } - parts = append(parts, s.Step+": "+mark) - } - return strings.Join(parts, "; ") -} - -// LiveRehearsal is the rehearsal as this machine reaches what it needs. -func LiveRehearsal() Rehearsal { - return Rehearsal{ - Docker: func(ctx context.Context, args ...string) (string, error) { - cmd := exec.CommandContext(ctx, "docker", args...) - var stderr strings.Builder - cmd.Stderr = &stderr - out, err := cmd.Output() - if err != nil { - return string(out), fmt.Errorf("docker %s: %v: %s", args[0], err, strings.TrimSpace(stderr.String())) - } - return string(out), nil - }, - Serves: servesStreams, - UpgradeOpen: func(context.Context) (bool, string, error) { - // The controller's own word, through the runtime: `bus` says the last step and whether it still runs. - raw, err := stdio.Ask("mesh-controller.bus", map[string]any{}) - if err != nil { - return false, "", err - } - return upgradeRunning(string(raw)) - }, - TempDir: func() (string, error) { - return os.MkdirTemp(env("MESH_NATS_REHEARSAL_DIR", os.TempDir()), "mesh-nats-rehearsal-*") - }, - RemoveAll: os.RemoveAll, - ID: func() string { b := make([]byte, 4); _, _ = rand.Read(b); return hex.EncodeToString(b) }, - UID: fmt.Sprintf("%d:%d", os.Getuid(), os.Getgid()), - BusContainer: env("MESH_NATS_CONTAINER", "mesh-broker-nats"), - } -} - -// upgradeRunning reads the controller's answer for the last bus step: running, or ended. An answer it cannot read is -// an error, so the rehearsal refuses rather than guesses. -func upgradeRunning(answer string) (bool, string, error) { - if strings.Contains(answer, "running since") { - i := strings.Index(answer, "last step") - if i < 0 { - i = 0 - } - return true, strings.TrimSpace(answer[i:min(len(answer), i+200)]), nil - } - if strings.Contains(answer, "the bus is ") || strings.Contains(answer, "last step") || strings.Contains(answer, "bus upgrade") { - return false, "", nil - } - return false, "", fmt.Errorf("the controller's answer for the bus is not one this tool reads: %.120q", answer) -} - -// servesStreams connects to the copy and checks each stream is there with at least its messages. -func servesStreams(ctx context.Context, port, user, password string, want []StreamWant) (string, error) { - var nc *nats.Conn - var err error - deadline := time.Now().Add(60 * time.Second) - for { - nc, err = nats.Connect("nats://127.0.0.1:"+port, nats.UserInfo(user, password), nats.Timeout(2*time.Second)) - if err == nil { - break - } - if time.Now().After(deadline) || ctx.Err() != nil { - return "", fmt.Errorf("the copy did not come up: %w", err) - } - time.Sleep(500 * time.Millisecond) - } - defer nc.Close() - js, err := nc.JetStream() - if err != nil { - return "", err - } - var missing []string - for _, w := range want { - info, err := js.StreamInfo(w.Name) - if err != nil { - missing = append(missing, w.Name+" ("+err.Error()+")") - continue - } - if info.State.Msgs < w.Messages { - missing = append(missing, fmt.Sprintf("%s holds %d of %d messages", w.Name, info.State.Msgs, w.Messages)) - } - } - if len(missing) > 0 { - return "", errors.New("the copy lacks: " + strings.Join(missing, "; ")) - } - return fmt.Sprintf("%d stream(s), each with at least the messages the snapshot holds", len(want)), nil -} diff --git a/modules/nats/cmd/nats-tools/rehearse_integration_test.go b/modules/nats/cmd/nats-tools/rehearse_integration_test.go deleted file mode 100644 index f818cfca..00000000 --- a/modules/nats/cmd/nats-tools/rehearse_integration_test.go +++ /dev/null @@ -1,136 +0,0 @@ -//go:build integration - -package main - -// The rehearsal against real containers: a source server filled and snapshotted by the real program, then the -// rehearsal run on that bus.tar with the manifest's pinned image, the real program and the real entrypoint. Run through -// mesh-controller.try (the merge check runs it with -tags integration); never by hand on a workstation. - -import ( - "context" - "encoding/json" - "fmt" - "os" - "os/exec" - "path/filepath" - "strings" - "testing" - "time" - - "github.com/nats-io/nats.go" -) - -func sh(t *testing.T, name string, args ...string) string { - t.Helper() - out, err := exec.Command(name, args...).CombinedOutput() - if err != nil { - t.Fatalf("%s %s: %v\n%s", name, strings.Join(args, " "), err, out) - } - return strings.TrimSpace(string(out)) -} - -func TestARehearsalRunsTheNewAndTheOldDeclarationOnACopyOfARealSnapshot(t *testing.T) { - if _, err := exec.LookPath("docker"); err != nil { - t.Fatalf("this test needs docker on the machine that runs it: %v", err) - } - raw, err := os.ReadFile("../../module.json") - if err != nil { - t.Fatal(err) - } - var m struct { - Build struct { - Artifacts []struct{ Name, From string } `json:"artifacts"` - } `json:"build"` - } - if err := json.Unmarshal(raw, &m); err != nil { - t.Fatal(err) - } - image := "" - for _, a := range m.Build.Artifacts { - if a.Name == "server" { - image = a.From - } - } - work := t.TempDir() - program, entry, snaps := filepath.Join(work, "program"), filepath.Join(work, "entry"), filepath.Join(work, "snaps") - for _, d := range []string{program, entry, snaps, filepath.Join(work, "srcdata")} { - if err := os.MkdirAll(d, 0o777); err != nil { - t.Fatal(err) - } - _ = os.Chmod(d, 0o777) - } - build := exec.Command("go", "build", "-trimpath", "-o", filepath.Join(program, "mesh-nats-snapshot"), "../mesh-nats-snapshot") - build.Env = append(os.Environ(), "CGO_ENABLED=0", "GOOS=linux") - if out, err := build.CombinedOutput(); err != nil { - t.Fatalf("building the snapshot program: %v\n%s", err, out) - } - script, err := os.ReadFile("../../program/entrypoint.sh") - if err != nil { - t.Fatal(err) - } - if err := os.WriteFile(filepath.Join(entry, "entrypoint.sh"), script, 0o644); err != nil { - t.Fatal(err) - } - - // The source: a plain server with the one account, filled, and snapshotted by the real program. - conf := filepath.Join(work, "srcconf") - if err := os.MkdirAll(conf, 0o755); err != nil { - t.Fatal(err) - } - _ = os.WriteFile(filepath.Join(conf, "nats.conf"), []byte(rehearsalConf), 0o644) - _ = os.WriteFile(filepath.Join(conf, "accounts.conf"), []byte(fmt.Sprintf(rehearsalAccounts, "admin", "admin")), 0o644) - src := fmt.Sprintf("mesh-rehearsal-source-%d", os.Getpid()) - uid := fmt.Sprintf("%d:%d", os.Getuid(), os.Getgid()) - sh(t, "docker", "run", "-d", "--rm", "--name", src, "--user", uid, "-p", "127.0.0.1::4222", - "-v", filepath.Join(work, "srcdata")+":/data", "-v", conf+":/etc/nats:ro", image, "-c", "/etc/nats/nats.conf") - t.Cleanup(func() { _ = exec.Command("docker", "rm", "-f", src).Run() }) - _, port, _ := strings.Cut(sh(t, "docker", "port", src, "4222/tcp"), ":") - var nc *nats.Conn - for deadline := time.Now().Add(30 * time.Second); ; time.Sleep(300 * time.Millisecond) { - nc, err = nats.Connect("nats://127.0.0.1:"+port, nats.UserInfo("admin", "admin")) - if err == nil { - break - } - if time.Now().After(deadline) { - t.Fatal(err) - } - } - defer nc.Close() - js, _ := nc.JetStream() - kv, err := js.CreateKeyValue(&nats.KeyValueConfig{Bucket: "mesh-rehearsal"}) - if err != nil { - t.Fatal(err) - } - for i := 0; i < 5; i++ { - if _, err := kv.Put(fmt.Sprintf("k%d", i), []byte("v")); err != nil { - t.Fatal(err) - } - } - cred, _ := json.Marshal(map[string]string{"url": "nats://127.0.0.1:" + port, "user": "admin", "password": "admin"}) - cmd := exec.Command(filepath.Join(program, "mesh-nats-snapshot"), "snapshot", "--server", "nats://127.0.0.1:"+port, "--credential", "-") - cmd.Stdin = strings.NewReader(string(cred)) - tarball, err := cmd.Output() - if err != nil { - t.Fatalf("the snapshot: %v", err) - } - if err := os.WriteFile(filepath.Join(snaps, "bus.tar"), tarball, 0o644); err != nil { - t.Fatal(err) - } - - r := LiveRehearsal() - r.UpgradeOpen = func(context.Context) (bool, string, error) { return false, "", nil } - r.BusContainer = "mesh-broker-nats-not-here" - ctx, cancel := context.WithTimeout(context.Background(), 8*time.Minute) - defer cancel() - got, err := r.Run(ctx, RehearseArgs{Snapshot: filepath.Join(snaps, "bus.tar"), NewImage: image, - NewProgram: program, NewEntrypoint: entry, OldImage: image}) - if err != nil { - t.Fatalf("%v\n%+v", err, got) - } - if got["ok"] != true { - t.Fatalf("answer %+v", got) - } - if left := sh(t, "docker", "ps", "-a", "-q", "--filter", "name=mesh-rehearsal-new", "--filter", "name=mesh-rehearsal-old"); left != "" { - t.Errorf("containers are left: %s", left) - } -} diff --git a/modules/nats/cmd/nats-tools/rehearse_test.go b/modules/nats/cmd/nats-tools/rehearse_test.go deleted file mode 100644 index 7301571a..00000000 --- a/modules/nats/cmd/nats-tools/rehearse_test.go +++ /dev/null @@ -1,262 +0,0 @@ -package main - -import ( - "context" - "errors" - "os" - "strings" - "testing" -) - -// fake is a scripted docker: every call is recorded; a call whose first argument and one later word match a key of -// fail answers that error. -type fake struct { - calls []string - outputs map[string]string // first word of the call -> output - fail map[string]error - removed []string -} - -func (f *fake) docker(_ context.Context, args ...string) (string, error) { - line := strings.Join(args, " ") - f.calls = append(f.calls, line) - for key, err := range f.fail { - if strings.Contains(line, key) { - return "", err - } - } - switch args[0] { - case "inspect": - return "nats:old@sha256:0\n", nil - case "port": - return "127.0.0.1:41000\n", nil - } - for _, a := range args { - if a == "tar" { - return `{"streams":[{"name":"KV_a","messages":3},{"name":"EVENTS","messages":10}]}`, nil - } - } - return f.outputs[args[0]], nil -} - -func rehearsal(t *testing.T, f *fake) (Rehearsal, *[]string) { - t.Helper() - dir := t.TempDir() - var serves []string - return Rehearsal{ - Docker: f.docker, - Serves: func(_ context.Context, port, user, password string, want []StreamWant) (string, error) { - serves = append(serves, port+" "+user) - if len(want) != 2 || want[0].Name != "KV_a" || want[0].Messages != 3 { - t.Errorf("the streams it was asked to find: %+v", want) - } - return "ok", nil - }, - UpgradeOpen: func(context.Context) (bool, string, error) { return false, "", nil }, - TempDir: func() (string, error) { return dir, nil }, - RemoveAll: func(p string) error { f.removed = append(f.removed, p); return os.RemoveAll(p) }, - ID: func() string { return "abcd" }, - UID: "1000:1000", - BusContainer: "mesh-broker-nats", - }, &serves -} - -var args = RehearseArgs{Snapshot: "/var/lib/snap/bus.tar", NewImage: "nats:2.11.17-alpine@sha256:aa", NewProgram: "/var/lib/p", NewEntrypoint: "/var/lib/e"} - -func TestARehearsalRunsTheNewDeclarationThenTheOldOnOneCopyAndRemovesEverything(t *testing.T) { - f := &fake{} - r, serves := rehearsal(t, f) - got, err := r.Run(context.Background(), args) - if err != nil { - t.Fatal(err) - } - if got["ok"] != true || got["reversible"] != true { - t.Fatalf("answer %v", got) - } - var names []string - for _, s := range got["steps"].([]Step) { - if !s.OK { - t.Errorf("step %q failed: %s", s.Step, s.Detail) - } - names = append(names, s.Step) - } - for _, want := range []string{"no bus upgrade is open", "the snapshot verifies", "the snapshot is restored into the copy", - "the new declaration serves every stream of the copy", "the old declaration serves every stream of the copy", "everything made is removed"} { - found := false - for _, n := range names { - found = found || n == want - } - if !found { - t.Errorf("no step %q in %v", want, names) - } - } - if len(*serves) != 2 { - t.Errorf("the copy was checked %d times, not twice (new, then old)", len(*serves)) - } - // New before old, on the same store; the old one is the running bus's image. - var runs []string - for _, c := range f.calls { - if strings.HasPrefix(c, "run -d") { - runs = append(runs, c) - } - } - if len(runs) != 2 || !strings.Contains(runs[0], "mesh-rehearsal-new-abcd") || !strings.Contains(runs[1], "mesh-rehearsal-old-abcd") { - t.Fatalf("containers started: %v", runs) - } - if !strings.Contains(runs[0], "/opt/mesh-entrypoint") || !strings.HasSuffix(runs[0], "sh /opt/mesh-entrypoint/entrypoint.sh") { - t.Errorf("the new declaration is not the mounted entrypoint: %s", runs[0]) - } - if !strings.HasSuffix(runs[1], "nats:old@sha256:0") { - t.Errorf("the old declaration is not the running bus's image: %s", runs[1]) - } - store := func(s string) string { return s[strings.Index(s, "-v ")+3 : strings.Index(s, ":/data")] } - if store(runs[0]) != store(runs[1]) { - t.Errorf("the two declarations ran on different stores: %s / %s", store(runs[0]), store(runs[1])) - } - // Everything is removed: both containers, and the directory. - for _, n := range []string{"mesh-rehearsal-new-abcd", "mesh-rehearsal-old-abcd"} { - if !contains(f.calls, "rm -f "+n) { - t.Errorf("%s was not removed", n) - } - } - if len(f.removed) != 1 { - t.Errorf("the throwaway directory was removed %d times", len(f.removed)) - } -} - -// Nothing the rehearsal runs names the running bus's container, directory or ports. -func TestARehearsalNeverTouchesTheRunningBus(t *testing.T) { - f := &fake{} - r, _ := rehearsal(t, f) - if _, err := r.Run(context.Background(), args); err != nil { - t.Fatal(err) - } - for _, c := range f.calls { - if strings.Contains(c, "mesh-broker-nats") && !strings.HasPrefix(c, "inspect -f {{.Config.Image}} mesh-broker-nats") { - t.Errorf("a call names the running bus: %s", c) - } - for _, bad := range []string{"/var/lib/mesh-broker-nats", "4222:4222", "8222", "exec "} { - if strings.Contains(c, bad) { - t.Errorf("a call reaches %q of the running bus: %s", bad, c) - } - } - if strings.HasPrefix(c, "run -d") && !strings.Contains(c, "-p 127.0.0.1::4222") { - t.Errorf("a container publishes more than a random loopback port: %s", c) - } - } -} - -func TestARehearsalRefusesWhileABusUpgradeIsOpen(t *testing.T) { - f := &fake{} - r, _ := rehearsal(t, f) - r.UpgradeOpen = func(context.Context) (bool, string, error) { return true, "step 3 running since 12:00", nil } - got, err := r.Run(context.Background(), args) - if err == nil || !strings.Contains(err.Error(), "a bus upgrade is open") { - t.Fatalf("err %v", err) - } - if got["ok"] != false { - t.Errorf("answer %v", got) - } - if len(f.calls) != 0 { - t.Errorf("it ran docker while an upgrade is open: %v", f.calls) - } -} - -func TestARehearsalRefusesWhenItCannotTellWhetherAnUpgradeIsOpen(t *testing.T) { - f := &fake{} - r, _ := rehearsal(t, f) - r.UpgradeOpen = func(context.Context) (bool, string, error) { return false, "", errors.New("no answer") } - if _, err := r.Run(context.Background(), args); err == nil || len(f.calls) != 0 { - t.Fatalf("err %v, calls %v", err, f.calls) - } -} - -func TestAFailedDeclarationStopsTheRehearsalSaysWhichAndStillRemovesEverything(t *testing.T) { - f := &fake{} - r, _ := rehearsal(t, f) - r.Serves = func(_ context.Context, port, _, _ string, _ []StreamWant) (string, error) { - return "", errors.New("KV_a holds 1 of 3 messages") - } - got, err := r.Run(context.Background(), args) - if err == nil || !strings.Contains(err.Error(), "the new declaration serves every stream of the copy") || !strings.Contains(err.Error(), "KV_a holds 1 of 3") { - t.Fatalf("err %v", err) - } - steps := got["steps"].([]Step) - if steps[len(steps)-1].OK { - t.Error("the last step is not the failed one") - } - for _, c := range f.calls { - if strings.Contains(c, "mesh-rehearsal-old") && strings.HasPrefix(c, "run") { - t.Error("the old declaration ran after the new one failed") - } - } - if !contains(f.calls, "rm -f mesh-rehearsal-new-abcd") || len(f.removed) != 1 { - t.Errorf("not cleaned up: %v %v", f.calls, f.removed) - } -} - -func TestABadSnapshotStopsTheRehearsalBeforeAnythingStarts(t *testing.T) { - f := &fake{fail: map[string]error{"verify": errors.New("archive does not match its manifest")}} - r, _ := rehearsal(t, f) - if _, err := r.Run(context.Background(), args); err == nil || !strings.Contains(err.Error(), "the snapshot verifies") { - t.Fatalf("err %v", err) - } - for _, c := range f.calls { - if strings.HasPrefix(c, "run -d") { - t.Error("a container started on an unverified snapshot") - } - } -} - -func TestTheOldImageNeedsToBeKnown(t *testing.T) { - f := &fake{fail: map[string]error{"inspect": errors.New("no such container")}} - r, _ := rehearsal(t, f) - if _, err := r.Run(context.Background(), args); err == nil || !strings.Contains(err.Error(), "no old image") { - t.Fatalf("err %v", err) - } - a := args - a.OldImage = "nats:given" - f2 := &fake{fail: map[string]error{"inspect": errors.New("no such container")}} - r2, _ := rehearsal(t, f2) - if _, err := r2.Run(context.Background(), a); err != nil { - t.Fatal(err) - } -} - -func TestPathsAreCheckedBeforeAnythingRuns(t *testing.T) { - f := &fake{} - r, _ := rehearsal(t, f) - for _, bad := range []RehearseArgs{{}, {Snapshot: "bus.tar", NewImage: "i", NewProgram: "/p", NewEntrypoint: "/e"}, {Snapshot: "/a:b/bus.tar", NewImage: "i", NewProgram: "/p", NewEntrypoint: "/e"}} { - if _, err := r.Run(context.Background(), bad); err == nil { - t.Errorf("%+v was accepted", bad) - } - } - if len(f.calls) != 0 { - t.Errorf("docker ran: %v", f.calls) - } -} - -func TestTheControllersWordForTheBusIsReadOrRefused(t *testing.T) { - for answer, want := range map[string]bool{ - `the bus is nats, built abc, on novox; never rolled out`: false, - "the bus is nats\nlast step 4: a → b on novox by x (y): running since 2026-10-22 12:00; snapshot s": true, - "last step 4: a → b on novox by x (y): done at 2026-10-22 12:00; snapshot s": false, - } { - open, _, err := upgradeRunning(answer) - if err != nil || open != want { - t.Errorf("%q: open %v, err %v", answer, open, err) - } - } - if _, _, err := upgradeRunning("{}"); err == nil { - t.Error("an answer it cannot read was taken for a closed upgrade") - } -} - -func contains(calls []string, want string) bool { - for _, c := range calls { - if c == want { - return true - } - } - return false -} diff --git a/modules/nats/module.json b/modules/nats/module.json index ca172c2f..b3bf5771 100644 --- a/modules/nats/module.json +++ b/modules/nats/module.json @@ -44,8 +44,7 @@ "nats_backlog", "nats_buckets", "nats_users", - "nats_user_can", - "nats_rehearse" + "nats_user_can" ], "guards": [ 8222 @@ -186,10 +185,7 @@ "env": { "MESH_NATS_MONITOR": "http://127.0.0.1:8222", "MESH_NATS_CONTAINER": "mesh-broker-nats", - "MESH_NATS_USERS_FILE": "/etc/nats/accounts.conf", - "MESH_NATS_SNAPSHOT": "${dir:snapshots}/bus.tar", - "MESH_NATS_PROGRAM_DIR": "/var/lib/nats-module/snapshot", - "MESH_NATS_ENTRYPOINT_DIR": "/var/lib/nats-module/entrypoint" + "MESH_NATS_USERS_FILE": "/etc/nats/accounts.conf" } }, { From 91877e3421d5d71f379838cedadc0f2b33bfa886 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 22 Oct 2026 10:04:31 +0200 Subject: [PATCH 6/9] Put the bus's two mounted programs under a directory the module declares, as the controller's module check requires (review round 1, C1) --- modules/nats/README.md | 6 +++--- modules/nats/cmd/nats-tools/manifest_test.go | 5 +++++ modules/nats/module.json | 15 +++++++++++---- 3 files changed, 19 insertions(+), 7 deletions(-) diff --git a/modules/nats/README.md b/modules/nats/README.md index 5a5fe68e..3100d5c6 100644 --- a/modules/nats/README.md +++ b/modules/nats/README.md @@ -59,7 +59,7 @@ live one**, which a person swaps in. image=$(docker inspect -f '{{.Config.Image}}' mesh-broker-nats) docker run --rm -v :/in:ro -v :/out \ - -v /var/lib/nats-module/snapshot:/opt/mesh:ro \ + -v /snapshot:/opt/mesh:ro \ --entrypoint /opt/mesh/mesh-nats-snapshot "$image" restore --into /out --from /in/bus.tar It starts a server of its own on the container's loopback with the bus's one account (`MESH`), @@ -129,5 +129,5 @@ Each directory or archive this module declares says how it treats a file it did - `/var/lib/nats-module/conf`: everything in it is the program's; the mesh moves nothing out (ADR 0422 §2). - `${dir:snapshots}`: everything in it is the program's; the mesh moves nothing out (ADR 0422 §2). - `${dir:mesh-state}`: everything in it is the program's; the mesh moves nothing out (ADR 0422 §2). -- `/var/lib/nats-module/snapshot`: the unpacked snapshot program; everything in it is the program's; the mesh moves nothing out (ADR 0422 §2). -- `/var/lib/nats-module/entrypoint`: the unpacked entrypoint; everything in it is the program's; the mesh moves nothing out (ADR 0422 §2). +- `${dir:state}/snapshot`: the unpacked snapshot program; everything in it is the program's; the mesh moves nothing out (ADR 0422 §2). +- `${dir:state}/entrypoint`: the unpacked entrypoint; everything in it is the program's; the mesh moves nothing out (ADR 0422 §2). diff --git a/modules/nats/cmd/nats-tools/manifest_test.go b/modules/nats/cmd/nats-tools/manifest_test.go index 5420fca2..b9d699d0 100644 --- a/modules/nats/cmd/nats-tools/manifest_test.go +++ b/modules/nats/cmd/nats-tools/manifest_test.go @@ -81,6 +81,11 @@ func TestTheBussManifestMountsItsProgramsIntoTheUpstreamImage(t *testing.T) { t.Errorf("%s is %v: an archive resource unpacking %s", p.resource, r, p.artifact) continue } + // Under a directory the module declares (ADR 0030: nothing is mounted from a path no resource covers). + if dir, _, _ := strings.Cut(strings.TrimPrefix(r["path"].(string), "${dir:"), "}"); !strings.HasPrefix(r["path"].(string), "${dir:") || + byID[dir] == nil || byID[dir]["type"] != "directory" { + t.Errorf("%s is at %v: not under a directory the module declares", p.resource, r["path"]) + } mounted := false for _, v := range volumes { if v == r["path"].(string)+p.mount { diff --git a/modules/nats/module.json b/modules/nats/module.json index b3bf5771..fe27a566 100644 --- a/modules/nats/module.json +++ b/modules/nats/module.json @@ -105,17 +105,24 @@ "template": "files/nats.conf", "mode": "0644" }, + { + "id": "state", + "type": "directory", + "mode": "0700", + "place": ".", + "unknown": "program" + }, { "id": "program-snapshot", "type": "archive", - "path": "/var/lib/nats-module/snapshot", + "path": "${dir:state}/snapshot", "artifact": "snapshot", "unknown": "program" }, { "id": "program-entrypoint", "type": "archive", - "path": "/var/lib/nats-module/entrypoint", + "path": "${dir:state}/entrypoint", "artifact": "entrypoint", "unknown": "program" }, @@ -131,8 +138,8 @@ "/var/lib/mesh-broker-nats:/data", "/var/lib/nats-module/conf:/etc/nats:ro", "${access:tls}:/tls:ro", - "/var/lib/nats-module/snapshot:/opt/mesh:ro", - "/var/lib/nats-module/entrypoint:/opt/mesh-entrypoint:ro" + "${dir:state}/snapshot:/opt/mesh:ro", + "${dir:state}/entrypoint:/opt/mesh-entrypoint:ro" ], "args": [ "sh", From c948f3a61c74257e6e3147effb396d9d6434f491 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 22 Oct 2026 10:18:15 +0200 Subject: [PATCH 7/9] Skip the bus's docker tests loudly where there is no docker, and run them by name in a docker step of the merge check where there is (ADR 0480 clause 6) --- merge-check.sh | 13 +++++++++++++ modules/nats/cmd/mesh-nats-snapshot/live_test.go | 8 +++++++- 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/merge-check.sh b/merge-check.sh index d3b7d97c..772823ff 100755 --- a/merge-check.sh +++ b/merge-check.sh @@ -192,6 +192,19 @@ for m in $touched; do # another process) run here and nowhere else: the unit suite stays under a second (novox/hq playbook 07). (cd "modules/$m" && CGO_ENABLED=$cgo GOPRIVATE=git.novox.be go vet -tags integration ${modfile:+"-modfile=$modfile"} ./... && CGO_ENABLED=$cgo GOPRIVATE=git.novox.be go test -tags integration ${modfile:+"-modfile=$modfile"} $race -count=1 ./...) + # The docker tests of the bus's snapshot program and entrypoint (novox/hq ADR 0480 clause 6) start throwaway nats + # containers, and the Go toolchain container holds no docker, so above they skip with a "NOT RUN" line. Where the + # build seat says docker exists (MESH_CHECK_DOCKER=1) they run here by name, and a missing docker fails the check + # (MESH_REQUIRE_DOCKER=1 makes the tests fail instead of skip). Without it the step says it did not run. + if [ "$m" = nats ]; then + if [ "${MESH_CHECK_DOCKER:-}" = 1 ]; then + (cd modules/nats && MESH_REQUIRE_DOCKER=1 CGO_ENABLED=$cgo GOPRIVATE=git.novox.be go test -tags integration ${modfile:+"-modfile=$modfile"} -count=1 \ + -run 'TestTheDeclaredDumpRunsInTheBussImage|TestTheEntrypointReloadsTheServerWithoutDroppingAConnection|TestTheStoreSurvivesTheNewDeclarationTheOldAndTheNewAgain|TestASnapshotOfALiveBusRestoresToTheSameContent|TestTheSnapshotUserCannotChangeTheBus' \ + ./cmd/mesh-nats-snapshot) || { echo "the bus's docker tests failed"; exit 1; } + else + echo "NOT RUN: the bus's five docker tests (modules/nats/cmd/mesh-nats-snapshot) skipped: this check has no docker (MESH_CHECK_DOCKER is not 1)" + fi + fi done if [ -n "$untestable" ]; then echo "the check fails: the tests of$untestable could not run (novox/hq issue 385)" diff --git a/modules/nats/cmd/mesh-nats-snapshot/live_test.go b/modules/nats/cmd/mesh-nats-snapshot/live_test.go index 95e8ea7d..3c173340 100644 --- a/modules/nats/cmd/mesh-nats-snapshot/live_test.go +++ b/modules/nats/cmd/mesh-nats-snapshot/live_test.go @@ -60,7 +60,13 @@ var snapshotGrants = struct{ publish, subscribe []string }{ func needDocker(t *testing.T) { t.Helper() if _, err := exec.LookPath("docker"); err != nil { - t.Fatalf("these tests start throwaway nats containers and need docker on the machine that runs them: %v", err) + // The Go toolchain container of the merge check holds no docker. Where docker is expected + // (MESH_REQUIRE_DOCKER=1, which merge-check.sh sets for its docker step) its absence is a failure. + if os.Getenv("MESH_REQUIRE_DOCKER") == "1" { + t.Fatalf("MESH_REQUIRE_DOCKER=1 and there is no docker here: %v", err) + } + t.Skipf("NOT RUN: this test starts a throwaway nats container and there is no docker in this toolchain (%v); "+ + "it is proof only where merge-check.sh's docker step runs it (MESH_CHECK_DOCKER=1)", err) } } From cff5ed46b7940d12f75cae18968b3d6b9926622e Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 22 Oct 2026 11:44:45 +0200 Subject: [PATCH 8/9] Check an image's Dockerfile place instead of refusing the image kind (ADR 0487 clause 13) --- checks/images/go.mod | 3 - checks/images/main.go | 80 ----------------------- checks/images/main_test.go | 51 --------------- merge-check.sh | 2 - modules/mailu/cmd/automx/manifest_test.go | 7 +- 5 files changed, 4 insertions(+), 139 deletions(-) delete mode 100644 checks/images/go.mod delete mode 100644 checks/images/main.go delete mode 100644 checks/images/main_test.go diff --git a/checks/images/go.mod b/checks/images/go.mod deleted file mode 100644 index 1137f5f0..00000000 --- a/checks/images/go.mod +++ /dev/null @@ -1,3 +0,0 @@ -module git.novox.be/novox/mesh-catalog/checks/images - -go 1.22 diff --git a/checks/images/main.go b/checks/images/main.go deleted file mode 100644 index ddd408a1..00000000 --- a/checks/images/main.go +++ /dev/null @@ -1,80 +0,0 @@ -// Command images refuses any manifest but build-agent's declaring an artifact of kind `image`, an image built from a -// Dockerfile (novox/hq ADR 0452 clause 1, ADR 0480 clause 11 as ADR 0487 clause 13 changed it). From R4d a module -// runs a pinned upstream image with its programs mounted in; build-agent is the one builder that still builds an -// image until R4g, when the catalogue refuses the kind outright (issue 1619) and this check goes with it. -// -// go run . -// -// Exits 1 on a finding. -package main - -import ( - "encoding/json" - "fmt" - "os" - "path/filepath" - "sort" -) - -// Exempt is the one module still allowed to declare the kind, until R4g. -var Exempt = map[string]bool{"build-agent": true} - -// Finding is one refusal: Module's artifact Artifact is an image built from a Dockerfile. -type Finding struct{ Module, Artifact string } - -// Check reads every manifest under root/modules and answers what it refuses. -func Check(root string) ([]Finding, error) { - manifests, err := filepath.Glob(filepath.Join(root, "modules", "*", "module.json")) - if err != nil { - return nil, err - } - sort.Strings(manifests) - var out []Finding - for _, path := range manifests { - raw, err := os.ReadFile(path) - if err != nil { - return nil, err - } - var m struct { - Module string `json:"module"` - Build struct { - Artifacts []struct { - Name string `json:"name"` - Kind string `json:"kind"` - } `json:"artifacts"` - } `json:"build"` - } - if err := json.Unmarshal(raw, &m); err != nil { - return nil, fmt.Errorf("%s: %w", path, err) - } - if Exempt[m.Module] { - continue - } - for _, a := range m.Build.Artifacts { - if a.Kind == "image" { - out = append(out, Finding{Module: m.Module, Artifact: a.Name}) - } - } - } - return out, nil -} - -func main() { - root := "." - if len(os.Args) > 1 { - root = os.Args[1] - } - found, err := Check(root) - if err != nil { - fmt.Fprintln(os.Stderr, err) - os.Exit(1) - } - for _, f := range found { - fmt.Printf("%s declares the artifact %s as an image built from a Dockerfile: run a pinned upstream image with the "+ - "module's programs mounted in (novox/hq ADR 0452 clause 1, ADR 0480 clause 11)\n", f.Module, f.Artifact) - } - if len(found) > 0 { - os.Exit(1) - } - fmt.Println("images: no module but build-agent builds an image from a Dockerfile") -} diff --git a/checks/images/main_test.go b/checks/images/main_test.go deleted file mode 100644 index 677c6009..00000000 --- a/checks/images/main_test.go +++ /dev/null @@ -1,51 +0,0 @@ -package main - -import ( - "os" - "path/filepath" - "reflect" - "testing" -) - -func catalogue(t *testing.T, manifests map[string]string) string { - t.Helper() - root := t.TempDir() - for name, body := range manifests { - dir := filepath.Join(root, "modules", name) - if err := os.MkdirAll(dir, 0o755); err != nil { - t.Fatal(err) - } - if err := os.WriteFile(filepath.Join(dir, "module.json"), []byte(body), 0o644); err != nil { - t.Fatal(err) - } - } - return root -} - -// A fixture manifest with an image artifact is refused, the builder's is not, and an upstream image is no image. -func TestOnlyBuildAgentMayDeclareAnImageBuiltFromADockerfile(t *testing.T) { - root := catalogue(t, map[string]string{ - "old": `{"module":"old","build":{"artifacts":[{"name":"server","kind":"image","from":"Dockerfile"}]}}`, - "build-agent": `{"module":"build-agent","build":{"artifacts":[{"name":"agent","kind":"image","from":"Dockerfile"}]}}`, - "new": `{"module":"new","build":{"artifacts":[{"name":"server","kind":"upstream","from":"alpine@sha256:00"},{"name":"p","kind":"bundle"}]}}`, - "none": `{"module":"none"}`, - }) - got, err := Check(root) - if err != nil { - t.Fatal(err) - } - if want := []Finding{{Module: "old", Artifact: "server"}}; !reflect.DeepEqual(got, want) { - t.Errorf("found %v, want %v", got, want) - } -} - -// The catalogue this check lives in passes it. -func TestTheCatalogueDeclaresNoImage(t *testing.T) { - got, err := Check("../..") - if err != nil { - t.Fatal(err) - } - if len(got) > 0 { - t.Errorf("the catalogue declares %v", got) - } -} diff --git a/merge-check.sh b/merge-check.sh index ab97f138..fb1d9b8b 100755 --- a/merge-check.sh +++ b/merge-check.sh @@ -57,8 +57,6 @@ set -eu (cd checks/words && go test -count=1 . && go run . ../..) # The toolchains (novox/hq ADR 0477 clause 4): the package go only in modules/go, nodejs and npm only in modules/nodejs. (cd checks/toolchains && go test -count=1 . && go run . ../..) -# The image (novox/hq ADR 0480 clause 11): no manifest but build-agent's declares an image built from a Dockerfile. -(cd checks/images && go test -count=1 . && go run . ../..) # The image builder (novox/hq ADR 0487 §3): only modules/buildah provides rootless-image-build, docker never. (cd checks/imagebuild && go test -count=1 . && go run . ../..) # The image's place (novox/hq ADR 0487 §1): every image builds from a Dockerfile in its own module's directory. diff --git a/modules/mailu/cmd/automx/manifest_test.go b/modules/mailu/cmd/automx/manifest_test.go index cf88b181..057c3b1b 100644 --- a/modules/mailu/cmd/automx/manifest_test.go +++ b/modules/mailu/cmd/automx/manifest_test.go @@ -107,8 +107,9 @@ func TestAutomxIsAGoBundleForAlpineMountedAtOptMeshAndRestartedWithIt(t *testing } } -// Clause 3: an upstream artifact pinned by digest, the one route-proxy's trust artifact runs; never an image -// built from a Dockerfile (clause 11), and no Dockerfile or base to build one on. +// Clause 3: an upstream artifact pinned by digest, the one route-proxy's trust artifact runs. automx's own design +// builds no image from a Dockerfile and has no Dockerfile or base to build one on; the catalogue as such does not +// refuse the image kind (novox/hq ADR 0487 clause 13). func TestAutomxRunsTheMirroredAlpineImageAndBuildsNoImage(t *testing.T) { m := readManifest(t) c := m.resource("automx") @@ -121,7 +122,7 @@ func TestAutomxRunsTheMirroredAlpineImageAndBuildsNoImage(t *testing.T) { } for _, a := range m.Build["artifacts"].([]any) { if a.(map[string]any)["kind"] == "image" { - t.Errorf("artifact %v is an image built from a Dockerfile (ADR 0452)", a) + t.Errorf("artifact %v is an image built from a Dockerfile, which automx's design (ADR 0480) does not use", a) } } if _, has := m.Build["on"]; has { From 68183ee4413d756f25b15bdca3b8f4c51e471dd6 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 22 Oct 2026 11:47:44 +0200 Subject: [PATCH 9/9] Keep n8n on its own Dockerfile: since ADR 0487 a module may ship one, and the move to the upstream image only added a config value and a uuid mount (the operator, 2026-10-22) --- modules/n8n/Dockerfile | 20 ++++++++++++++++++++ modules/n8n/README.md | 7 +------ modules/n8n/module.json | 41 ++++++++--------------------------------- 3 files changed, 29 insertions(+), 39 deletions(-) create mode 100644 modules/n8n/Dockerfile diff --git a/modules/n8n/Dockerfile b/modules/n8n/Dockerfile new file mode 100644 index 00000000..7d15a2bf --- /dev/null +++ b/modules/n8n/Dockerfile @@ -0,0 +1,20 @@ +# n8n with what its workflows reach for beyond the upstream image. +# +# The base is named, not pinned here (novox/hq issue 044): module.json's `build.on` declares N8N_BASE +# as the upstream image by digest, and the mesh hands the build its own copy (ADR 0097). +ARG N8N_BASE +FROM ${N8N_BASE} + +USER root +# - `media` (GID 2000), with `node` in it: the shared media library is group-writable by the +# operator's media group, and a workflow files downloads into it. A container resource cannot add +# a supplementary group, so the image's own /etc/group carries it. 2000 is the operator's media +# group today; novox/hq 153 proposes reading it from the accessed data (${access::gid}). +# - uuid, pinned to the version the workflows were written against: Code nodes require() it +# (NODE_FUNCTION_ALLOW_EXTERNAL=*), and a Code node can only require what is installed. +RUN apk add --no-cache shadow \ + && groupadd -g 2000 media \ + && usermod -aG media node \ + && npm install -g uuid@14.0.1 + +USER node diff --git a/modules/n8n/README.md b/modules/n8n/README.md index 751b5f70..c911c864 100644 --- a/modules/n8n/README.md +++ b/modules/n8n/README.md @@ -1,10 +1,6 @@ # n8n -The `n8n` module of the mesh catalogue: runs the upstream image `n8nio/n8n` by digest with the `uuid` package mounted into it (ADR 0480 clauses 1, 3, 7 and 8); runs the container image `redis`; runs the container image `selenium/standalone-chrome`. - -- The `server` container no longer comes from a Dockerfile. Its program is the upstream image's, bit for bit the one the module built on before (digest `4846eb2f...`; the running container reports release 1.71.3). -- `uuid` 14.0.1, which Code nodes `require`, is fetched once by its pinned sha256 from the package registry and mounted read-only at `/usr/local/lib/node_modules/uuid` (where `npm install -g` put it in the image the module built). -- The server joins the media library's group as a supplementary group, `groups` from the config key `media-gid`; the key has no default, so a machine that does not say it refuses n8n. +The `n8n` module of the mesh catalogue: runs the container image `redis`; runs the container image `selenium/standalone-chrome`. ## Files it does not know @@ -12,5 +8,4 @@ Each directory or archive this module declares says how it treats a file it did - `${dir:state}`: everything in it is the program's; the mesh moves nothing out (ADR 0422 §2). - `${dir:data}`: everything in it is the program's; the mesh moves nothing out (ADR 0422 §2). -- `${dir:state}/uuid`: everything in it is the program's; the mesh moves nothing out (ADR 0422 §2). - `${dir:cache}`: everything in it is the program's; the mesh moves nothing out (ADR 0422 §2). diff --git a/modules/n8n/module.json b/modules/n8n/module.json index 04bec8ff..975247fa 100644 --- a/modules/n8n/module.json +++ b/modules/n8n/module.json @@ -30,15 +30,6 @@ "mode": "read-write" } ], - "config": { - "media-gid": { - "type": "number", - "description": "The number of the group that owns the media library on this machine, which the server joins as a supplementary group so a workflow can file into the library (novox/hq ADR 0480 clause 7). Set per machine; until the container takes it from the media library itself (issue 1618) nothing writes a default, so a machine that does not say it refuses n8n by name.", - "minimum": 1, - "required": true, - "trusted": true - } - }, "listens": [ { "name": "web", @@ -101,13 +92,6 @@ "mode": "0600", "template": "files/server.env" }, - { - "id": "uuid", - "type": "archive", - "path": "${dir:state}/uuid", - "artifact": "uuid", - "unknown": "program" - }, { "id": "net", "type": "network", @@ -135,11 +119,7 @@ "volumes": [ "${dir:data}:/home/node/.n8n", "${dir:state}/n8n-database.secret:/run/secrets/database:ro", - "${dir:state}/uuid:/usr/local/lib/node_modules/uuid:ro", "${access:media}:/media-library" - ], - "groups": [ - "${config:media-gid}" ] }, { @@ -171,22 +151,17 @@ } ], "build": { + "on": [ + { + "arg": "N8N_BASE", + "image": "n8nio/n8n@sha256:4846eb2f4b874ab04cde7fc1e249d2ddaec66e9aea64439beb2972cfea88e3c0" + } + ], "artifacts": [ { "name": "server", - "kind": "upstream", - "from": "n8nio/n8n@sha256:4846eb2f4b874ab04cde7fc1e249d2ddaec66e9aea64439beb2972cfea88e3c0" - }, - { - "name": "uuid", - "kind": "archive", - "url": "https://registry.npmjs.org/uuid/-/uuid-14.0.1.tgz", - "sha256": "e062c57ed120ea135478f305da04802bf69eab23fbc0ca2bae6989b6855390c4", - "version": "14.0.1", - "within": "package", - "names-on-purpose": { - "registry.npmjs.org": "the npm registry publishes the uuid package there; the build seat fetches it once, by its pinned sha256 (novox/hq ADR 0316 §2, ADR 0480 §8)" - } + "kind": "image", + "from": "Dockerfile" }, { "name": "files",