From e6d5ead50dfd681813aee4216df1e2f74990e271 Mon Sep 17 00:00:00 2001 From: jochen Date: Sat, 10 Oct 2026 16:00:52 +0200 Subject: [PATCH] nats: a WebSocket listener for a page in a browser, from the overlay only (hq research 036) The bus has no way for a browser to reach it (research 036, gap G1). The server now also listens over WebSocket on 4223: published from the mesh like 4222, so the machine's filter admits it from the overlay and nothing else; no TLS, acceptable only because the overlay is WireGuard and a browser cannot be made to trust the mesh's own certificate; compression on. The user a page connects as is the read-only view the controller composes (mesh-controller, `bus view-credential`). A test parses the declared configuration with the server itself, holds the opening, the published port and the listener to one port, and connects a client over WebSocket. The bus module's policy is `record`: this reaches the bus's machine only through the planned step a person starts (`bus upgrade`), as every bus change does. --- modules/nats/README.md | 11 + modules/nats/cmd/nats-tools/websocket_test.go | 190 ++++++++++++++++++ modules/nats/module.json | 10 +- 3 files changed, 210 insertions(+), 1 deletion(-) create mode 100644 modules/nats/cmd/nats-tools/websocket_test.go diff --git a/modules/nats/README.md b/modules/nats/README.md index f07245faa..dcfd95373 100644 --- a/modules/nats/README.md +++ b/modules/nats/README.md @@ -4,6 +4,17 @@ The bus server (novox/hq design 25), the configuration the module owns (`nats.co JetStream's store) and the user list the controller composes (`accounts.conf`), reloaded in place by the entrypoint. Its tools (`cmd/nats-tools`) are served by the node's runtime and only read. +## Its ports + +`4222` is the bus: TLS, the mesh's own certificate, every link the mesh has. `4223` is the same server over +WebSocket, for a client in a browser (nats.ws): the issue tracker's view page, connecting directly as the +read-only user the controller composes for it (`bus view-credential`; novox/hq research 036). Both are +published from the mesh only — the machine's filter admits them from the overlay and nothing else — and the +WebSocket listener has no TLS *because* of that: the overlay is WireGuard, so every byte reaching it crossed +the tunnel, and a browser cannot be made to trust the mesh's own certificate. The user still authenticates +with the password the mesh minted, and is granted reads and nothing else. `8222` is the server's monitoring, +on the machine's loopback alone. + ## Its data, and how it is backed up Everything the bus keeps is in JetStream: every stream, and every key-value bucket (a stream named diff --git a/modules/nats/cmd/nats-tools/websocket_test.go b/modules/nats/cmd/nats-tools/websocket_test.go new file mode 100644 index 000000000..a1b92d74a --- /dev/null +++ b/modules/nats/cmd/nats-tools/websocket_test.go @@ -0,0 +1,190 @@ +package main + +import ( + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/x509" + "crypto/x509/pkix" + "encoding/json" + "encoding/pem" + "math/big" + "net" + "os" + "path/filepath" + "strconv" + "strings" + "testing" + "time" + + "github.com/nats-io/nats-server/v2/server" + "github.com/nats-io/nats.go" +) + +// The WebSocket listener the manifest declares (novox/hq research 036, gap G1), read by the server the +// image runs: the module's own configuration, as declared, parsed by nats-server itself, with its TLS +// paths and store pointed at this test's directories and a user list of its own beside it — and a +// client connected over WebSocket to it, through the port the manifest opens from the mesh and the +// container publishes. +func TestTheDeclaredConfigurationListensForWebSocketFromTheMesh(t *testing.T) { + m := manifest(t) + + // The three places the listener is declared agree on one port: the server's own configuration, the + // opening from the mesh (the machine's filter admits it from the overlay only), and the container's + // published port. + var opening map[string]any + for _, l := range m["listens"].([]any) { + if entry := l.(map[string]any); entry["name"] == "bus-websocket" { + opening = entry + } + } + if opening == nil { + t.Fatal("the manifest opens no bus-websocket port") + } + if opening["from"] != "mesh" { + t.Fatalf("the WebSocket listener is open from %q; it is open from the mesh only, because it has no TLS", opening["from"]) + } + port := int(opening["port"].(float64)) + published := false + for _, r := range m["resources"].([]any) { + if res := r.(map[string]any); res["id"] == "server" { + for _, p := range res["ports"].([]any) { + published = published || p.(string) == itoa(port)+":"+itoa(port) + } + } + } + if !published { + t.Fatalf("the container does not publish %d:%d", port, port) + } + + dir := t.TempDir() + conf := serverConfOf(t, m) + conf = strings.ReplaceAll(conf, "/tls/", selfSignedInto(t, dir)+"/") + conf = strings.ReplaceAll(conf, `store_dir: "/data"`, `store_dir: "`+filepath.Join(dir, "data")+`"`) + if err := os.WriteFile(filepath.Join(dir, "nats.conf"), []byte(conf), 0o600); err != nil { + t.Fatal(err) + } + // A user list of this test's own, in the place the mesh writes it. + users := "accounts { MESH { jetstream: enabled\n users = [ { user: \"view\", password: \"view\", permissions: {\n" + + " publish: { allow: [\"probe\"] }\n subscribe: { allow: [\"probe\", \"_INBOX.view.>\"] } } } ]\n} }\n" + if err := os.WriteFile(filepath.Join(dir, "accounts.conf"), []byte(users), 0o600); err != nil { + t.Fatal(err) + } + opts, err := server.ProcessConfigFile(filepath.Join(dir, "nats.conf")) + if err != nil { + t.Fatalf("the server refuses the module's own configuration: %v", err) + } + if opts.Websocket.Port != port { + t.Fatalf("the server's WebSocket listener is on %d and the opening from the mesh is %d", opts.Websocket.Port, port) + } + if !opts.Websocket.NoTLS { + t.Fatal("the WebSocket listener wants TLS: a browser cannot be made to trust the mesh's own certificate, and the overlay is the tunnel") + } + if !opts.Websocket.Compression { + t.Fatal("the WebSocket listener does not compress") + } + if opts.Websocket.Host != "0.0.0.0" { + t.Fatalf("inside the container the listener binds %q; it binds every interface, and the machine publishes it from the mesh", opts.Websocket.Host) + } + + // Run it, on ports the system chooses, so this runs beside a live bus. + opts.Host, opts.Port = "127.0.0.1", -1 + opts.HTTPHost, opts.HTTPPort = "127.0.0.1", 0 + opts.Websocket.Host, opts.Websocket.Port = "127.0.0.1", -1 + opts.NoLog, opts.NoSigs = true, true + s, err := server.NewServer(opts) + if err != nil { + t.Fatal(err) + } + go s.Start() + if !s.ReadyForConnections(10 * time.Second) { + t.Fatal("the server did not start") + } + defer s.Shutdown() + + nc, err := nats.Connect(s.WebsocketURL(), nats.UserInfo("view", "view"), nats.CustomInboxPrefix("_INBOX.view"), nats.Compression(true)) + if err != nil { + t.Fatalf("a WebSocket client could not connect to %s: %v", s.WebsocketURL(), err) + } + defer nc.Close() + if !strings.HasPrefix(nc.ConnectedUrl(), "ws://") { + t.Fatalf("connected to %s, not over WebSocket", nc.ConnectedUrl()) + } + sub, err := nc.SubscribeSync("probe") + if err != nil { + t.Fatal(err) + } + if err := nc.Publish("probe", []byte("round trip")); err != nil { + t.Fatal(err) + } + if msg, err := sub.NextMsg(5 * time.Second); err != nil || string(msg.Data) != "round trip" { + t.Fatalf("no round trip over WebSocket: %v", err) + } + // And the client port beside it still requires TLS: a plain connection is refused. + if _, err := nats.Connect(s.ClientURL(), nats.UserInfo("view", "view"), nats.Timeout(2*time.Second)); err == nil { + t.Fatal("the client port accepted a connection without TLS") + } +} + +func itoa(n int) string { return strconv.Itoa(n) } + +// manifest is the nats module's own definition, beside this program. +func manifest(t *testing.T) map[string]any { + t.Helper() + raw, err := os.ReadFile("../../module.json") + if err != nil { + t.Fatal(err) + } + var m map[string]any + if err := json.Unmarshal(raw, &m); err != nil { + t.Fatal(err) + } + return m +} + +func serverConfOf(t *testing.T, m map[string]any) string { + t.Helper() + for _, r := range m["resources"].([]any) { + if res := r.(map[string]any); res["id"] == "server-conf" { + return res["content"].(string) + } + } + t.Fatal("the nats module declares no server configuration") + return "" +} + +// selfSignedInto writes a throwaway certificate and key as tls.crt and tls.key into a directory, as the +// mesh mounts the broker's at /tls, and returns that directory. +func selfSignedInto(t *testing.T, dir string) string { + t.Helper() + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + t.Fatal(err) + } + template := &x509.Certificate{ + SerialNumber: big.NewInt(1), Subject: pkix.Name{CommonName: "test bus"}, + NotBefore: time.Now().Add(-time.Hour), NotAfter: time.Now().Add(time.Hour), + KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + IPAddresses: []net.IP{net.ParseIP("127.0.0.1")}, + } + der, err := x509.CreateCertificate(rand.Reader, template, template, &key.PublicKey, key) + if err != nil { + t.Fatal(err) + } + raw, err := x509.MarshalECPrivateKey(key) + if err != nil { + t.Fatal(err) + } + tlsDir := filepath.Join(dir, "tls") + if err := os.MkdirAll(tlsDir, 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(tlsDir, "tls.crt"), pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}), 0o600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(tlsDir, "tls.key"), pem.EncodeToMemory(&pem.Block{Type: "EC PRIVATE KEY", Bytes: raw}), 0o600); err != nil { + t.Fatal(err) + } + return tlsDir +} diff --git a/modules/nats/module.json b/modules/nats/module.json index d8647b3ef..fd15e86d5 100644 --- a/modules/nats/module.json +++ b/modules/nats/module.json @@ -33,6 +33,13 @@ "protocol": "tcp", "from": "mesh", "why": "the mesh bus — every link the mesh has, over TLS, reached across the overlay" + }, + { + "name": "bus-websocket", + "port": 4223, + "protocol": "tcp", + "from": "mesh", + "why": "the bus over WebSocket, for a page in a browser on the overlay — the view's read-only user, no TLS because the overlay is WireGuard (research 036)" } ], "tools": [ @@ -98,7 +105,7 @@ "id": "server-conf", "type": "file", "path": "/var/lib/nats-module/conf/nats.conf", - "content": "# The nats module's own server settings. Declared by the module, because a port, a TLS path\n# and a store directory are properties of the container this module raises: they live in its\n# image and its mounts and change when it does.\n#\n# The mesh writes accounts.conf beside this one and nothing else. A controller that wrote the\n# whole file would have to be kept in step with a Dockerfile it never sees.\n\nport: 4222\n# Monitoring on every interface *inside* the container, so the publish below can reach it; the publish\n# is 127.0.0.1:8222 on the machine, so nothing beyond the machine reaches it, and the module guards 8222\n# too. Bound to the container's own loopback until 2026-10-04, the published port answered nothing\n# (connection reset), and the only way in was `docker exec`.\nhttp: 0.0.0.0:8222\n\n# The mesh's own broker certificate — the one every machine already pins by fingerprint and the\n# controller already trusts (MESH_BROKER_CERTIFICATE). Serving the new bus with it means no\n# machine's pin changes when it moves, and no second certificate exists to be wrong about.\ntls {\n cert_file: \"/tls/tls.crt\"\n key_file: \"/tls/tls.key\"\n}\n\n# **No `verify`, deliberately, and it was `verify: true` until a probe ran this image.** That\n# setting makes the server demand a *client* certificate, and nothing in the mesh presents one: a\n# host pins this server's exact certificate and authenticates with the password the mesh minted\n# (novox/hq ADR 0004, design 25 §4), and so does a module's runtime. With it on, every connection\n# in the mesh is refused at the TLS handshake, before any password is looked at — and the error is\n# \"client didn't provide a certificate\", which reads as a client fault.\n#\n# TLS is still required: a tls block is what makes it required, and verify only decides whether\n# client certificates are checked. What is given up is a second factor the mesh has no machinery\n# to issue or rotate — a certificate per module per node — and what is kept is stronger than a\n# name check in both directions: an exact pin outward, a per-user password inward.\n\njetstream {\n store_dir: \"/data\"\n}\n\n# Every user of the mesh, composed by the controller and rewritten whenever a module is\n# assigned, a node enrols or a person's access changes.\n#\n# **Relative, and in this same directory, because it has to be.** An absolute include path is\n# resolved relative to the including file's directory, not from the root: nats-server given\n# `include /etc/nats/accounts.conf` from /etc/nats-server/nats.conf looks for\n# /etc/nats-server/etc/nats/accounts.conf and refuses to start. Verified against the server.\ninclude accounts.conf\n", + "content": "# The nats module's own server settings. Declared by the module, because a port, a TLS path\n# and a store directory are properties of the container this module raises: they live in its\n# image and its mounts and change when it does.\n#\n# The mesh writes accounts.conf beside this one and nothing else. A controller that wrote the\n# whole file would have to be kept in step with a Dockerfile it never sees.\n\nport: 4222\n# Monitoring on every interface *inside* the container, so the publish below can reach it; the publish\n# is 127.0.0.1:8222 on the machine, so nothing beyond the machine reaches it, and the module guards 8222\n# too. Bound to the container's own loopback until 2026-10-04, the published port answered nothing\n# (connection reset), and the only way in was `docker exec`.\nhttp: 0.0.0.0:8222\n\n# The mesh's own broker certificate — the one every machine already pins by fingerprint and the\n# controller already trusts (MESH_BROKER_CERTIFICATE). Serving the new bus with it means no\n# machine's pin changes when it moves, and no second certificate exists to be wrong about.\ntls {\n cert_file: \"/tls/tls.crt\"\n key_file: \"/tls/tls.key\"\n}\n\n# **No `verify`, deliberately, and it was `verify: true` until a probe ran this image.** That\n# setting makes the server demand a *client* certificate, and nothing in the mesh presents one: a\n# host pins this server's exact certificate and authenticates with the password the mesh minted\n# (novox/hq ADR 0004, design 25 §4), and so does a module's runtime. With it on, every connection\n# in the mesh is refused at the TLS handshake, before any password is looked at — and the error is\n# \"client didn't provide a certificate\", which reads as a client fault.\n#\n# TLS is still required: a tls block is what makes it required, and verify only decides whether\n# client certificates are checked. What is given up is a second factor the mesh has no machinery\n# to issue or rotate — a certificate per module per node — and what is kept is stronger than a\n# name check in both directions: an exact pin outward, a per-user password inward.\n\n# A WebSocket listener, for a client in a browser (nats.ws; novox/hq research 036, gap G1): the issue\n# tracker's view page watches the bus directly, as the read-only user the controller composes for it\n# (`bus view-credential`). Published from the mesh like the bus port (`listens`, from mesh): the machine's\n# filter admits it from the overlay only, and nothing off the overlay reaches it.\n#\n# **`no_tls`, and only because the overlay is WireGuard.** Every byte that reaches this port has crossed the\n# tunnel, encrypted and authenticated by the peers' keys before it was on the wire; the listener adds no\n# second layer, because a browser would have to be made to trust the mesh's own certificate for it, and off\n# the overlay the port is closed. The user still authenticates: the password the mesh minted, checked by the\n# server, with a permission list that grants it reads and nothing else. The client port beside it keeps its\n# TLS; nothing of the mesh moves here. Compression is on: a page watching a bucket reads JSON.\nwebsocket {\n listen: \"0.0.0.0:4223\"\n no_tls: true\n compression: true\n}\n\njetstream {\n store_dir: \"/data\"\n}\n\n# Every user of the mesh, composed by the controller and rewritten whenever a module is\n# assigned, a node enrols or a person's access changes.\n#\n# **Relative, and in this same directory, because it has to be.** An absolute include path is\n# resolved relative to the including file's directory, not from the root: nats-server given\n# `include /etc/nats/accounts.conf` from /etc/nats-server/nats.conf looks for\n# /etc/nats-server/etc/nats/accounts.conf and refuses to start. Verified against the server.\ninclude accounts.conf\n", "mode": "0644" }, { @@ -107,6 +114,7 @@ "name": "mesh-broker-nats", "ports": [ "4222:4222", + "4223:4223", "127.0.0.1:8222:8222" ], "volumes": [