From b87dc29706f5d86377e509d857d8f4a44ee79612 Mon Sep 17 00:00:00 2001
From: jochen <jschoubben@novox.be>
Date: Tue, 6 Oct 2026 02:33:32 +0200
Subject: [PATCH] supabase: keep logflare's API key out of its log (hq issue
 268)

vector handed logflare its API key as ?api_key= in every sink URL, and
logflare 1.4.0 prints a failed request's whole URL in its Plug.Cowboy
error report. Its ingest fails on every request here, so the key was in
the analytics log about every ten seconds. The report is an error, so no
log level hides it.

Every sink now sends the key in the x-api-key header, which logflare
reads first. A start script refuses to start logflare while the vector
config it is given still puts the key in a URL.

The key is marked "applied", not "at-start": logflare writes it into its
default user once and never updates it, so the mesh must not rotate it
by restarting.
---
 modules/supabase/module.json | 28 +++++++++++++++++++++++++---
 1 file changed, 25 insertions(+), 3 deletions(-)

diff --git a/modules/supabase/module.json b/modules/supabase/module.json
index 7d613cd..c578306 100644
--- a/modules/supabase/module.json
+++ b/modules/supabase/module.json
@@ -37,7 +37,10 @@
     "service-role-key": "${dir:state}/service-role-key.secret",
     "dashboard-user": "${dir:state}/dashboard-user.secret",
     "dashboard": "${dir:state}/dashboard.secret",
-    "logflare": "${dir:state}/logflare.secret",
+    "logflare": {
+      "path": "${dir:state}/logflare.secret",
+      "taken": "applied"
+    },
     "pooler-vault": "${dir:state}/pooler-vault.secret",
     "key-base-a": "${dir:state}/key-base-a.secret",
     "key-base-b": "${dir:state}/key-base-b.secret",
@@ -130,7 +133,7 @@
       "type": "file",
       "path": "${dir:state}/vector.yml",
       "mode": "0600",
-      "content": "# Written by the mesh (modules/supabase).\napi:\n  enabled: true\n  address: 0.0.0.0:9001\n\nsources:\n  docker_host:\n    type: docker_logs\n    include_containers:\n      - supabase-kong\n      - supabase-auth\n      - supabase-rest\n      - realtime-dev.supabase-realtime\n      - supabase-storage\n      - supabase-edge-functions\n      - supabase-db\n\ntransforms:\n  project_logs:\n    type: remap\n    inputs:\n      - docker_host\n    source: |-\n      .project = \"default\"\n      .event_message = del(.message)\n      .appname = del(.container_name)\n      del(.container_created_at)\n      del(.container_id)\n      del(.source_type)\n      del(.stream)\n      del(.label)\n      del(.image)\n      del(.host)\n      del(.stream)\n  router:\n    type: route\n    inputs:\n      - project_logs\n    route:\n      kong: '.appname == \"supabase-kong\"'\n      auth: '.appname == \"supabase-auth\"'\n      rest: '.appname == \"supabase-rest\"'\n      realtime: '.appname == \"realtime-dev.supabase-realtime\"'\n      storage: '.appname == \"supabase-storage\"'\n      functions: '.appname == \"supabase-edge-functions\"'\n      db: '.appname == \"supabase-db\"'\n  # Ignores non nginx errors since they are related with kong booting up\n  kong_logs:\n    type: remap\n    inputs:\n      - router.kong\n    source: |-\n      req, err = parse_nginx_log(.event_message, \"combined\")\n      if err == null {\n          .timestamp = req.timestamp\n          .metadata.request.headers.referer = req.referer\n          .metadata.request.headers.user_agent = req.agent\n          .metadata.request.headers.cf_connecting_ip = req.client\n          .metadata.request.method = req.method\n          .metadata.request.path = req.path\n          .metadata.request.protocol = req.protocol\n          .metadata.response.status_code = req.status\n      }\n      if err != null {\n        abort\n      }\n  # Ignores non nginx errors since they are related with kong booting up\n  kong_err:\n    type: remap\n    inputs:\n      - router.kong\n    source: |-\n      .metadata.request.method = \"GET\"\n      .metadata.response.status_code = 200\n      parsed, err = parse_nginx_log(.event_message, \"error\")\n      if err == null {\n          .timestamp = parsed.timestamp\n          .severity = parsed.severity\n          .metadata.request.host = parsed.host\n          .metadata.request.headers.cf_connecting_ip = parsed.client\n          url, err = split(parsed.request, \" \")\n          if err == null {\n              .metadata.request.method = url[0]\n              .metadata.request.path = url[1]\n              .metadata.request.protocol = url[2]\n          }\n      }\n      if err != null {\n        abort\n      }\n  # Gotrue logs are structured json strings which frontend parses directly. But we keep metadata for consistency.\n  auth_logs:\n    type: remap\n    inputs:\n      - router.auth\n    source: |-\n      parsed, err = parse_json(.event_message)\n      if err == null {\n          .metadata.timestamp = parsed.time\n          .metadata = merge!(.metadata, parsed)\n      }\n  # PostgREST logs are structured so we separate timestamp from message using regex\n  rest_logs:\n    type: remap\n    inputs:\n      - router.rest\n    source: |-\n      parsed, err = parse_regex(.event_message, r'^(?P<time>.*): (?P<msg>.*)$')\n      if err == null {\n          .event_message = parsed.msg\n          .timestamp = to_timestamp!(parsed.time)\n          .metadata.host = .project\n      }\n  # Realtime logs are structured so we parse the severity level using regex (ignore time because it has no date)\n  realtime_logs:\n    type: remap\n    inputs:\n      - router.realtime\n    source: |-\n      .metadata.project = del(.project)\n      .metadata.external_id = .metadata.project\n      parsed, err = parse_regex(.event_message, r'^(?P<time>\\d+:\\d+:\\d+\\.\\d+) \\[(?P<level>\\w+)\\] (?P<msg>.*)$')\n      if err == null {\n          .event_message = parsed.msg\n          .metadata.level = parsed.level\n      }\n  # Storage logs may contain json objects so we parse them for completeness\n  storage_logs:\n    type: remap\n    inputs:\n      - router.storage\n    source: |-\n      .metadata.project = del(.project)\n      .metadata.tenantId = .metadata.project\n      parsed, err = parse_json(.event_message)\n      if err == null {\n          .event_message = parsed.msg\n          .metadata.level = parsed.level\n          .metadata.timestamp = parsed.time\n          .metadata.context[0].host = parsed.hostname\n          .metadata.context[0].pid = parsed.pid\n      }\n  # Postgres logs some messages to stderr which we map to warning severity level\n  db_logs:\n    type: remap\n    inputs:\n      - router.db\n    source: |-\n      .metadata.host = \"db-default\"\n      .metadata.parsed.timestamp = .timestamp\n\n      parsed, err = parse_regex(.event_message, r'.*(?P<level>INFO|NOTICE|WARNING|ERROR|LOG|FATAL|PANIC?):.*', numeric_groups: true)\n\n      if err != null || parsed == null {\n        .metadata.parsed.error_severity = \"info\"\n      }\n      if parsed != null {\n       .metadata.parsed.error_severity = parsed.level\n      }\n      if .metadata.parsed.error_severity == \"info\" {\n          .metadata.parsed.error_severity = \"log\"\n      }\n      .metadata.parsed.error_severity = upcase!(.metadata.parsed.error_severity)\n\nsinks:\n  logflare_auth:\n    type: 'http'\n    inputs:\n      - auth_logs\n    encoding:\n      codec: 'json'\n    method: 'post'\n    request:\n      retry_max_duration_secs: 10\n    uri: 'http://supabase-analytics:4000/api/logs?source_name=gotrue.logs.prod&api_key=${secret:logflare}'\n  logflare_realtime:\n    type: 'http'\n    inputs:\n      - realtime_logs\n    encoding:\n      codec: 'json'\n    method: 'post'\n    request:\n      retry_max_duration_secs: 10\n    uri: 'http://supabase-analytics:4000/api/logs?source_name=realtime.logs.prod&api_key=${secret:logflare}'\n  logflare_rest:\n    type: 'http'\n    inputs:\n      - rest_logs\n    encoding:\n      codec: 'json'\n    method: 'post'\n    request:\n      retry_max_duration_secs: 10\n    uri: 'http://supabase-analytics:4000/api/logs?source_name=postgREST.logs.prod&api_key=${secret:logflare}'\n  logflare_db:\n    type: 'http'\n    inputs:\n      - db_logs\n    encoding:\n      codec: 'json'\n    method: 'post'\n    request:\n      retry_max_duration_secs: 10\n    # We must route the sink through kong because ingesting logs before logflare is fully initialised will\n    # lead to broken queries from studio. This works by the assumption that containers are started in the\n    # following order: vector > db > logflare > kong\n    uri: 'http://supabase-kong:8000/analytics/v1/api/logs?source_name=postgres.logs&api_key=${secret:logflare}'\n  logflare_functions:\n    type: 'http'\n    inputs:\n      - router.functions\n    encoding:\n      codec: 'json'\n    method: 'post'\n    request:\n      retry_max_duration_secs: 10\n    uri: 'http://supabase-analytics:4000/api/logs?source_name=deno-relay-logs&api_key=${secret:logflare}'\n  logflare_storage:\n    type: 'http'\n    inputs:\n      - storage_logs\n    encoding:\n      codec: 'json'\n    method: 'post'\n    request:\n      retry_max_duration_secs: 10\n    uri: 'http://supabase-analytics:4000/api/logs?source_name=storage.logs.prod.2&api_key=${secret:logflare}'\n  logflare_kong:\n    type: 'http'\n    inputs:\n      - kong_logs\n      - kong_err\n    encoding:\n      codec: 'json'\n    method: 'post'\n    request:\n      retry_max_duration_secs: 10\n    uri: 'http://supabase-analytics:4000/api/logs?source_name=cloudflare.logs.prod&api_key=${secret:logflare}'\n"
+      "content": "# Written by the mesh (modules/supabase).\n#\n# Every sink hands logflare its API key in the x-api-key header, never as an api_key query parameter:\n# logflare prints a failed request's whole URL, query string included, in its error report, so a\n# key in the URL is a key in its log (novox/hq issue 268). supabase-analytics refuses to start\n# while this file still puts the key in a URL.\napi:\n  enabled: true\n  address: 0.0.0.0:9001\n\nsources:\n  docker_host:\n    type: docker_logs\n    include_containers:\n      - supabase-kong\n      - supabase-auth\n      - supabase-rest\n      - realtime-dev.supabase-realtime\n      - supabase-storage\n      - supabase-edge-functions\n      - supabase-db\n\ntransforms:\n  project_logs:\n    type: remap\n    inputs:\n      - docker_host\n    source: |-\n      .project = \"default\"\n      .event_message = del(.message)\n      .appname = del(.container_name)\n      del(.container_created_at)\n      del(.container_id)\n      del(.source_type)\n      del(.stream)\n      del(.label)\n      del(.image)\n      del(.host)\n      del(.stream)\n  router:\n    type: route\n    inputs:\n      - project_logs\n    route:\n      kong: '.appname == \"supabase-kong\"'\n      auth: '.appname == \"supabase-auth\"'\n      rest: '.appname == \"supabase-rest\"'\n      realtime: '.appname == \"realtime-dev.supabase-realtime\"'\n      storage: '.appname == \"supabase-storage\"'\n      functions: '.appname == \"supabase-edge-functions\"'\n      db: '.appname == \"supabase-db\"'\n  # Ignores non nginx errors since they are related with kong booting up\n  kong_logs:\n    type: remap\n    inputs:\n      - router.kong\n    source: |-\n      req, err = parse_nginx_log(.event_message, \"combined\")\n      if err == null {\n          .timestamp = req.timestamp\n          .metadata.request.headers.referer = req.referer\n          .metadata.request.headers.user_agent = req.agent\n          .metadata.request.headers.cf_connecting_ip = req.client\n          .metadata.request.method = req.method\n          .metadata.request.path = req.path\n          .metadata.request.protocol = req.protocol\n          .metadata.response.status_code = req.status\n      }\n      if err != null {\n        abort\n      }\n  # Ignores non nginx errors since they are related with kong booting up\n  kong_err:\n    type: remap\n    inputs:\n      - router.kong\n    source: |-\n      .metadata.request.method = \"GET\"\n      .metadata.response.status_code = 200\n      parsed, err = parse_nginx_log(.event_message, \"error\")\n      if err == null {\n          .timestamp = parsed.timestamp\n          .severity = parsed.severity\n          .metadata.request.host = parsed.host\n          .metadata.request.headers.cf_connecting_ip = parsed.client\n          url, err = split(parsed.request, \" \")\n          if err == null {\n              .metadata.request.method = url[0]\n              .metadata.request.path = url[1]\n              .metadata.request.protocol = url[2]\n          }\n      }\n      if err != null {\n        abort\n      }\n  # Gotrue logs are structured json strings which frontend parses directly. But we keep metadata for consistency.\n  auth_logs:\n    type: remap\n    inputs:\n      - router.auth\n    source: |-\n      parsed, err = parse_json(.event_message)\n      if err == null {\n          .metadata.timestamp = parsed.time\n          .metadata = merge!(.metadata, parsed)\n      }\n  # PostgREST logs are structured so we separate timestamp from message using regex\n  rest_logs:\n    type: remap\n    inputs:\n      - router.rest\n    source: |-\n      parsed, err = parse_regex(.event_message, r'^(?P<time>.*): (?P<msg>.*)$')\n      if err == null {\n          .event_message = parsed.msg\n          .timestamp = to_timestamp!(parsed.time)\n          .metadata.host = .project\n      }\n  # Realtime logs are structured so we parse the severity level using regex (ignore time because it has no date)\n  realtime_logs:\n    type: remap\n    inputs:\n      - router.realtime\n    source: |-\n      .metadata.project = del(.project)\n      .metadata.external_id = .metadata.project\n      parsed, err = parse_regex(.event_message, r'^(?P<time>\\d+:\\d+:\\d+\\.\\d+) \\[(?P<level>\\w+)\\] (?P<msg>.*)$')\n      if err == null {\n          .event_message = parsed.msg\n          .metadata.level = parsed.level\n      }\n  # Storage logs may contain json objects so we parse them for completeness\n  storage_logs:\n    type: remap\n    inputs:\n      - router.storage\n    source: |-\n      .metadata.project = del(.project)\n      .metadata.tenantId = .metadata.project\n      parsed, err = parse_json(.event_message)\n      if err == null {\n          .event_message = parsed.msg\n          .metadata.level = parsed.level\n          .metadata.timestamp = parsed.time\n          .metadata.context[0].host = parsed.hostname\n          .metadata.context[0].pid = parsed.pid\n      }\n  # Postgres logs some messages to stderr which we map to warning severity level\n  db_logs:\n    type: remap\n    inputs:\n      - router.db\n    source: |-\n      .metadata.host = \"db-default\"\n      .metadata.parsed.timestamp = .timestamp\n\n      parsed, err = parse_regex(.event_message, r'.*(?P<level>INFO|NOTICE|WARNING|ERROR|LOG|FATAL|PANIC?):.*', numeric_groups: true)\n\n      if err != null || parsed == null {\n        .metadata.parsed.error_severity = \"info\"\n      }\n      if parsed != null {\n       .metadata.parsed.error_severity = parsed.level\n      }\n      if .metadata.parsed.error_severity == \"info\" {\n          .metadata.parsed.error_severity = \"log\"\n      }\n      .metadata.parsed.error_severity = upcase!(.metadata.parsed.error_severity)\n\nsinks:\n  logflare_auth:\n    type: 'http'\n    inputs:\n      - auth_logs\n    encoding:\n      codec: 'json'\n    method: 'post'\n    request:\n      retry_max_duration_secs: 10\n      headers:\n        x-api-key: '${secret:logflare}'\n    uri: 'http://supabase-analytics:4000/api/logs?source_name=gotrue.logs.prod'\n  logflare_realtime:\n    type: 'http'\n    inputs:\n      - realtime_logs\n    encoding:\n      codec: 'json'\n    method: 'post'\n    request:\n      retry_max_duration_secs: 10\n      headers:\n        x-api-key: '${secret:logflare}'\n    uri: 'http://supabase-analytics:4000/api/logs?source_name=realtime.logs.prod'\n  logflare_rest:\n    type: 'http'\n    inputs:\n      - rest_logs\n    encoding:\n      codec: 'json'\n    method: 'post'\n    request:\n      retry_max_duration_secs: 10\n      headers:\n        x-api-key: '${secret:logflare}'\n    uri: 'http://supabase-analytics:4000/api/logs?source_name=postgREST.logs.prod'\n  logflare_db:\n    type: 'http'\n    inputs:\n      - db_logs\n    encoding:\n      codec: 'json'\n    method: 'post'\n    request:\n      retry_max_duration_secs: 10\n      headers:\n        x-api-key: '${secret:logflare}'\n    # We must route the sink through kong because ingesting logs before logflare is fully initialised will\n    # lead to broken queries from studio. This works by the assumption that containers are started in the\n    # following order: vector > db > logflare > kong\n    uri: 'http://supabase-kong:8000/analytics/v1/api/logs?source_name=postgres.logs'\n  logflare_functions:\n    type: 'http'\n    inputs:\n      - router.functions\n    encoding:\n      codec: 'json'\n    method: 'post'\n    request:\n      retry_max_duration_secs: 10\n      headers:\n        x-api-key: '${secret:logflare}'\n    uri: 'http://supabase-analytics:4000/api/logs?source_name=deno-relay-logs'\n  logflare_storage:\n    type: 'http'\n    inputs:\n      - storage_logs\n    encoding:\n      codec: 'json'\n    method: 'post'\n    request:\n      retry_max_duration_secs: 10\n      headers:\n        x-api-key: '${secret:logflare}'\n    uri: 'http://supabase-analytics:4000/api/logs?source_name=storage.logs.prod.2'\n  logflare_kong:\n    type: 'http'\n    inputs:\n      - kong_logs\n      - kong_err\n    encoding:\n      codec: 'json'\n    method: 'post'\n    request:\n      retry_max_duration_secs: 10\n      headers:\n        x-api-key: '${secret:logflare}'\n    uri: 'http://supabase-analytics:4000/api/logs?source_name=cloudflare.logs.prod'\n"
     },
     {
       "id": "pooler-conf",
@@ -237,6 +240,13 @@
       "mode": "0600",
       "content": "LOGFLARE_NODE_HOST=127.0.0.1\nDB_USERNAME=supabase_admin\nDB_DATABASE=_supabase\nDB_HOSTNAME=supabase-db\nDB_PORT=5432\nDB_PASSWORD=${secret:postgres}\nDB_SCHEMA=_analytics\nLOGFLARE_API_KEY=${secret:logflare}\nLOGFLARE_SINGLE_TENANT=true\nLOGFLARE_SUPABASE_MODE=true\nLOGFLARE_MIN_CLUSTER_SIZE=1\nPOSTGRES_BACKEND_URL=postgresql://supabase_admin:${secret:postgres}@supabase-db:5432/_supabase\nPOSTGRES_BACKEND_SCHEMA=_analytics\nLOGFLARE_FEATURE_FLAG_OVERRIDE=multibackend=true\n"
     },
+    {
+      "id": "analytics-start",
+      "type": "file",
+      "path": "${dir:state}/analytics-start.sh",
+      "mode": "0644",
+      "content": "#!/bin/sh\n# Generated by the mesh. Do not edit: module supabase writes this file and replaces it at every push.\n#\n# logflare 1.4.0 prints the whole URL of every request that fails, query string included, in the\n# error report Plug.Cowboy writes (\"Request: POST /api/logs?source_name=...&api_key=...\"). That is\n# an error, so no log level hides it while errors are logged at all (novox/hq issue 268). An API key\n# passed in the URL is therefore in this container's log. logflare reads the key from the\n# x-api-key header as well, and the mesh's caller, vector, sends it there; this refuses to start\n# logflare while the vector config it is given still passes the key in a URL: an analytics that\n# does not start says why here, and one that leaks says nothing.\nset -e\nconf=/run/logflare/vector.yml\nif [ ! -r \"$conf\" ]; then\n  echo \"analytics: $conf is not there to check, so whether the API key travels in a URL is unknown; not starting (novox/hq issue 268)\" >&2\n  exit 1\nfi\nif grep -v '^[[:space:]]*#' \"$conf\" | grep -q 'api_key='; then\n  echo \"analytics: $conf passes the API key in a URL, and logflare prints a failed request's URL; not starting (novox/hq issue 268)\" >&2\n  exit 1\nfi\ncd /opt/app/rel/logflare/bin\nexec sh run.sh\n"
+    },
     {
       "id": "studio-env",
       "type": "file",
@@ -346,7 +356,19 @@
       "env-file": [
         "${dir:state}/analytics-env.env"
       ],
-      "secrets-in-environment": "Logflare reads its database password and API key from the environment only"
+      "volumes": [
+        "${dir:state}/analytics-start.sh:/run/logflare/start.sh:ro",
+        "${dir:state}/vector.yml:/run/logflare/vector.yml:ro"
+      ],
+      "args": [
+        "sh",
+        "/run/logflare/start.sh"
+      ],
+      "restart-on": [
+        "analytics-start",
+        "vector-conf"
+      ],
+      "secrets-in-environment": "Logflare reads its database password and API key from the environment only. The API key is never in a URL the mesh writes: logflare prints a failed request's URL, so vector sends the key in the x-api-key header, and the start script refuses to start logflare while vector.yml says otherwise"
     },
     {
       "id": "analytics-ready",