Keep secrets off command lines the runtime records (hq issue 282)
mosquitto passed the broker's admin password to mosquitto_ctrl as -P on every docker exec, and the container runtime keeps every exec's command line in its event stream, where docker_events returned it. The admin credentials now reach mosquitto_ctrl as a 0600 options file fed on stdin, client passwords at its own prompt, and an argv carrying a secret is refused before it runs. The admin secret says it is taken at start: the bootstrap re-runs when the mesh replaces it and re-keys the broker online from the value it last applied, so it can be rotated. docker_events redacts what an exec's command line carried, and docker_secrets_in_events names such secrets by name. keycloak's repair hands kcadm its passwords through KC_CLI_PASSWORD; minio gives mc its root alias through MC_HOST_mesh.
This commit is contained in:
jochen
@@ -0,0 +1,256 @@
|
||||
package main
|
||||
|
||||
// A secret on a command line (novox/hq issue 282).
|
||||
//
|
||||
// **The leak this catches.** The runtime records the command line of every exec — `docker exec`, and
|
||||
// a health check, which is one — in its event stream, as the event's action (`exec_create: <argv
|
||||
// joined by spaces>`). A program that hands a password to a tool as an argument (`-P <password>`,
|
||||
// `--password <password>`, `PGPASSWORD=<password> psql`) has therefore given it to everyone who may
|
||||
// ask the runtime what happened, for as long as the runtime keeps its events — and, through
|
||||
// docker_events, to every transcript of an agent that asked. The mosquitto module did exactly that
|
||||
// with the broker's admin password, on every administrative call.
|
||||
//
|
||||
// **What is known here.** As for a log: the values of the container's environment named like a
|
||||
// secret and the passwords inside its URIs, by name; and, whatever their source, the values a
|
||||
// command line carries by its shape — the word after a flag that takes a password, a NAME=value
|
||||
// whose name says secret, the password a dynsec command sets. A secret given as a file and passed by
|
||||
// a flag the shapes do not know is not caught.
|
||||
//
|
||||
// **Never the value.** What is shown carries `[redacted: <what it was>]` in its place, and a finding
|
||||
// names the container, the module and what it was, by name.
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"path"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// passwordFlags take a secret as their next word, whatever the program.
|
||||
var passwordFlags = map[string]bool{
|
||||
"-P": true, "--password": true, "--pass": true, "--passwd": true, "--secret": true, "--secret-key": true,
|
||||
"--token": true, "--api-key": true, "--apikey": true, "--auth": true,
|
||||
}
|
||||
|
||||
// programFlags take a secret as their next word for one program only: elsewhere the same flag means
|
||||
// something else (redis-cli's -a is its password; nft's -a is not).
|
||||
var programFlags = map[string]map[string]bool{
|
||||
"redis-cli": {"-a": true},
|
||||
"keydb-cli": {"-a": true},
|
||||
"valkey-cli": {"-a": true},
|
||||
"mosquitto_ctrl": {"-p": true}, // createClient -p <password>; the connect -p is a port, and a port is ordinary
|
||||
}
|
||||
|
||||
// positionalSecret is where a dynsec command carries a password as an argument: the word that many
|
||||
// places after the command's name.
|
||||
var positionalSecret = map[string]int{"setClientPassword": 2, "init": 3}
|
||||
|
||||
// commandSecret is one secret a command line carried, by what it was.
|
||||
type commandSecret struct {
|
||||
Name string
|
||||
Value string
|
||||
}
|
||||
|
||||
// secretsOnCommandLine are the values a command line carries by their shape, by what each one is.
|
||||
func secretsOnCommandLine(words []string) []commandSecret {
|
||||
var out []commandSecret
|
||||
add := func(name, value string) {
|
||||
if len(value) < leastSecret || masked.MatchString(value) || ordinary.MatchString(value) {
|
||||
return
|
||||
}
|
||||
out = append(out, commandSecret{name, value})
|
||||
}
|
||||
program := ""
|
||||
for i, w := range words {
|
||||
base := path.Base(w)
|
||||
if _, known := programFlags[base]; known || base == "mosquitto_ctrl" {
|
||||
program = base
|
||||
}
|
||||
if flag, value, ok := strings.Cut(w, "="); ok && strings.HasPrefix(flag, "-") {
|
||||
if passwordFlags[flag] || programFlags[program][flag] {
|
||||
add("the value of "+flag, value)
|
||||
}
|
||||
continue
|
||||
}
|
||||
if name, value, ok := strings.Cut(w, "="); ok && name != "" && !strings.HasPrefix(name, "-") &&
|
||||
secretName.MatchString(name) && !notAValue.MatchString(name) && !strings.ContainsAny(name, "/:") {
|
||||
add("the value of "+name, value)
|
||||
continue
|
||||
}
|
||||
if i+1 < len(words) && (passwordFlags[w] || programFlags[program][w]) {
|
||||
add("the word after "+w+" in a "+orProgram(program, words)+" command line", words[i+1])
|
||||
}
|
||||
if program == "mosquitto_ctrl" {
|
||||
if at, ok := positionalSecret[w]; ok && i+at < len(words) && dynsecVerb(words, i) {
|
||||
add("the password given to dynsec "+w, words[i+at])
|
||||
}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// dynsecVerb says the word at i is a dynsec command's name: it follows "dynsec".
|
||||
func dynsecVerb(words []string, i int) bool {
|
||||
for j := i - 1; j >= 0; j-- {
|
||||
if words[j] == "dynsec" {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func orProgram(program string, words []string) string {
|
||||
if program != "" {
|
||||
return program
|
||||
}
|
||||
if len(words) > 0 {
|
||||
return path.Base(words[0])
|
||||
}
|
||||
return "program's"
|
||||
}
|
||||
|
||||
// redactCommand is a command line with every known secret, every password inside a URI and every
|
||||
// value its shape says is a secret replaced by a mark naming what was there; and what was replaced,
|
||||
// by name.
|
||||
func redactCommand(command string, known []knownSecret) (string, []string) {
|
||||
var names []string
|
||||
for _, s := range known {
|
||||
for _, f := range forms(s.Value) {
|
||||
if strings.Contains(command, f) {
|
||||
command = strings.ReplaceAll(command, f, "[redacted: "+s.Name+"]")
|
||||
names = append(names, s.Name)
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, s := range secretsOnCommandLine(strings.Fields(command)) {
|
||||
if strings.Contains(command, s.Value) {
|
||||
command = strings.ReplaceAll(command, s.Value, "[redacted: "+s.Name+"]")
|
||||
names = append(names, s.Name)
|
||||
}
|
||||
}
|
||||
if line, n := redact(command, nil); n > 0 {
|
||||
command = line
|
||||
names = append(names, "a password in a URI")
|
||||
}
|
||||
return command, names
|
||||
}
|
||||
|
||||
// execCommand is the command line an exec event carries, and whether it carries one.
|
||||
func execCommand(action string) (verb, command string, ok bool) {
|
||||
verb, command, ok = strings.Cut(action, ": ")
|
||||
if !ok || !strings.HasPrefix(verb, "exec_") {
|
||||
return "", "", false
|
||||
}
|
||||
return verb, command, true
|
||||
}
|
||||
|
||||
// envCache reads each container's environment once per call.
|
||||
type envCache struct {
|
||||
c *Client
|
||||
ctx context.Context
|
||||
seen map[string][]knownSecret
|
||||
}
|
||||
|
||||
func (e *envCache) of(id string) []knownSecret {
|
||||
if e.seen == nil {
|
||||
e.seen = map[string][]knownSecret{}
|
||||
}
|
||||
if k, ok := e.seen[id]; ok {
|
||||
return k
|
||||
}
|
||||
env, _ := e.c.envOf(e.ctx, id) // a container gone since: its shapes are still caught
|
||||
e.seen[id] = secretsIn(env)
|
||||
return e.seen[id]
|
||||
}
|
||||
|
||||
// CommandLeak is one secret the runtime recorded on exec command lines: by name, never by value.
|
||||
type CommandLeak struct {
|
||||
Container string `json:"container"`
|
||||
HeldBy string `json:"held_by,omitempty"`
|
||||
Module string `json:"module,omitempty"`
|
||||
Secret string `json:"secret"`
|
||||
Execs int `json:"execs"`
|
||||
Program string `json:"program"`
|
||||
First string `json:"first"`
|
||||
Last string `json:"last"`
|
||||
}
|
||||
|
||||
// SecretsInEvents reads the runtime's exec events in a window ending now and says which secrets
|
||||
// their command lines carried, by container and name.
|
||||
func (c *Client) SecretsInEvents(ctx context.Context, minutes int) (map[string]any, error) {
|
||||
out, err := c.docker(ctx, "events", "--since", fmt.Sprintf("%dm", minutes), "--until", "0s",
|
||||
"--filter", "type=container", "--filter", "event=exec_create", "--format", "{{json .}}")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
raw, err := jsonLines[runtimeEvent](out)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
envs := &envCache{c: c, ctx: ctx}
|
||||
type key struct{ container, secret string }
|
||||
found := map[key]*CommandLeak{}
|
||||
execs := 0
|
||||
for _, e := range raw {
|
||||
verb, command, ok := execCommand(e.Action)
|
||||
if !ok || verb != "exec_create" {
|
||||
continue // an exec_start repeats its exec_create's command line
|
||||
}
|
||||
execs++
|
||||
_, names := redactCommand(command, envs.of(e.Actor.ID))
|
||||
if len(names) == 0 {
|
||||
continue
|
||||
}
|
||||
at := time.Unix(0, e.TimeNano).UTC().Format(time.RFC3339)
|
||||
held := e.Actor.Attributes[MeshLabel]
|
||||
module, _, _ := strings.Cut(held, ".")
|
||||
program := ""
|
||||
if f := strings.Fields(command); len(f) > 0 {
|
||||
program = path.Base(f[0])
|
||||
}
|
||||
for _, n := range names {
|
||||
k := key{e.Actor.Attributes["name"], n}
|
||||
l, ok := found[k]
|
||||
if !ok {
|
||||
l = &CommandLeak{Container: k.container, HeldBy: held, Module: module, Secret: n, Program: program, First: at}
|
||||
found[k] = l
|
||||
}
|
||||
l.Execs++
|
||||
l.Last = at
|
||||
}
|
||||
}
|
||||
leaks := []CommandLeak{}
|
||||
for _, l := range found {
|
||||
leaks = append(leaks, *l)
|
||||
}
|
||||
sort.Slice(leaks, func(i, j int) bool {
|
||||
if leaks[i].Container != leaks[j].Container {
|
||||
return leaks[i].Container < leaks[j].Container
|
||||
}
|
||||
return leaks[i].Secret < leaks[j].Secret
|
||||
})
|
||||
verdict := fmt.Sprintf("no exec in the last %d minutes carried a secret on its command line", minutes)
|
||||
if len(leaks) > 0 {
|
||||
verdict = fmt.Sprintf("%d secret(s) on exec command lines the runtime recorded: the code that runs the exec must hand "+
|
||||
"them over another way (a file, stdin), and each is rotated once it does (novox/hq issue 282)", len(leaks))
|
||||
}
|
||||
return map[string]any{
|
||||
"verdict": verdict, "leaks": leaks, "count": len(leaks), "execs_read": execs, "minutes": minutes,
|
||||
"knows": "values of each container's environment named like a secret, passwords in URIs, and by shape: the word after " +
|
||||
"a password flag, a NAME=value named like a secret, and the password a dynsec command sets",
|
||||
"history": "the runtime keeps a bounded number of events, so a window longer than what it holds reads only what it still has",
|
||||
}, nil
|
||||
}
|
||||
|
||||
// runtimeEvent is one line of `docker events --format '{{json .}}'`.
|
||||
type runtimeEvent struct {
|
||||
Type, Action string
|
||||
Actor struct {
|
||||
ID string
|
||||
Attributes map[string]string
|
||||
}
|
||||
TimeNano int64 `json:"timeNano"`
|
||||
}
|
||||
Reference in New Issue
Block a user