mesh-catalog/modules/docker/cmd/docker-tools/secrets.go

package main

// A container's secrets in its own output (novox/hq issue 268).
//
// **The leak this catches.** Software prints what it was given: a server that announces its
// password when it starts in secure mode, a startup script that echoes the database URI it
// connects with, password and all. The container's log is then a copy of the secret that every
// reader of the log holds — this bundle's docker_logs, the journal where a container logs there,
// and whatever kept a transcript of either. Nothing in the mesh noticed, because nothing looked.
//
// **What is known here, and what is not.** A container's environment is readable through the
// runtime (docker inspect), so its values can be compared with what it printed: a variable named
// like a secret (PASSWORD, SECRET, TOKEN, KEY, …) and the password inside any URI a variable holds.
// Beside that, a credential-bearing URI anywhere in a line (`scheme://user:password@`) is caught
// by its shape, whatever the source. A secret handed only as a mounted file and never in the
// environment is not known to this bundle — it runs as the operator account, which cannot read
// the files the host writes at 0600 — and is caught only if the program prints it inside a URI.
//
// **Never the value.** A finding names the container, the module and the variable; it carries
// no value and no line. A tool that quoted the leak to report it would be a second leak.

import (
	"context"
	"encoding/json"
	"fmt"
	"net/url"
	"regexp"
	"sort"
	"strconv"
	"strings"
	"sync"
	"time"
)

// secretName is a variable name that says its value is a secret.
var secretName = regexp.MustCompile(`(?i)(pass(word|wd|phrase)?|secret|token|api_?key|private_?key|access_?key|credential|auth)`)

// notAValue is a name that says its value is where a secret is, not the secret: a file or a path.
var notAValue = regexp.MustCompile(`(?i)(_FILE|FILE|_PATH|_DIR)$`)

// uriPassword is a URI carrying a password in its userinfo: scheme://user:password@.
var uriPassword = regexp.MustCompile(`[A-Za-z][A-Za-z0-9+.-]*://[^\s/:@'"]*:([^\s/@'"]+)@`)

// masked is a password a program already hid: ***, xxx, <redacted>, [REDACTED].
var masked = regexp.MustCompile(`^(\*+|x+|X+|<[^>]*>|\[[^\]]*\]|%2A+)$`)

// ordinary is a value under a secret's name that is not one: a path, an address, a number, a switch.
var ordinary = regexp.MustCompile(`^(/.*|[A-Za-z][A-Za-z0-9+.-]*://.*|[0-9.]+[a-z]?|(?i:true|false|yes|no|on|off|none|null))$`)

// leastSecret is the shortest value compared as a secret: a shorter one matches ordinary words.
const leastSecret = 6

// knownSecret is one value a container was given, by the name it came under.
type knownSecret struct {
	Name  string
	Value string
}

// secretsIn are the values in a container's environment that must never appear in its output.
func secretsIn(env []string) []knownSecret {
	var out []knownSecret
	seen := map[string]bool{}
	add := func(name, value string) {
		if len(value) < leastSecret || masked.MatchString(value) || seen[name+"\x00"+value] {
			return
		}
		seen[name+"\x00"+value] = true
		out = append(out, knownSecret{name, value})
	}
	for _, e := range env {
		name, value, ok := strings.Cut(e, "=")
		if !ok || value == "" {
			continue
		}
		for _, m := range uriPassword.FindAllStringSubmatch(value, -1) {
			add(name+" (the password in its URI)", m[1])
			if dec, err := url.PathUnescape(m[1]); err == nil && dec != m[1] {
				add(name+" (the password in its URI)", dec)
			}
		}
		if secretName.MatchString(name) && !notAValue.MatchString(name) && !ordinary.MatchString(value) {
			add(name, value)
		}
	}
	return out
}

// forms are the ways a value may appear printed: as given, and URL-encoded.
func forms(value string) []string {
	out := []string{value}
	for _, f := range []string{url.QueryEscape(value), url.PathEscape(value)} {
		if f != value && !contains(out, f) {
			out = append(out, f)
		}
	}
	return out
}

func contains(list []string, s string) bool {
	for _, x := range list {
		if x == s {
			return true
		}
	}
	return false
}

// leaksIn is, per secret name, how many lines carry that secret; and how many carry a URI with a
// password that is none of the known ones. The lines are read and forgotten.
func leaksIn(lines []string, known []knownSecret) (byName map[string][]string, uris []string) {
	byName = map[string][]string{}
	for _, l := range lines {
		hit := false
		for _, s := range known {
			for _, f := range forms(s.Value) {
				if strings.Contains(l, f) {
					byName[s.Name] = append(byName[s.Name], stamp(l))
					hit = true
					break
				}
			}
		}
		if hit {
			continue
		}
		for _, m := range uriPassword.FindAllStringSubmatch(l, -1) {
			if !masked.MatchString(m[1]) {
				uris = append(uris, stamp(l))
				break
			}
		}
	}
	return byName, uris
}

// redact is a line with every known secret, and every password inside a URI, replaced by a mark
// naming what was there.
func redact(line string, known []knownSecret) (string, int) {
	n := 0
	for _, s := range known {
		for _, f := range forms(s.Value) {
			if c := strings.Count(line, f); c > 0 {
				line = strings.ReplaceAll(line, f, "[redacted: "+s.Name+"]")
				n += c
			}
		}
	}
	line = uriPassword.ReplaceAllStringFunc(line, func(m string) string {
		sub := uriPassword.FindStringSubmatch(m)
		if masked.MatchString(sub[1]) {
			return m
		}
		n++
		return strings.TrimSuffix(m, sub[1]+"@") + "[redacted: a password in a URI]@"
	})
	return line, n
}

// stamp is the timestamp leading a line `docker logs --timestamps` printed.
func stamp(line string) string {
	t, _, _ := strings.Cut(line, " ")
	return t
}

// envOf is one container's environment, values included — kept inside this process.
func (c *Client) envOf(ctx context.Context, ref string) ([]string, error) {
	out, err := c.docker(ctx, "container", "inspect", "--format", "{{json .Config.Env}}", ref)
	if err != nil {
		return nil, err
	}
	var env []string
	if err := json.Unmarshal([]byte(strings.TrimSpace(out)), &env); err != nil {
		return nil, fmt.Errorf("docker inspect answered an environment that is not JSON")
	}
	return env, nil
}

// Leak is one secret a container printed: by name, never by value.
type Leak struct {
	Container string `json:"container"`
	HeldBy    string `json:"held_by,omitempty"`
	Module    string `json:"module,omitempty"`
	Secret    string `json:"secret"`
	Lines     int    `json:"lines"`
	First     string `json:"first"`
	Last      string `json:"last"`
}

// ScanBudget is how long a scan may take: below the runtime's thirty-second call limit, so a scan of
// a machine with many containers answers what it read rather than nothing. ScanWidth is how many
// containers are read at once.
const (
	ScanBudget = 25 * time.Second
	ScanWidth  = 6
)

// scanned is what one container's log held.
type scanned struct {
	leaks []Leak
	lines int
	why   string
}

// scanOne reads one container's environment and log, and keeps only what was printed, by name.
func (c *Client) scanOne(ctx context.Context, ct Container, tail int) scanned {
	env, err := c.envOf(ctx, ct.ID)
	if err != nil {
		return scanned{why: err.Error()}
	}
	lines, err := c.logLines(ctx, []string{"logs", "--timestamps", "--tail", strconv.Itoa(tail), ct.ID})
	if err != nil {
		if ctx.Err() != nil {
			return scanned{why: "not read within the scan's " + ScanBudget.String()}
		}
		return scanned{why: err.Error()}
	}
	byName, uris := leaksIn(lines, secretsIn(env))
	if len(uris) > 0 {
		byName["a password inside a URI (not from its environment)"] = uris
	}
	names := make([]string, 0, len(byName))
	for n := range byName {
		names = append(names, n)
	}
	sort.Strings(names)
	out := scanned{lines: len(lines)}
	for _, n := range names {
		at := byName[n]
		sort.Strings(at)
		out.leaks = append(out.leaks, Leak{Container: ct.Name, HeldBy: ct.HeldBy, Module: ct.Module, Secret: n,
			Lines: len(at), First: at[0], Last: at[len(at)-1]})
	}
	return out
}

// SecretsInLogs scans the last `tail` lines of each container — the mesh's, or every one — for the
// secrets it was given. A container whose log could not be read is listed as unread, never as clean.
func (c *Client) SecretsInLogs(ctx context.Context, held string, tail int) (map[string]any, error) {
	all, err := c.Containers(ctx, held, "", "")
	if err != nil {
		return nil, err
	}
	ctx, cancel := context.WithTimeout(ctx, ScanBudget)
	defer cancel()
	results := make([]scanned, len(all))
	var wg sync.WaitGroup
	slots := make(chan struct{}, ScanWidth)
	for i, ct := range all {
		wg.Add(1)
		go func(i int, ct Container) {
			defer wg.Done()
			select {
			case slots <- struct{}{}:
				defer func() { <-slots }()
				results[i] = c.scanOne(ctx, ct, tail)
			case <-ctx.Done():
				results[i] = scanned{why: "not read within the scan's " + ScanBudget.String()}
			}
		}(i, ct)
	}
	wg.Wait()

	leaks := []Leak{}
	unread := []map[string]string{}
	count, read := 0, 0
	for i, r := range results {
		if r.why != "" {
			unread = append(unread, map[string]string{"container": all[i].Name, "why": r.why})
			continue
		}
		count++
		read += r.lines
		leaks = append(leaks, r.leaks...)
	}
	verdict := "no container printed a secret it was given in the lines read"
	if len(leaks) > 0 {
		verdict = fmt.Sprintf("%d secret(s) printed into container logs: rotate each one after the program stops printing it, "+
			"and recreate the container to drop its old log", len(leaks))
	}
	if len(unread) > 0 {
		verdict += fmt.Sprintf("; %d container(s) not read, so not known to be clean", len(unread))
	}
	return map[string]any{
		"verdict": verdict, "leaks": leaks, "count": len(leaks), "containers_scanned": count, "lines_read": read,
		"unread": unread,
		"knows": "values in each container's environment named like a secret, the password in any URI it holds, and any " +
			"URI carrying a password; a secret delivered only as a mounted file is caught only inside a URI",
	}, nil
}