mesh-catalog/checks/secrets/main.go

// Command secrets refuses, in every module's manifest, a mergeable file that a setting could fill with a
// password, a token or an API key in place of the vault's (novox/hq issue 342).
//
// A password is not a setting: it comes from the key vault, and it can be rotated there. A file that says
// `"merge": "json"` takes the keys of the mesh-wide and the node's layers over its own content, so a key the
// content fills with `${secret:<name>}` is one a layer could replace, and the vault's value would lose to it
// until somebody removed it by hand. The controller refuses a setting of a key the file keeps as `protected`,
// naming the layer and the key, never the value. This check holds every such file to that: each top-level key
// whose value holds `${secret:…}`, anywhere below it, is either listed in the file's `protected`, or left out
// of the file's `takes-settings` (novox/hq ADR 0326), so that no layer reaches it.
//
// A secret nested below a key the file takes for its other values (`server.secret_key` below `server`) cannot
// be protected alone today: the controller protects top-level keys. Such a file is named in the file `known`
// beside this check, one line per key, until the controller refuses a nested key. The list only shrinks: a
// line there that no longer names a finding is refused, and so is a line the trunk's copy (the merge base's,
// given with -base) does not hold. While the trunk holds no copy, the lines in seed are the trunk's.
//
//	go run . [-base <the trunk's known>] <catalogue root>
//
// Exits 1 on a finding.
package main

import (
	"bufio"
	"encoding/json"
	"flag"
	"fmt"
	"os"
	"path/filepath"
	"sort"
	"strings"
)

// Finding is one key of one mergeable file a layer could fill in place of the vault.
type Finding struct {
	Module, ID, Key string
}

func (f Finding) String() string { return f.Module + " " + f.ID + " " + f.Key }

// Check reads every manifest under root/modules and answers each secret-bearing key a layer could replace.
func Check(root string) ([]Finding, error) {
	manifests, err := filepath.Glob(filepath.Join(root, "modules", "*", "module.json"))
	if err != nil {
		return nil, err
	}
	sort.Strings(manifests)
	var out []Finding
	for _, path := range manifests {
		raw, err := os.ReadFile(path)
		if err != nil {
			return nil, err
		}
		var m struct {
			Module    string           `json:"module"`
			Resources []map[string]any `json:"resources"`
		}
		if err := json.Unmarshal(raw, &m); err != nil {
			return nil, fmt.Errorf("%s: %w", path, err)
		}
		for _, r := range m.Resources {
			if how, _ := r["merge"].(string); how == "" {
				continue
			}
			id, _ := r["id"].(string)
			content, _ := r["content"].(string)
			var doc map[string]any
			if strings.TrimSpace(content) == "" {
				continue
			}
			if err := json.Unmarshal([]byte(content), &doc); err != nil {
				return nil, fmt.Errorf("%s: %s merges as JSON and its content is not JSON: %w", m.Module, id, err)
			}
			protected := set(r["protected"])
			takes, names := r["takes-settings"]
			taken := set(takes)
			for _, key := range sortedKeys(doc) {
				for _, at := range secretsBelow(key, doc[key]) {
					switch {
					case protected[key]:
					case names && !taken[key]:
					default:
						out = append(out, Finding{m.Module, id, at})
					}
				}
			}
		}
	}
	return out, nil
}

// secretsBelow answers the dotted path of every string at or below value that names a vault secret.
func secretsBelow(at string, value any) []string {
	switch v := value.(type) {
	case string:
		if strings.Contains(v, "${secret:") {
			return []string{at}
		}
	case map[string]any:
		var out []string
		for _, k := range sortedKeys(v) {
			out = append(out, secretsBelow(at+"."+k, v[k])...)
		}
		return out
	case []any:
		var out []string
		for i, e := range v {
			out = append(out, secretsBelow(fmt.Sprintf("%s[%d]", at, i), e)...)
		}
		return out
	}
	return nil
}

func set(v any) map[string]bool {
	out := map[string]bool{}
	list, _ := v.([]any)
	for _, e := range list {
		if s, ok := e.(string); ok {
			out[s] = true
		}
	}
	return out
}

func sortedKeys(m map[string]any) []string {
	keys := make([]string, 0, len(m))
	for k := range m {
		keys = append(keys, k)
	}
	sort.Strings(keys)
	return keys
}

// Known reads the lines of the known list: "<module> <resource> <key>", # comments and blank lines skipped.
func Known(path string) (map[string]bool, error) {
	f, err := os.Open(path)
	if os.IsNotExist(err) {
		return map[string]bool{}, nil
	}
	if err != nil {
		return nil, err
	}
	defer f.Close()
	out := map[string]bool{}
	s := bufio.NewScanner(f)
	for s.Scan() {
		line := strings.TrimSpace(s.Text())
		if line == "" || strings.HasPrefix(line, "#") {
			continue
		}
		out[strings.Join(strings.Fields(line), " ")] = true
	}
	return out, s.Err()
}

// seed is the known list as it was first written, standing for the trunk's copy until the trunk holds one.
var seed = map[string]bool{
	"searxng settings server.secret_key": true,
}

// Added answers the lines of known that base does not hold: a line may leave the list, never join it.
func Added(known, base map[string]bool) []string {
	var out []string
	for line := range known {
		if !base[line] {
			out = append(out, line)
		}
	}
	sort.Strings(out)
	return out
}

// Judge splits the findings into those refused and the known lines that no longer name one.
func Judge(found []Finding, known map[string]bool) (refused []Finding, stale []string) {
	seen := map[string]bool{}
	for _, f := range found {
		seen[f.String()] = true
		if !known[f.String()] {
			refused = append(refused, f)
		}
	}
	for line := range known {
		if !seen[line] {
			stale = append(stale, line)
		}
	}
	sort.Strings(stale)
	return refused, stale
}

// remedy says how a finding is put right.
func remedy(key string) string {
	top, _, nested := strings.Cut(key, ".")
	if !nested {
		return fmt.Sprintf("list %q in the file's \"protected\", so the controller refuses a setting of it", top)
	}
	return fmt.Sprintf("the controller protects a top-level key only; list %q in the file's \"protected\" if "+
		"nothing else below it is the operator's, or leave it out of the file's \"takes-settings\"", top)
}

func main() {
	basePath := flag.String("base", "", "the trunk's copy of known (the merge base's); seed when absent")
	flag.Parse()
	root := "."
	if flag.NArg() > 0 {
		root = flag.Arg(0)
	}
	found, err := Check(root)
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(1)
	}
	known, err := Known("known")
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(1)
	}
	base := seed
	if *basePath != "" {
		if base, err = Known(*basePath); err != nil {
			fmt.Fprintln(os.Stderr, err)
			os.Exit(1)
		}
	}
	added := Added(known, base)
	for _, line := range added {
		fmt.Printf("checks/secrets/known gains %q, which the trunk's copy does not hold: the list only shrinks — "+
			"protect the key instead (novox/hq issue 342)\n", line)
	}
	refused, stale := Judge(found, known)
	for _, f := range refused {
		fmt.Printf("%s: %s fills %s with a vault secret, and a setting could replace it (novox/hq issue 342): %s\n",
			f.Module, f.ID, f.Key, remedy(f.Key))
	}
	for _, line := range stale {
		fmt.Printf("checks/secrets/known names %q, which no longer holds a secret a setting could replace: "+
			"remove the line\n", line)
	}
	if len(refused) > 0 || len(stale) > 0 || len(added) > 0 {
		os.Exit(1)
	}
	fmt.Printf("secrets: no setting can replace a vault secret in a mergeable file (%d known, nested)\n", len(known))
}