diff --git a/go/outward/outward.go b/go/outward/outward.go index 33f0221..7c28916 100644 --- a/go/outward/outward.go +++ b/go/outward/outward.go @@ -40,9 +40,18 @@ var ( reBotToken = regexp.MustCompile(`\b\d{6,}:[A-Za-z0-9_-]{30,}`) reKnown = regexp.MustCompile(`\b(gh[pousr]_[A-Za-z0-9]{20,}|glpat-[A-Za-z0-9_-]{16,}|sk-[A-Za-z0-9_-]{16,}|xox[abprs]-[A-Za-z0-9-]{10,}|AKIA[0-9A-Z]{16})`) reAssigned = regexp.MustCompile(`(?i)\b(password|passwd|passphrase|secret|token|api[_-]?key|apikey|credential|private[_-]?key)\s*[=:]\s*\S`) - reHex = regexp.MustCompile(`(?i)\b[0-9a-f]{32,}\b`) - reRun = regexp.MustCompile(`[A-Za-z0-9+/=_]{20,}`) - reWinPath = regexp.MustCompile(`(?i)\b[a-z]:\\`) + // reNamedSecret is a value given to a name that ends in a secret's word after `_` or `-`, as an environment + // or configuration name is written: MESH_TOKEN=, DB_PASS=, aws_secret_access_key = …, SUDO_PASSWORD value. + // The word ends the name: what follows it is `=`, `:` or a space and then a value. + reNamedSecret = regexp.MustCompile(`(?i)(^|[^a-z0-9_-])[a-z0-9]+[_-]([a-z0-9]+[_-])*(password|passwd|passphrase|pass|pwd|secret|token|api[_-]?key|apikey|access[_-]?key|secret[_-]?key|private[_-]?key|credentials?|psk)(\s*[=:]\s*|\s+)[^\s=:]`) + // reURLCredential is a password inside an address: scheme://user:password@host. + reURLCredential = regexp.MustCompile(`(?i)\b[a-z][a-z0-9+.-]*://[^/\s:@]+:[^/\s@]+@`) + // reSecretRun is a run CheckSecret judges: reRun with `-`, so a base64url value is judged whole. + reSecretRun = regexp.MustCompile(`[A-Za-z0-9+/=_-]{20,}`) + reUUID = regexp.MustCompile(`^(?i)[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$`) + reHex = regexp.MustCompile(`(?i)\b[0-9a-f]{32,}\b`) + reRun = regexp.MustCompile(`[A-Za-z0-9+/=_]{20,}`) + reWinPath = regexp.MustCompile(`(?i)\b[a-z]:\\`) ) // topLevel are names that end a host name — the generic and country ones a mesh's names use, and the @@ -109,12 +118,29 @@ func CheckSecret(text string) (Refusal, bool) { return randomRunOutsidePaths(text) } -// randomRunOutsidePaths is randomRun with each run that holds a slash judged by its pieces between the slashes. +// randomRunOutsidePaths judges every long run of a value's characters, a path's pieces between its slashes +// alone (the review of mesh-sdk #23): a run is read as a path only where it is one — it starts with a slash, +// follows a dot or a tilde (a host's or a home's path), or a name's `=` gives it one — so a base64 value, whose +// slashes fall anywhere, is judged whole. A UUID is a name, never a secret. func randomRunOutsidePaths(text string) (Refusal, bool) { - for _, run := range reRun.FindAllString(text, -1) { - for _, piece := range strings.Split(run, "/") { - if len(piece) >= 20 && looksRandom(piece) { - return Refusal{"secret", "a long random-looking string"}, false + for _, loc := range reSecretRun.FindAllStringIndex(text, -1) { + var before byte + if loc[0] > 0 { + before = text[loc[0]-1] + } + run := text[loc[0]:loc[1]] + // Standard base64 ends in its padding: such a run is a value, even when it starts with a slash. + padded := strings.HasSuffix(run, "=") && !strings.Contains(strings.TrimRight(run, "="), "=") + for i, seg := range strings.Split(run, "=") { + path := !padded && (strings.HasPrefix(seg, "/") || (i == 0 && (before == '.' || before == '~'))) + pieces := []string{seg} + if path { + pieces = strings.Split(seg, "/") + } + for _, piece := range pieces { + if len(piece) >= 20 && !reUUID.MatchString(piece) && looksRandomValue(piece) { + return Refusal{"secret", "a long random-looking string"}, false + } } } } @@ -132,7 +158,9 @@ func secretShape(text string) (Refusal, bool) { return Refusal{"secret", "a bot token"}, false case reKnown.MatchString(text): return Refusal{"secret", "a known token shape"}, false - case reAssigned.MatchString(text): + case reURLCredential.MatchString(text): + return Refusal{"secret", "a password in an address"}, false + case reAssigned.MatchString(text), reNamedSecret.MatchString(text): return Refusal{"secret", "a value given to a secret's name"}, false case reHex.MatchString(text): return Refusal{"secret", "a long hexadecimal string"}, false @@ -207,6 +235,33 @@ func looksRandom(s string) bool { return entropy(counts, len(s)) >= 3.3 } +// looksRandomValue is looksRandom for a value in a file: letters and digits mixed and spread as a random string's +// are; letters alone when the case changes as a random string's does — about as many capitals as small letters, +// where a CamelCase name (sshd's PasswordAuthentication) has one a word. +func looksRandomValue(s string) bool { + var upper, lower, digits int + counts := map[rune]int{} + for _, r := range s { + counts[r]++ + switch { + case unicode.IsUpper(r): + upper++ + case unicode.IsLower(r): + lower++ + case unicode.IsDigit(r): + digits++ + } + } + if upper+lower == 0 { + return false + } + if digits > 0 { + return entropy(counts, len(s)) >= 3.3 + } + share := float64(upper) / float64(upper+lower) + return share >= 0.3 && share <= 0.7 && entropy(counts, len(s)) >= 3.5 +} + func hasUpperAndLower(s string) bool { return strings.IndexFunc(s, unicode.IsUpper) >= 0 && strings.IndexFunc(s, unicode.IsLower) >= 0 } diff --git a/go/outward/outward_test.go b/go/outward/outward_test.go index c947930..5edffba 100644 --- a/go/outward/outward_test.go +++ b/go/outward/outward_test.go @@ -2,7 +2,12 @@ package outward // The operator channel's cases, carried here with the rule (novox/hq ADR 0310 §4): the rule moved, its meaning did not. -import "testing" +import ( + "encoding/base64" + "fmt" + "math/rand" + "testing" +) func TestRolesAndWordsMayLeave(t *testing.T) { for _, s := range []string{ @@ -90,3 +95,70 @@ func TestCheckSecretRefusesASecretsShapeAndNothingElse(t *testing.T) { } } } + +// The review of mesh-sdk #23: the shapes node-reads meets in env files, unit files, configurations and +// history files, each refused by its shape whatever the value looks like. +func TestCheckSecretRefusesTheShapesOfConfigurationFiles(t *testing.T) { + for _, text := range []string{ + "url = nats://user:pa55word@host:4222", + "DATABASE_URL=postgres://app:hunter2@db/app", + "MESH_TOKEN=s3cr3t-v4lue", + "DB_PASS=hunter2", + "export GITEA_SECRET=abc", + "Environment=APP_API_KEY=xyz", + "SUDO_PASSWORD hunter2", + "aws_secret_access_key = wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", + "password = hunter2", + "wifi_psk: correcthorse", + "value: wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", + } { + if r, ok := CheckSecret(text); ok || r.Class != "secret" { + t.Errorf("CheckSecret(%q) let it through (%v)", text, r) + } + } + // What a configuration file says that is not a secret. + for _, text := range []string{ + "ExecStart=/usr/lib/systemd/systemd-resolved", + "UUID=3f2a9c1e-5b7d-4f6a-8c0e-2b4d6f8a0c2e /home btrfs defaults 0 0", + "password required pam_unix.so", + "bypass = true", + "PasswordAuthentication no", + "url = https://git.example.org/novox/mesh-sdk.git", + "HOME=/var/lib/mesh-reads", + "TOKEN_FILE=/var/lib/app/token", + } { + if r, ok := CheckSecret(text); !ok { + t.Errorf("CheckSecret(%q) withheld it: %s", text, r) + } + } +} + +// Random values as base64 and base64url, wherever their slashes fall: every one is refused. +func TestCheckSecretRefusesRandomValuesAsBase64(t *testing.T) { + rng := rand.New(rand.NewSource(418)) + passed := map[string]int{} + for i := 0; i < 1000; i++ { + for _, n := range []int{16, 32} { + b := make([]byte, n) + rng.Read(b) + for name, enc := range map[string]*base64.Encoding{"std": base64.StdEncoding, "url": base64.RawURLEncoding} { + v := enc.EncodeToString(b) + if _, ok := CheckSecret("key " + v); ok { + passed[fmt.Sprintf("%s/%d", name, n)]++ + } + } + } + } + if len(passed) > 0 { + t.Errorf("random values passed as clean, of 1000 each: %v", passed) + } +} + +// A CamelCase name is a word, not a value: the case changes once a word. +func TestCheckSecretLetsACamelCaseNameThrough(t *testing.T) { + for _, text := range []string{"PasswordAuthenticationIsOff", "ChallengeResponseAuthentication no", "KbdInteractiveAuthentication"} { + if r, ok := CheckSecret(text); !ok { + t.Errorf("CheckSecret(%q) withheld it: %s", text, r) + } + } +}