go/outward: refuse a password in an address, a value given to an env-style secret name, and base64 wherever its slashes fall (review 53, issue 418)

node-reads applies CheckSecret to env, unit and configuration files, where
user:pass@ in a URL, DB_PASS=, MESH_TOKEN= and name = value with spaces passed,
and a random base64 value passed when its slashes or dashes cut it into short
pieces. A run is now judged in pieces only where it is a path, base64 padding
marks a value, and letters alone are random when the case changes as a random
string's does, so a CamelCase name still passes.

go/outward/outward.go

@@ -40,6 +40,15 @@ var (
 	reBotToken = regexp.MustCompile(`\b\d{6,}:[A-Za-z0-9_-]{30,}`)
 	reKnown    = regexp.MustCompile(`\b(gh[pousr]_[A-Za-z0-9]{20,}|glpat-[A-Za-z0-9_-]{16,}|sk-[A-Za-z0-9_-]{16,}|xox[abprs]-[A-Za-z0-9-]{10,}|AKIA[0-9A-Z]{16})`)
 	reAssigned = regexp.MustCompile(`(?i)\b(password|passwd|passphrase|secret|token|api[_-]?key|apikey|credential|private[_-]?key)\s*[=:]\s*\S`)
+	// reNamedSecret is a value given to a name that ends in a secret's word after `_` or `-`, as an environment
+	// or configuration name is written: MESH_TOKEN=, DB_PASS=, aws_secret_access_key = …, SUDO_PASSWORD value.
+	// The word ends the name: what follows it is `=`, `:` or a space and then a value.
+	reNamedSecret = regexp.MustCompile(`(?i)(^|[^a-z0-9_-])[a-z0-9]+[_-]([a-z0-9]+[_-])*(password|passwd|passphrase|pass|pwd|secret|token|api[_-]?key|apikey|access[_-]?key|secret[_-]?key|private[_-]?key|credentials?|psk)(\s*[=:]\s*|\s+)[^\s=:]`)
+	// reURLCredential is a password inside an address: scheme://user:password@host.
+	reURLCredential = regexp.MustCompile(`(?i)\b[a-z][a-z0-9+.-]*://[^/\s:@]+:[^/\s@]+@`)
+	// reSecretRun is a run CheckSecret judges: reRun with `-`, so a base64url value is judged whole.
+	reSecretRun = regexp.MustCompile(`[A-Za-z0-9+/=_-]{20,}`)
+	reUUID      = regexp.MustCompile(`^(?i)[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$`)
 	reHex       = regexp.MustCompile(`(?i)\b[0-9a-f]{32,}\b`)
 	reRun       = regexp.MustCompile(`[A-Za-z0-9+/=_]{20,}`)
 	reWinPath   = regexp.MustCompile(`(?i)\b[a-z]:\\`)
@@ -109,15 +118,32 @@ func CheckSecret(text string) (Refusal, bool) {
 	return randomRunOutsidePaths(text)
 }
 
-// randomRunOutsidePaths is randomRun with each run that holds a slash judged by its pieces between the slashes.
+// randomRunOutsidePaths judges every long run of a value's characters, a path's pieces between its slashes
+// alone (the review of mesh-sdk #23): a run is read as a path only where it is one — it starts with a slash,
+// follows a dot or a tilde (a host's or a home's path), or a name's `=` gives it one — so a base64 value, whose
+// slashes fall anywhere, is judged whole. A UUID is a name, never a secret.
 func randomRunOutsidePaths(text string) (Refusal, bool) {
-	for _, run := range reRun.FindAllString(text, -1) {
+	for _, loc := range reSecretRun.FindAllStringIndex(text, -1) {
-		for _, piece := range strings.Split(run, "/") {
+		var before byte
-			if len(piece) >= 20 && looksRandom(piece) {
+		if loc[0] > 0 {
+			before = text[loc[0]-1]
+		}
+		run := text[loc[0]:loc[1]]
+		// Standard base64 ends in its padding: such a run is a value, even when it starts with a slash.
+		padded := strings.HasSuffix(run, "=") && !strings.Contains(strings.TrimRight(run, "="), "=")
+		for i, seg := range strings.Split(run, "=") {
+			path := !padded && (strings.HasPrefix(seg, "/") || (i == 0 && (before == '.' || before == '~')))
+			pieces := []string{seg}
+			if path {
+				pieces = strings.Split(seg, "/")
+			}
+			for _, piece := range pieces {
+				if len(piece) >= 20 && !reUUID.MatchString(piece) && looksRandomValue(piece) {
 					return Refusal{"secret", "a long random-looking string"}, false
 				}
 			}
 		}
+	}
 	return Refusal{}, true
 }
 
@@ -132,7 +158,9 @@ func secretShape(text string) (Refusal, bool) {
 		return Refusal{"secret", "a bot token"}, false
 	case reKnown.MatchString(text):
 		return Refusal{"secret", "a known token shape"}, false
-	case reAssigned.MatchString(text):
+	case reURLCredential.MatchString(text):
+		return Refusal{"secret", "a password in an address"}, false
+	case reAssigned.MatchString(text), reNamedSecret.MatchString(text):
 		return Refusal{"secret", "a value given to a secret's name"}, false
 	case reHex.MatchString(text):
 		return Refusal{"secret", "a long hexadecimal string"}, false
@@ -207,6 +235,33 @@ func looksRandom(s string) bool {
 	return entropy(counts, len(s)) >= 3.3
 }
 
+// looksRandomValue is looksRandom for a value in a file: letters and digits mixed and spread as a random string's
+// are; letters alone when the case changes as a random string's does — about as many capitals as small letters,
+// where a CamelCase name (sshd's PasswordAuthentication) has one a word.
+func looksRandomValue(s string) bool {
+	var upper, lower, digits int
+	counts := map[rune]int{}
+	for _, r := range s {
+		counts[r]++
+		switch {
+		case unicode.IsUpper(r):
+			upper++
+		case unicode.IsLower(r):
+			lower++
+		case unicode.IsDigit(r):
+			digits++
+		}
+	}
+	if upper+lower == 0 {
+		return false
+	}
+	if digits > 0 {
+		return entropy(counts, len(s)) >= 3.3
+	}
+	share := float64(upper) / float64(upper+lower)
+	return share >= 0.3 && share <= 0.7 && entropy(counts, len(s)) >= 3.5
+}
+
 func hasUpperAndLower(s string) bool {
 	return strings.IndexFunc(s, unicode.IsUpper) >= 0 && strings.IndexFunc(s, unicode.IsLower) >= 0
 }

go/outward/outward_test.go

@@ -2,7 +2,12 @@ package outward
 
 // The operator channel's cases, carried here with the rule (novox/hq ADR 0310 §4): the rule moved, its meaning did not.
 
-import "testing"
+import (
+	"encoding/base64"
+	"fmt"
+	"math/rand"
+	"testing"
+)
 
 func TestRolesAndWordsMayLeave(t *testing.T) {
 	for _, s := range []string{
@@ -90,3 +95,70 @@ func TestCheckSecretRefusesASecretsShapeAndNothingElse(t *testing.T) {
 		}
 	}
 }
+
+// The review of mesh-sdk #23: the shapes node-reads meets in env files, unit files, configurations and
+// history files, each refused by its shape whatever the value looks like.
+func TestCheckSecretRefusesTheShapesOfConfigurationFiles(t *testing.T) {
+	for _, text := range []string{
+		"url = nats://user:pa55word@host:4222",
+		"DATABASE_URL=postgres://app:hunter2@db/app",
+		"MESH_TOKEN=s3cr3t-v4lue",
+		"DB_PASS=hunter2",
+		"export GITEA_SECRET=abc",
+		"Environment=APP_API_KEY=xyz",
+		"SUDO_PASSWORD hunter2",
+		"aws_secret_access_key = wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY",
+		"password = hunter2",
+		"wifi_psk: correcthorse",
+		"value: wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY",
+	} {
+		if r, ok := CheckSecret(text); ok || r.Class != "secret" {
+			t.Errorf("CheckSecret(%q) let it through (%v)", text, r)
+		}
+	}
+	// What a configuration file says that is not a secret.
+	for _, text := range []string{
+		"ExecStart=/usr/lib/systemd/systemd-resolved",
+		"UUID=3f2a9c1e-5b7d-4f6a-8c0e-2b4d6f8a0c2e /home btrfs defaults 0 0",
+		"password   required   pam_unix.so",
+		"bypass = true",
+		"PasswordAuthentication no",
+		"url = https://git.example.org/novox/mesh-sdk.git",
+		"HOME=/var/lib/mesh-reads",
+		"TOKEN_FILE=/var/lib/app/token",
+	} {
+		if r, ok := CheckSecret(text); !ok {
+			t.Errorf("CheckSecret(%q) withheld it: %s", text, r)
+		}
+	}
+}
+
+// Random values as base64 and base64url, wherever their slashes fall: every one is refused.
+func TestCheckSecretRefusesRandomValuesAsBase64(t *testing.T) {
+	rng := rand.New(rand.NewSource(418))
+	passed := map[string]int{}
+	for i := 0; i < 1000; i++ {
+		for _, n := range []int{16, 32} {
+			b := make([]byte, n)
+			rng.Read(b)
+			for name, enc := range map[string]*base64.Encoding{"std": base64.StdEncoding, "url": base64.RawURLEncoding} {
+				v := enc.EncodeToString(b)
+				if _, ok := CheckSecret("key " + v); ok {
+					passed[fmt.Sprintf("%s/%d", name, n)]++
+				}
+			}
+		}
+	}
+	if len(passed) > 0 {
+		t.Errorf("random values passed as clean, of 1000 each: %v", passed)
+	}
+}
+
+// A CamelCase name is a word, not a value: the case changes once a word.
+func TestCheckSecretLetsACamelCaseNameThrough(t *testing.T) {
+	for _, text := range []string{"PasswordAuthenticationIsOff", "ChallengeResponseAuthentication no", "KbdInteractiveAuthentication"} {
+		if r, ok := CheckSecret(text); !ok {
+			t.Errorf("CheckSecret(%q) withheld it: %s", text, r)
+		}
+	}
+}