Merge pull request 'go/outward: the content rule in one place, for the operator channel and node-reads (hq ADR 0310 §4, issue 418)' (#23) from fix/418-node-reads into main
This commit was merged in pull request #23.
This commit is contained in:
@@ -0,0 +1,276 @@
|
||||
// Package outward is the content rule (novox/hq ADR 0310 §4, to-be 45 §5, research 028 Q8, ADR 0227): what
|
||||
// may leave the mesh, and what an answer may carry at all. One rule, here, so every module that answers outward
|
||||
// applies the same one rather than a copy: the operator channel's router and every node-reads answer.
|
||||
//
|
||||
// Two strengths:
|
||||
//
|
||||
// - Check is the operator channel's: roles and words only. A message carrying an address, a path or
|
||||
// anything shaped like a secret is refused, because a channel such as Telegram is not end-to-end
|
||||
// encrypted for bots and this rule is the only thing between a condition's words and someone else's
|
||||
// server. Deliberately wider than it must be: a commit id or a long random name is refused too.
|
||||
// - CheckSecret is the secret class alone: what leaves the mesh nowhere and is answered nowhere. A path
|
||||
// or an address is what a file read or an operator's approval is about; a secret's shape never is.
|
||||
//
|
||||
// A refusal says its class and a few words, never the text itself.
|
||||
package outward
|
||||
|
||||
import (
|
||||
"math"
|
||||
"regexp"
|
||||
"strings"
|
||||
"unicode"
|
||||
)
|
||||
|
||||
// Refusal says why a text may not leave: the class of what it carried, never the text itself.
|
||||
type Refusal struct {
|
||||
Class string // address, path or secret
|
||||
What string // a few words: "an IPv4 address", "a URL", …
|
||||
}
|
||||
|
||||
func (r Refusal) String() string { return r.Class + " (" + r.What + ")" }
|
||||
|
||||
var (
|
||||
reURL = regexp.MustCompile(`(?i)\b[a-z][a-z0-9+.-]*://`)
|
||||
reEmail = regexp.MustCompile(`[A-Za-z0-9._%+-]+@[A-Za-z0-9-]+(\.[A-Za-z0-9-]+)*\.[A-Za-z]{2,}`)
|
||||
reIPv4 = regexp.MustCompile(`\b\d{1,3}(\.\d{1,3}){3}\b`)
|
||||
reIPv6 = regexp.MustCompile(`(?i)(^|[^0-9a-z:])(([0-9a-f]{1,4}:){4,7}[0-9a-f]{1,4}|([0-9a-f]{1,4}:)*[0-9a-f]{0,4}::([0-9a-f]{1,4}:)*[0-9a-f]{0,4})([^0-9a-z:]|$)`)
|
||||
reMAC = regexp.MustCompile(`(?i)\b([0-9a-f]{2}[:-]){5}[0-9a-f]{2}\b`)
|
||||
rePEM = regexp.MustCompile(`-----BEGIN [A-Z ]+-----`)
|
||||
reJWT = regexp.MustCompile(`\beyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}`)
|
||||
reBotToken = regexp.MustCompile(`\b\d{6,}:[A-Za-z0-9_-]{30,}`)
|
||||
reKnown = regexp.MustCompile(`\b(gh[pousr]_[A-Za-z0-9]{20,}|glpat-[A-Za-z0-9_-]{16,}|sk-[A-Za-z0-9_-]{16,}|xox[abprs]-[A-Za-z0-9-]{10,}|AKIA[0-9A-Z]{16})`)
|
||||
reAssigned = regexp.MustCompile(`(?i)\b(password|passwd|passphrase|secret|token|api[_-]?key|apikey|credential|private[_-]?key)\s*[=:]\s*\S`)
|
||||
// reNamedSecret is a value given to a name that ends in a secret's word after `_` or `-`, as an environment
|
||||
// or configuration name is written: MESH_TOKEN=, DB_PASS=, aws_secret_access_key = …, SUDO_PASSWORD value.
|
||||
// The word ends the name: what follows it is `=`, `:` or a space and then a value.
|
||||
reNamedSecret = regexp.MustCompile(`(?i)(^|[^a-z0-9_-])[a-z0-9]+[_-]([a-z0-9]+[_-])*(password|passwd|passphrase|pass|pwd|secret|token|api[_-]?key|apikey|access[_-]?key|secret[_-]?key|private[_-]?key|credentials?|psk)(\s*[=:]\s*|\s+)[^\s=:]`)
|
||||
// reURLCredential is a password inside an address: scheme://user:password@host.
|
||||
reURLCredential = regexp.MustCompile(`(?i)\b[a-z][a-z0-9+.-]*://[^/\s:@]+:[^/\s@]+@`)
|
||||
// reSecretRun is a run CheckSecret judges: reRun with `-`, so a base64url value is judged whole.
|
||||
reSecretRun = regexp.MustCompile(`[A-Za-z0-9+/=_-]{20,}`)
|
||||
reUUID = regexp.MustCompile(`^(?i)[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$`)
|
||||
reHex = regexp.MustCompile(`(?i)\b[0-9a-f]{32,}\b`)
|
||||
reRun = regexp.MustCompile(`[A-Za-z0-9+/=_]{20,}`)
|
||||
reWinPath = regexp.MustCompile(`(?i)\b[a-z]:\\`)
|
||||
)
|
||||
|
||||
// topLevel are names that end a host name — the generic and country ones a mesh's names use, and the
|
||||
// private ones (.internal, .lan, .home, .local) a mesh is likelier to.
|
||||
var topLevel = map[string]bool{}
|
||||
|
||||
func init() {
|
||||
for _, t := range strings.Fields(`com net org edu gov mil int io dev app cloud ai co me info biz xyz
|
||||
site online tech page link
|
||||
be nl de fr uk lu eu ch at it es pt se no dk fi pl cz us ca au nz jp cn ru in br ie
|
||||
internal lan home local localdomain corp intranet private arpa test example invalid localhost`) {
|
||||
topLevel[t] = true
|
||||
}
|
||||
}
|
||||
|
||||
// Check says whether a text may leave the mesh, and when not, why.
|
||||
func Check(text string) (Refusal, bool) {
|
||||
switch {
|
||||
case reURL.MatchString(text):
|
||||
return Refusal{"address", "a URL"}, false
|
||||
case reEmail.MatchString(text):
|
||||
return Refusal{"address", "a mail address"}, false
|
||||
case reIPv4.MatchString(text):
|
||||
return Refusal{"address", "an IPv4 address"}, false
|
||||
case reMAC.MatchString(text):
|
||||
return Refusal{"address", "a hardware address"}, false
|
||||
case reIPv6.MatchString(text):
|
||||
return Refusal{"address", "an IPv6 address"}, false
|
||||
}
|
||||
if refusal, ok := secretShape(text); !ok {
|
||||
return refusal, false
|
||||
}
|
||||
if reWinPath.MatchString(text) {
|
||||
return Refusal{"path", "a drive path"}, false
|
||||
}
|
||||
if refusal, ok := randomRun(text); !ok {
|
||||
return refusal, false
|
||||
}
|
||||
for _, word := range strings.FieldsFunc(text, func(r rune) bool {
|
||||
return unicode.IsSpace(r) || strings.ContainsRune("\"'`()[]{}<>,;|", r)
|
||||
}) {
|
||||
w := strings.TrimRight(word, ".:!?")
|
||||
if isPath(w) {
|
||||
return Refusal{"path", "a file path"}, false
|
||||
}
|
||||
if isHostName(w) {
|
||||
return Refusal{"address", "a host name"}, false
|
||||
}
|
||||
}
|
||||
return Refusal{}, true
|
||||
}
|
||||
|
||||
// CheckSecret says whether a text carries a secret's shape — the one class that leaves the mesh nowhere. It is
|
||||
// what an ask's whole words (asks.Ask.Whole, novox/hq issue 383) are held to: on a kind that proves who answers,
|
||||
// a path or an address is what the operator approves and may be shown, a secret never. A path is read as one:
|
||||
// a run with a slash in it is judged piece by piece between the slashes, so "/mnt/Photos_2024/Jochen" is a
|
||||
// path and not a random string (the review of 2026-10-10); the named shapes — a key block, a signed token, a
|
||||
// bot token, a known token shape, a value given to a secret's name, a long hexadecimal string — hold whatever
|
||||
// the run holds.
|
||||
func CheckSecret(text string) (Refusal, bool) {
|
||||
if refusal, ok := secretShape(text); !ok {
|
||||
return refusal, false
|
||||
}
|
||||
return randomRunOutsidePaths(text)
|
||||
}
|
||||
|
||||
// randomRunOutsidePaths judges every long run of a value's characters, a path's pieces between its slashes
|
||||
// alone (the review of mesh-sdk #23): a run is read as a path only where it is one — it starts with a slash,
|
||||
// follows a dot or a tilde (a host's or a home's path), or a name's `=` gives it one — so a base64 value, whose
|
||||
// slashes fall anywhere, is judged whole. A UUID is a name, never a secret.
|
||||
func randomRunOutsidePaths(text string) (Refusal, bool) {
|
||||
for _, loc := range reSecretRun.FindAllStringIndex(text, -1) {
|
||||
var before byte
|
||||
if loc[0] > 0 {
|
||||
before = text[loc[0]-1]
|
||||
}
|
||||
run := text[loc[0]:loc[1]]
|
||||
// Standard base64 ends in its padding: such a run is a value, even when it starts with a slash.
|
||||
padded := strings.HasSuffix(run, "=") && !strings.Contains(strings.TrimRight(run, "="), "=")
|
||||
for i, seg := range strings.Split(run, "=") {
|
||||
path := !padded && (strings.HasPrefix(seg, "/") || (i == 0 && (before == '.' || before == '~')))
|
||||
pieces := []string{seg}
|
||||
if path {
|
||||
pieces = strings.Split(seg, "/")
|
||||
}
|
||||
for _, piece := range pieces {
|
||||
if len(piece) >= 20 && !reUUID.MatchString(piece) && looksRandomValue(piece) {
|
||||
return Refusal{"secret", "a long random-looking string"}, false
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return Refusal{}, true
|
||||
}
|
||||
|
||||
// secretShape is the secret shapes a pattern names.
|
||||
func secretShape(text string) (Refusal, bool) {
|
||||
switch {
|
||||
case rePEM.MatchString(text):
|
||||
return Refusal{"secret", "a key block"}, false
|
||||
case reJWT.MatchString(text):
|
||||
return Refusal{"secret", "a signed token"}, false
|
||||
case reBotToken.MatchString(text):
|
||||
return Refusal{"secret", "a bot token"}, false
|
||||
case reKnown.MatchString(text):
|
||||
return Refusal{"secret", "a known token shape"}, false
|
||||
case reURLCredential.MatchString(text):
|
||||
return Refusal{"secret", "a password in an address"}, false
|
||||
case reAssigned.MatchString(text), reNamedSecret.MatchString(text):
|
||||
return Refusal{"secret", "a value given to a secret's name"}, false
|
||||
case reHex.MatchString(text):
|
||||
return Refusal{"secret", "a long hexadecimal string"}, false
|
||||
}
|
||||
return Refusal{}, true
|
||||
}
|
||||
|
||||
// randomRun is a long unbroken run of characters spread as a random string's are.
|
||||
func randomRun(text string) (Refusal, bool) {
|
||||
for _, run := range reRun.FindAllString(text, -1) {
|
||||
if looksRandom(run) {
|
||||
return Refusal{"secret", "a long random-looking string"}, false
|
||||
}
|
||||
}
|
||||
return Refusal{}, true
|
||||
}
|
||||
|
||||
// isPath: absolute, home-relative or dot-relative, or two separators deep. A mesh address names one
|
||||
// machine and one tool (`ace/postgres.query`) and has one; a ratio ("3/4") has digits only.
|
||||
func isPath(w string) bool {
|
||||
if w == "" {
|
||||
return false
|
||||
}
|
||||
if strings.HasPrefix(w, "/") && len(w) > 1 {
|
||||
return true
|
||||
}
|
||||
for _, p := range []string{"~/", "./", "../", "$HOME", "${"} {
|
||||
if strings.HasPrefix(w, p) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return strings.Count(w, "/") >= 2 || strings.Contains(w, "\\")
|
||||
}
|
||||
|
||||
// isHostName: two names or more, the last a top-level one. A condition key's last name is its kind
|
||||
// (`machine.ace.silent`), which none of these is.
|
||||
func isHostName(w string) bool {
|
||||
w = strings.ToLower(w)
|
||||
if w == "localhost" {
|
||||
return true
|
||||
}
|
||||
parts := strings.Split(w, ".")
|
||||
if len(parts) < 2 {
|
||||
return false
|
||||
}
|
||||
for _, p := range parts {
|
||||
if p == "" {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return topLevel[parts[len(parts)-1]]
|
||||
}
|
||||
|
||||
// looksRandom: letters and digits mixed, and the characters spread as a random string's are. A
|
||||
// sentence's words are separated, so only an unbroken run reaches here.
|
||||
func looksRandom(s string) bool {
|
||||
var letters, digits int
|
||||
counts := map[rune]int{}
|
||||
for _, r := range s {
|
||||
counts[r]++
|
||||
switch {
|
||||
case unicode.IsLetter(r):
|
||||
letters++
|
||||
case unicode.IsDigit(r):
|
||||
digits++
|
||||
}
|
||||
}
|
||||
if letters == 0 || digits == 0 {
|
||||
// One class only: a word, or a number. A long number of digits alone is a count or a time.
|
||||
return letters > 0 && hasUpperAndLower(s) && entropy(counts, len(s)) >= 4.0
|
||||
}
|
||||
return entropy(counts, len(s)) >= 3.3
|
||||
}
|
||||
|
||||
// looksRandomValue is looksRandom for a value in a file: letters and digits mixed and spread as a random string's
|
||||
// are; letters alone when the case changes as a random string's does — about as many capitals as small letters,
|
||||
// where a CamelCase name (sshd's PasswordAuthentication) has one a word.
|
||||
func looksRandomValue(s string) bool {
|
||||
var upper, lower, digits int
|
||||
counts := map[rune]int{}
|
||||
for _, r := range s {
|
||||
counts[r]++
|
||||
switch {
|
||||
case unicode.IsUpper(r):
|
||||
upper++
|
||||
case unicode.IsLower(r):
|
||||
lower++
|
||||
case unicode.IsDigit(r):
|
||||
digits++
|
||||
}
|
||||
}
|
||||
if upper+lower == 0 {
|
||||
return false
|
||||
}
|
||||
if digits > 0 {
|
||||
return entropy(counts, len(s)) >= 3.3
|
||||
}
|
||||
share := float64(upper) / float64(upper+lower)
|
||||
return share >= 0.3 && share <= 0.7 && entropy(counts, len(s)) >= 3.5
|
||||
}
|
||||
|
||||
func hasUpperAndLower(s string) bool {
|
||||
return strings.IndexFunc(s, unicode.IsUpper) >= 0 && strings.IndexFunc(s, unicode.IsLower) >= 0
|
||||
}
|
||||
|
||||
func entropy(counts map[rune]int, n int) float64 {
|
||||
e := 0.0
|
||||
for _, c := range counts {
|
||||
p := float64(c) / float64(n)
|
||||
e -= p * math.Log2(p)
|
||||
}
|
||||
return e
|
||||
}
|
||||
@@ -0,0 +1,164 @@
|
||||
package outward
|
||||
|
||||
// The operator channel's cases, carried here with the rule (novox/hq ADR 0310 §4): the rule moved, its meaning did not.
|
||||
|
||||
import (
|
||||
"encoding/base64"
|
||||
"fmt"
|
||||
"math/rand"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestRolesAndWordsMayLeave(t *testing.T) {
|
||||
for _, s := range []string{
|
||||
"URGENT: the home server has not been heard for 15 min",
|
||||
"key: provider.keycloak.ace.gitea.failing",
|
||||
"more: conditions show machine.ace.silent",
|
||||
"since: 2026-10-06 12:30 UTC",
|
||||
"call call-1759752000123456789-12 has run past its bound of 10 min",
|
||||
"ace/postgres.query answers no more",
|
||||
"3/4 machines answered; plan 41 waits on build 7f3a9c1e",
|
||||
"the bus advisory slow-consumer for ace_records",
|
||||
"core.controller.novox.rolled-back",
|
||||
"seat node-notifier has no live holder",
|
||||
"HELD BACK: 5 message(s) over the cap of 20 an hour",
|
||||
"STILL OPEN after 1.5 h: the controller's event loop takes no message",
|
||||
"node-engine and node tools builds differ from the plan's",
|
||||
} {
|
||||
if r, ok := Check(s); !ok {
|
||||
t.Errorf("refused %q: %s", s, r)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnAddressAPathOrASecretMayNot(t *testing.T) {
|
||||
for s, class := range map[string]string{
|
||||
"cannot reach 192.168.1.20": "address",
|
||||
"listening on 10.0.0.7:5432": "address",
|
||||
"route fd00:1234:5678::1 is gone": "address",
|
||||
"fe80::1 answered": "address",
|
||||
"2001:db8:0:0:0:0:2:1 answered": "address",
|
||||
"see https://git.example.org/x": "address",
|
||||
"git.novox.internal does not answer": "address",
|
||||
"the mail for admin@example.org bounced": "address",
|
||||
"zurag.be is down": "address",
|
||||
"localhost refused": "address",
|
||||
"the card aa:bb:cc:dd:ee:ff went away": "address",
|
||||
"/var/lib/mesh-controller is full": "path",
|
||||
"~/.config/hal/env changed": "path",
|
||||
"read ./grants.json": "path",
|
||||
"services/postgres/data/pg_hba.conf": "path",
|
||||
"C:\\Users\\x": "path",
|
||||
"token 123456789:AAHdqTcvCH1vGWJxfSeofSAs0K5PALDsaw": "secret",
|
||||
"ghp_abcdefghijklmnopqrstuvwxyz0123456789": "secret",
|
||||
"password=hunter2": "secret",
|
||||
"api_key: x": "secret",
|
||||
"-----BEGIN OPENSSH PRIVATE KEY-----": "secret",
|
||||
"eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiIxMjM0NTY3ODkwIn0": "secret",
|
||||
"commit 3f2a9c1e5b7d4f6a8c0e2b4d6f8a0c2e4b6d8f0a": "secret",
|
||||
"value Zx8Qp2Lm9Rt4Vb7Nc1Kd5Hs3Jf6": "secret",
|
||||
} {
|
||||
r, ok := Check(s)
|
||||
if ok {
|
||||
t.Errorf("let %q leave", s)
|
||||
continue
|
||||
}
|
||||
if r.Class != class {
|
||||
t.Errorf("%q refused as %s, want %s", s, r, class)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The secret class alone (novox/hq ADR 0310 §4, issue 383): a path and an address pass, a path's pieces are
|
||||
// judged between its slashes, and every named secret shape and a random run outside a path are refused.
|
||||
func TestCheckSecretRefusesASecretsShapeAndNothingElse(t *testing.T) {
|
||||
for text, may := range map[string]bool{
|
||||
"+ shares: photos=/mnt/Photos_2024/Jochen": true,
|
||||
"+ sources: games=smb://nas.lan/Recalbox_Games2024": true,
|
||||
"+ library: /srv/media/Series_Archive/Season01": true,
|
||||
"root:x:0:0::/root:/usr/bin/bash": true,
|
||||
"nameserver 192.168.1.1": true,
|
||||
"drwxr-x--- jochen users /home/jochen": true,
|
||||
"+ key: xK9mQ2vL8pR4tW7yB3nF6hJ1dG5sA0zC": false,
|
||||
"-----BEGIN OPENSSH PRIVATE KEY-----": false,
|
||||
"password=hunter2": false,
|
||||
"Authorization: token ghp_abcdefghijklmnopqrstuvwxyz0123456789": false,
|
||||
"eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiIxMjM0NTY3ODkwIn0": false,
|
||||
"sha 3f2a9c1e5b7d4f6a8c0e2b4d6f8a0c2e4b6d8f0a": false,
|
||||
} {
|
||||
r, ok := CheckSecret(text)
|
||||
if ok != may {
|
||||
t.Errorf("CheckSecret(%q) = %v (%s), want %v", text, ok, r, may)
|
||||
}
|
||||
if !ok && r.Class != "secret" {
|
||||
t.Errorf("CheckSecret(%q) refused as %s: only the secret class is refused there", text, r)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The review of mesh-sdk #23: the shapes node-reads meets in env files, unit files, configurations and
|
||||
// history files, each refused by its shape whatever the value looks like.
|
||||
func TestCheckSecretRefusesTheShapesOfConfigurationFiles(t *testing.T) {
|
||||
for _, text := range []string{
|
||||
"url = nats://user:pa55word@host:4222",
|
||||
"DATABASE_URL=postgres://app:hunter2@db/app",
|
||||
"MESH_TOKEN=s3cr3t-v4lue",
|
||||
"DB_PASS=hunter2",
|
||||
"export GITEA_SECRET=abc",
|
||||
"Environment=APP_API_KEY=xyz",
|
||||
"SUDO_PASSWORD hunter2",
|
||||
"aws_secret_access_key = wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY",
|
||||
"password = hunter2",
|
||||
"wifi_psk: correcthorse",
|
||||
"value: wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY",
|
||||
} {
|
||||
if r, ok := CheckSecret(text); ok || r.Class != "secret" {
|
||||
t.Errorf("CheckSecret(%q) let it through (%v)", text, r)
|
||||
}
|
||||
}
|
||||
// What a configuration file says that is not a secret.
|
||||
for _, text := range []string{
|
||||
"ExecStart=/usr/lib/systemd/systemd-resolved",
|
||||
"UUID=3f2a9c1e-5b7d-4f6a-8c0e-2b4d6f8a0c2e /home btrfs defaults 0 0",
|
||||
"password required pam_unix.so",
|
||||
"bypass = true",
|
||||
"PasswordAuthentication no",
|
||||
"url = https://git.example.org/novox/mesh-sdk.git",
|
||||
"HOME=/var/lib/mesh-reads",
|
||||
"TOKEN_FILE=/var/lib/app/token",
|
||||
} {
|
||||
if r, ok := CheckSecret(text); !ok {
|
||||
t.Errorf("CheckSecret(%q) withheld it: %s", text, r)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Random values as base64 and base64url, wherever their slashes fall: every one is refused.
|
||||
func TestCheckSecretRefusesRandomValuesAsBase64(t *testing.T) {
|
||||
rng := rand.New(rand.NewSource(418))
|
||||
passed := map[string]int{}
|
||||
for i := 0; i < 1000; i++ {
|
||||
for _, n := range []int{16, 32} {
|
||||
b := make([]byte, n)
|
||||
rng.Read(b)
|
||||
for name, enc := range map[string]*base64.Encoding{"std": base64.StdEncoding, "url": base64.RawURLEncoding} {
|
||||
v := enc.EncodeToString(b)
|
||||
if _, ok := CheckSecret("key " + v); ok {
|
||||
passed[fmt.Sprintf("%s/%d", name, n)]++
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(passed) > 0 {
|
||||
t.Errorf("random values passed as clean, of 1000 each: %v", passed)
|
||||
}
|
||||
}
|
||||
|
||||
// A CamelCase name is a word, not a value: the case changes once a word.
|
||||
func TestCheckSecretLetsACamelCaseNameThrough(t *testing.T) {
|
||||
for _, text := range []string{"PasswordAuthenticationIsOff", "ChallengeResponseAuthentication no", "KbdInteractiveAuthentication"} {
|
||||
if r, ok := CheckSecret(text); !ok {
|
||||
t.Errorf("CheckSecret(%q) withheld it: %s", text, r)
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user