package builder // **Every line of a check's output is redacted before it is kept** (novox/hq issue 462). // // A repository's own check prints into the build's log, which the bus keeps for days and anyone who may read // its events reads, and into the verdict, which the forge shows on the pull request. Software prints what it // was given — a URL carrying a password, a token in a flag — and a pull request may print on purpose. // So a line is said only after every secret the builder knows (the forge credential's password) and every // value whose shape says it is one is replaced by a mark naming what was there, as the journal verb does. // // Copied from the journal tool's redactor (mesh-catalog, modules/systemd/cmd/systemd-tools/secrets.go, // itself a copy of the docker module's), narrowed to a line's shapes, with the token shapes a check's output // may carry added. A third copy: sharing them through mesh-sdk is novox/hq issue 471. import ( "net/url" "regexp" "strings" ) // secretName is a variable name that says its value is a secret. var secretName = regexp.MustCompile(`(?i)(pass(word|wd|phrase)?|secret|token|api_?key|private_?key|access_?key|credential|auth)`) // notAValue is a name that says its value is where a secret is, not the secret: a file or a path. var notAValue = regexp.MustCompile(`(?i)(_FILE|FILE|_PATH|_DIR)$`) // uriPassword is a URI carrying a password in its userinfo: scheme://user:password@. var uriPassword = regexp.MustCompile(`[A-Za-z][A-Za-z0-9+.-]*://[^\s/:@'"]*:([^\s/@'"]+)@`) // tokenShaped are tokens recognised by their own prefix, whatever surrounds them: a forge's or a host's // access token, a JSON web token, a NATS seed. var tokenShaped = []struct { name string re *regexp.Regexp }{ {"an access token", regexp.MustCompile(`\b(gh[pousr]_[A-Za-z0-9]{20,}|github_pat_[A-Za-z0-9_]{20,}|glpat-[A-Za-z0-9_-]{20,}|xox[abpr]-[A-Za-z0-9-]{10,}|sk-ant-[A-Za-z0-9_-]{20,})`)}, {"a JSON web token", regexp.MustCompile(`\beyJ[A-Za-z0-9_-]{8,}\.eyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]+`)}, {"a NATS seed", regexp.MustCompile(`\bS[ACNOU][A-Z2-7]{56}\b`)}, } // masked is a password a program already hid: ***, xxx, , [REDACTED]. var masked = regexp.MustCompile(`^(\*+|x+|X+|<[^>]*>|\[[^\]]*\]|%2A+)$`) // ordinary is a value under a secret's name that is not one: a path, an address, a number, a switch. var ordinary = regexp.MustCompile(`^(/.*|[A-Za-z][A-Za-z0-9+.-]*://.*|[0-9.]+[a-z]?|(?i:true|false|yes|no|on|off|none|null))$`) // leastSecret is the shortest value compared as a secret: a shorter one matches ordinary words. const leastSecret = 6 // passwordFlags take a secret as their next word, or after `=`, whatever the program. var passwordFlags = map[string]bool{ "-P": true, "--password": true, "--pass": true, "--passwd": true, "--secret": true, "--secret-key": true, "--token": true, "--api-key": true, "--apikey": true, "--auth": true, } // knownSecret is one value the builder holds, by the name it is said under. type knownSecret struct { Name string Value string } // redactor hides the secrets it knows and those a line's shapes say are secrets. type redactor struct{ known []knownSecret } // redactorFor knows the forge credential's password, and its user's name with it, in every form git or a // program may print them. func redactorFor(forge GitCredential) redactor { var r redactor if forge.URL == "" { return r } for _, m := range uriPassword.FindAllStringSubmatch(forge.URL, -1) { r.add("the forge credential", m[1]) if dec, err := url.PathUnescape(m[1]); err == nil && dec != m[1] { r.add("the forge credential", dec) } } return r } func (r *redactor) add(name, value string) { if len(value) < leastSecret || masked.MatchString(value) { return } for _, k := range r.known { if k.Value == value { return } } r.known = append(r.known, knownSecret{name, value}) } // redact is a text with every known secret, every value its shape says is one, and every password inside a // URI replaced by a mark naming what was there. Line by line: a shape is judged within its line. func (r redactor) redact(text string) string { if !strings.ContainsAny(text, "\n") { return r.line(text) } lines := strings.Split(text, "\n") for i, l := range lines { lines[i] = r.line(l) } return strings.Join(lines, "\n") } func (r redactor) line(line string) string { replace := func(s knownSecret) { for _, f := range forms(s.Value) { line = strings.ReplaceAll(line, f, "[redacted: "+s.Name+"]") } } for _, s := range r.known { replace(s) } for _, s := range shaped(line) { replace(s) } line = uriPassword.ReplaceAllStringFunc(line, func(m string) string { sub := uriPassword.FindStringSubmatch(m) if masked.MatchString(sub[1]) || strings.HasPrefix(sub[1], "[redacted") { return m } return strings.TrimSuffix(m, sub[1]+"@") + "[redacted: a password in a URI]@" }) for _, t := range tokenShaped { line = t.re.ReplaceAllString(line, "[redacted: "+t.name+"]") } return line } // forms are the ways a value may appear printed: as given, and URL-encoded. func forms(value string) []string { out := []string{value} for _, f := range []string{url.QueryEscape(value), url.PathEscape(value)} { if f != value && !hasString(out, f) { out = append(out, f) } } return out } func hasString(list []string, s string) bool { for _, x := range list { if x == s { return true } } return false } // shaped are the values a line carries by their shape: the word after a password flag, or the value of one // given with `=`, and a NAME=value whose name says secret. func shaped(line string) []knownSecret { var out []knownSecret add := func(name, value string) { value = strings.Trim(value, `"',;`) if len(value) < leastSecret || masked.MatchString(value) || ordinary.MatchString(value) || strings.HasPrefix(value, "[redacted") { return } out = append(out, knownSecret{name, value}) } words := strings.Fields(line) for i, w := range words { if flag, value, ok := strings.Cut(w, "="); ok && strings.HasPrefix(flag, "-") { if passwordFlags[flag] { add("the value of "+flag, value) } continue } if name, value, ok := strings.Cut(w, "="); ok && name != "" && secretName.MatchString(name) && !notAValue.MatchString(name) && !strings.ContainsAny(name, "/:") { add("the value of "+name, value) continue } if i+1 < len(words) && passwordFlags[w] { add("the word after "+w, words[i+1]) } } return out } // withoutUserinfo is a URL with its userinfo left out, and whether it carried any. func withoutUserinfo(raw string) (string, bool) { u, err := url.Parse(raw) if err != nil || u.User == nil { return raw, false } u.User = nil return u.String(), true }