mesh-catalog/modules/betterbird/cmd/betterbird-tools/betterbird.go

package main

// Betterbird as the tools see it. Betterbird is the mail client's one process tree: the main process
// (comm betterbird-bin, no -contentproc) owns every window — the mail window, the calendar's reminder
// dialog, the composer — and its children (-contentproc) render add-ons and media. The tools read only:
//   - the account's processes, and of the main process where its output goes (fd 1 and 2) and which
//     profile it holds open;
//   - profiles.ini, to find the profile, and the profile's prefs.js and user.js, of which only the
//     preferences asked for are answered and never one that can hold a credential or names an account
//     (passwordSecret, accountPrefix);
//   - the profile's extensions.json, for the add-ons' ids and whether each is on;
//   - the module's settings as the node-engine writes them: prefs (~/.config/betterbird/prefs.json, prefs.go),
//     accounts (~/.config/betterbird/accounts.json, accounts.go), and identities, calendars, address-books,
//     filters and tags (configuration.json and tags.json, configuration.go);
//   - of logins.json, only the server name each kept sign-in is for, which Betterbird keeps in the clear,
//     to count the declared accounts that have not signed in yet (novox/hq ADR 0316 §16).
//
// Never read: the mail store, the address books' contacts, a user name or password in logins.json, key4.db,
// cert9.db, cookies, the calendar's events and caches. The operator's own configuration is answered with its
// values (novox/hq ADR 0329 §1), never a credential (§2).
//
// The files a tool writes in the profile are user.js, and only the module's own (betterbird_user_js with apply,
// betterbird_restart, and the start from the launcher); and, only while Betterbird is not running, the module's
// own keys in the lists of prefs.js, the removals of what it created, and each declared account's
// msgFilterRules.dat (accounts.go, owned.go). A user.js that is not the module's is never overwritten.

import (
	"bufio"
	"bytes"
	"errors"
	"fmt"
	"os"
	"path/filepath"
	"regexp"
	"sort"
	"strconv"
	"strings"
	"time"
)

const (
	mainComm = "betterbird-bin"
	// startBin is Betterbird itself, where the mesh's release and the community package both put it. The
	// module's start (/usr/local/bin/betterbird) runs the preparation first; a tool that starts Betterbird
	// prepares itself and starts this.
	startBin = installDir + "/betterbird"
	// packageFor is the community package the mesh's release replaces (novox/hq ADR 0316 §4).
	packageFor = "betterbird-bin"
	appIni     = "/opt/betterbird/application.ini"
	restartAs  = "mesh-betterbird"
	testUnit   = "mesh-betterbird-reminder-test"
	// profilesDir is where Betterbird keeps its profiles: Thunderbird's place, which Betterbird shares.
	profilesDir = ".thunderbird"
	// mark is the first line of the module's user.js: how a user.js is known to be the module's.
	mark = "// Betterbird's user.js (module betterbird"
)

// readLink is how a process's file descriptors are read; tests replace it, so no test makes a link.
var readLink = os.Readlink

// mainProcs are Betterbird's main processes: comm betterbird-bin without -contentproc.
func (m *Machine) mainProcs() []Proc {
	var out []Proc
	for _, p := range m.procs(mainComm) {
		// The reminder test's Betterbird is never the operator's: its command line names its own profile.
		if !strings.Contains(p.Command, "-contentproc") && !strings.Contains(p.Command, testUnit) {
			out = append(out, p)
		}
	}
	return out
}

// Stream is where a process's standard output or error goes.
type Stream struct {
	FD   int    `json:"fd"`
	Goes string `json:"goes"`
	// Dead is a pipe no process of the account reads: a write there fails.
	Dead bool `json:"dead,omitempty"`
}

// streams says where fd 1 and 2 of a process go.
func (m *Machine) streams(pid int) []Stream {
	var out []Stream
	for _, fd := range []int{1, 2} {
		target, err := readLink(m.path(fmt.Sprintf("/proc/%d/fd/%d", pid, fd)))
		s := Stream{FD: fd}
		switch {
		case err != nil:
			s.Goes = "unreadable"
		case target == "/dev/null":
			s.Goes = "discarded (/dev/null)"
		case strings.HasPrefix(target, "socket:"):
			s.Goes = "a socket (the journal, when started by the module)"
		case strings.HasPrefix(target, "pipe:"):
			if r := m.pipeReaders(target, pid); len(r) > 0 {
				s.Goes = "a pipe read by " + strings.Join(r, ", ")
			} else {
				s.Goes, s.Dead = "a pipe nobody reads", true
			}
		default:
			s.Goes = m.tilde(target)
		}
		out = append(out, s)
	}
	return out
}

// pipeReaders are the account's processes holding the pipe open for reading (O_RDONLY in fdinfo).
func (m *Machine) pipeReaders(pipe string, except int) []string {
	entries, _ := os.ReadDir(m.path("/proc"))
	seen := map[string]bool{}
	var out []string
	for _, e := range entries {
		pid, err := strconv.Atoi(e.Name())
		if err != nil || pid == except {
			continue
		}
		dir := m.path(filepath.Join("/proc", e.Name()))
		if m.uidOf(dir) != m.UID {
			continue
		}
		fds, _ := os.ReadDir(filepath.Join(dir, "fd"))
		for _, fd := range fds {
			if t, err := readLink(filepath.Join(dir, "fd", fd.Name())); err != nil || t != pipe {
				continue
			}
			if readOnly(readTrimmed(filepath.Join(dir, "fdinfo", fd.Name()))) {
				name := fmt.Sprintf("%s (pid %d)", readTrimmed(filepath.Join(dir, "comm")), pid)
				if !seen[name] {
					seen[name] = true
					out = append(out, name)
				}
			}
		}
	}
	sort.Strings(out)
	return out
}

// readOnly reads an fdinfo's flags (octal): the access mode is its low two bits, 0 for reading.
func readOnly(fdinfo string) bool {
	for _, l := range strings.Split(fdinfo, "\n") {
		if f := strings.Fields(l); len(f) == 2 && f[0] == "flags:" {
			n, err := strconv.ParseInt(f[1], 8, 64)
			return err == nil && n&3 == 0
		}
	}
	return false
}

// iniSections reads an INI file into its sections, in order.
func iniSections(raw []byte) (names []string, sections map[string]map[string]string) {
	sections = map[string]map[string]string{}
	cur := ""
	s := bufio.NewScanner(bytes.NewReader(raw))
	for s.Scan() {
		l := strings.TrimSpace(s.Text())
		switch {
		case l == "" || strings.HasPrefix(l, ";") || strings.HasPrefix(l, "#"):
		case strings.HasPrefix(l, "[") && strings.HasSuffix(l, "]"):
			cur = l[1 : len(l)-1]
			if _, ok := sections[cur]; !ok {
				names = append(names, cur)
				sections[cur] = map[string]string{}
			}
		default:
			if i := strings.IndexByte(l, '='); i > 0 && cur != "" {
				sections[cur][strings.TrimSpace(l[:i])] = strings.TrimSpace(l[i+1:])
			}
		}
	}
	return names, sections
}

// Profile is the profile the tools act on, and how it was found.
type Profile struct {
	Dir     string `json:"dir"`
	FoundBy string `json:"found_by"`
	path    string
}

// profile finds the profile: the one the running Betterbird holds open, else the installation's
// default in profiles.ini ([Install…] Default), else the profile marked Default=1.
func (m *Machine) profile() (Profile, error) {
	base := m.home(profilesDir)
	if ps := m.mainProcs(); len(ps) > 0 {
		counts := map[string]int{}
		fds, _ := os.ReadDir(m.path(fmt.Sprintf("/proc/%d/fd", ps[0].PID)))
		for _, fd := range fds {
			t, err := readLink(m.path(fmt.Sprintf("/proc/%d/fd/%s", ps[0].PID, fd.Name())))
			if err != nil {
				continue
			}
			rel, err := filepath.Rel(filepath.Join(m.Home, profilesDir), t)
			if err != nil || strings.HasPrefix(rel, "..") || !strings.Contains(rel, "/") {
				continue
			}
			counts[strings.SplitN(rel, "/", 2)[0]]++
		}
		best, n := "", 0
		for d, c := range counts {
			if c > n || (c == n && d < best) {
				best, n = d, c
			}
		}
		if best != "" {
			return Profile{Dir: m.tilde(filepath.Join(m.Home, profilesDir, best)), path: filepath.Join(base, best),
				FoundBy: fmt.Sprintf("held open by the running Betterbird (pid %d, %d files)", ps[0].PID, n)}, nil
		}
	}
	raw, err := readBounded(filepath.Join(base, "profiles.ini"))
	if err != nil {
		return Profile{}, fmt.Errorf("no profile: Betterbird is not running and ~/%s/profiles.ini is unreadable (%v)", profilesDir, err)
	}
	names, sections := iniSections(raw)
	pick := func(rel string, by string) (Profile, error) {
		dir := filepath.Join(base, filepath.FromSlash(rel))
		if filepath.IsAbs(rel) {
			dir = m.path(rel)
		}
		if !strings.HasPrefix(filepath.Clean(dir), filepath.Clean(base)+string(filepath.Separator)) {
			return Profile{}, fmt.Errorf("profiles.ini names a profile outside ~/%s: %s", profilesDir, rel)
		}
		return Profile{Dir: m.tilde(strings.TrimPrefix(dir, m.Root)), path: dir, FoundBy: by}, nil
	}
	var installs []string
	for _, n := range names {
		if strings.HasPrefix(n, "Install") && sections[n]["Default"] != "" {
			installs = append(installs, n)
		}
	}
	if len(installs) == 1 {
		return pick(sections[installs[0]]["Default"], "profiles.ini: the installation's default ["+installs[0]+"]")
	}
	for _, n := range names {
		if strings.HasPrefix(n, "Profile") && sections[n]["Default"] == "1" && sections[n]["Path"] != "" {
			return pick(sections[n]["Path"], "profiles.ini: the profile marked Default=1 ["+n+"]")
		}
	}
	return Profile{}, fmt.Errorf("profiles.ini names %d installation defaults and no profile marked default: start Betterbird once, then ask again", len(installs))
}

// prefLine is one user_pref(…) or pref(…) line of prefs.js or user.js.
var prefLine = regexp.MustCompile(`^\s*(?:user_)?pref\(\s*"((?:[^"\\]|\\.)*)"\s*,\s*(.*?)\s*\)\s*;\s*$`)

// readPrefs reads the preferences of a prefs.js or user.js: name to its value, as written.
func readPrefs(raw []byte) map[string]string {
	out := map[string]string{}
	s := bufio.NewScanner(bytes.NewReader(raw))
	s.Buffer(make([]byte, 64<<10), 4<<20)
	for s.Scan() {
		if mm := prefLine.FindStringSubmatch(s.Text()); mm != nil {
			out[mm[1]] = mm[2]
		}
	}
	return out
}

// passwordSecret and accountPrefix say which preferences are never answered with their value: one
// whose name can hold a credential, and one that names an account, a server or a person.
var (
	passwordSecret = regexp.MustCompile(`(?i)(password|passwd|secret|token|oauth|auth|cookie|credential|signon|apikey|api_key|privatekey|private_key|\.key$)`)
	accountPrefix  = []string{"mail.server.", "mail.identity.", "mail.smtpserver.", "mail.smtp.", "mail.account.",
		"mail.accountmanager.", "ldap_2.", "extensions.owl.", "calendar.registry.", "mail.root.",
		"services.sync.", "mail.displayname", "mail.default_", "mail.last_msg", "messenger.account.", "mail.cloud_files."}
	// allowedExtensions are the only extensions.* preferences answered: the add-ons' own preferences can
	// hold anything, and a theme's id is what a window's look depends on.
	allowedExtensions = map[string]bool{"extensions.activeThemeID": true}
	// mailnewsAccount is a mailnews.* name that holds a server, an account, an identity or the operator's own
	// tags: only those are withheld of mailnews.*, so a setting such as mailnews.start_page.enabled or
	// mailnews.default_view_flags is answered (novox/hq issue 345).
	mailnewsAccount = regexp.MustCompile(`(?i)^mailnews\..*(server|account|ident|smtp|ldap|imap|pop3|nntp|tags\.|email|addr)`)
)

// accountOwned are the preferences of an account and of the account lists: the setting accounts' alone, so
// the setting prefs refuses them and an account has one owner (novox/hq ADR 0316 §11).
var accountOwned = []string{"mail.account.", "mail.identity.", "mail.server.", "mail.smtpserver.",
	"mail.accountmanager.", "mail.smtpservers", "mail.smtp.defaultserver"}

// withheld says why a preference's value is never answered, or "" when it may be.
func withheld(name string) string {
	if strings.HasPrefix(name, "extensions.") && !allowedExtensions[name] {
		return "an add-on's own preference: only the theme's id (extensions.activeThemeID) is answered"
	}
	if passwordSecret.MatchString(name) {
		return "its name says it can hold a credential"
	}
	for _, p := range accountPrefix {
		if strings.HasPrefix(name, p) {
			return "it names an account, a server, a calendar or a person"
		}
	}
	if mailnewsAccount.MatchString(name) {
		return "it names an account, a server, a calendar or a person"
	}
	return ""
}

// Pref is one preference as the profile holds it.
type Pref struct {
	Name string `json:"name"`
	// Value is what Betterbird uses: the policy's lock, else user.js's at the start, else prefs.js's, else the
	// default, when a defaults file gives it.
	Value string `json:"value,omitempty"`
	From  string `json:"from,omitempty"`
	// Default is the default Betterbird starts with, and the file that gives it: Betterbird's own (omni.ja), or
	// the module's (novox/hq issue 345). Absent for a preference compiled into Betterbird.
	Default     string `json:"default,omitempty"`
	DefaultFrom string `json:"default_from,omitempty"`
	Withheld    string `json:"withheld,omitempty"`
}

// mostPrefs bounds how many preferences one answer carries.
const mostPrefs = 200

// Prefs answers the named preferences (a name, or a prefix ending in . or *) from the profile.
func (m *Machine) Prefs(names []string) (map[string]any, error) {
	if len(names) == 0 {
		return nil, errors.New("names is required: preferences by name, or a prefix ending in . or *, e.g. calendar.alarms.")
	}
	for _, n := range names {
		if strings.Trim(strings.TrimSpace(n), ".*") == "" {
			return nil, fmt.Errorf("%q asks for every preference: name one, or a prefix such as calendar.alarms.", n)
		}
	}
	p, err := m.profile()
	if err != nil {
		return nil, err
	}
	prefsJS, _ := readBounded(filepath.Join(p.path, "prefs.js"))
	userJS, _ := readBounded(filepath.Join(p.path, "user.js"))
	fromPrefs, fromUser := readPrefs(prefsJS), statementsOf(userJS)
	own, ownErr := m.defaults(true)
	locked := m.locks()
	all := map[string]bool{}
	for n := range fromPrefs {
		all[n] = true
	}
	for n := range fromUser {
		all[n] = true
	}
	var out []Pref
	cut := false
	seen := map[string]bool{}
	for _, want := range names {
		prefix := strings.HasSuffix(want, ".") || strings.HasSuffix(want, "*")
		want = strings.TrimSuffix(want, "*")
		var matched []string
		if prefix {
			for n := range all {
				if strings.HasPrefix(n, want) {
					matched = append(matched, n)
				}
			}
			sort.Strings(matched)
		} else {
			matched = []string{want}
		}
		for _, n := range matched {
			if seen[n] {
				continue
			}
			seen[n] = true
			if len(out) == mostPrefs {
				cut = true
				break
			}
			pr := Pref{Name: n}
			d, hasDefault := own[n]
			v, from := fromPrefs[n], "prefs.js"
			if l, ok := locked[n]; ok {
				v, from = l, "locked by the module's policy"
			} else if u, ok := fromUser[n]; ok {
				v, from = u, "user.js (wins at the start)"
			} else if _, ok := fromPrefs[n]; !ok {
				switch {
				case hasDefault:
					v, from = d.Value, "not set: "+d.From
				default:
					v, from = "", "not set: Betterbird's built-in default (in no defaults file)"
				}
			}
			pr.From = from
			if hasDefault {
				pr.DefaultFrom = d.From
			}
			if why := withheld(n); why != "" {
				pr.Withheld = why
			} else if allowedExtensions[n] {
				pr.Value = v
				pr.Default = d.Value
			} else {
				pr.Value = m.maskValue(v)
				pr.Default = m.maskValue(d.Value)
			}
			out = append(out, pr)
		}
	}
	answer := map[string]any{"profile": p, "prefs": out}
	if ownErr != nil {
		answer["defaults_problem"] = ownErr.Error()
	}
	if cut {
		answer["cut"] = fmt.Sprintf("the first %d; ask for a narrower prefix", mostPrefs)
	}
	return answer, nil
}

// statement is the one thing a line of the module's user.js may be, beside a comment.
var statement = regexp.MustCompile(`^user_pref\("[A-Za-z0-9_.\-\[\]]+",\s*(true|false|-?[0-9]+|"(?:[^"\\]|\\.)*")\);$`)

// validUserJS says what is wrong with a user.js the module would place, or nil. It holds comments and
// user_pref(…) statements only — the setting is one line, so several statements may share one — and
// sets no preference whose name could hold a credential.
func validUserJS(text string) error {
	if !strings.HasPrefix(text, mark) {
		return fmt.Errorf("it does not begin with the module's mark (%q)", mark)
	}
	for i, line := range strings.Split(text, "\n") {
		line = strings.TrimSpace(line)
		if line == "" || strings.HasPrefix(line, "//") {
			continue
		}
		for _, st := range splitStatements(line) {
			if st == "" || strings.HasPrefix(st, "//") {
				continue
			}
			if !statement.MatchString(st) {
				return fmt.Errorf("line %d holds %q, which is not user_pref(\"name\", value);", i+1, st)
			}
			name := st[len(`user_pref("`):strings.Index(st, `",`)]
			if passwordSecret.MatchString(name) {
				return fmt.Errorf("line %d sets %s, a preference that can hold a credential: not through the mesh", i+1, name)
			}
			for _, p := range accountOwned {
				if strings.HasPrefix(name, p) {
					return fmt.Errorf("line %d sets %s, an account's preference: an account is declared in the setting "+
						"accounts, never in prefs (novox/hq ADR 0316 §11)", i+1, name)
				}
			}
			for _, p := range configurationOwned {
				if strings.HasPrefix(name, p) {
					return fmt.Errorf("line %d sets %s, which the setting calendars, address-books or tags owns, never "+
						"prefs (novox/hq ADR 0329 §6)", i+1, name)
				}
			}
		}
	}
	return nil
}

// splitStatements cuts a line after each `);`, outside a string.
func splitStatements(line string) []string {
	var out []string
	in, esc, start := false, false, 0
	for i := 0; i < len(line); i++ {
		c := line[i]
		switch {
		case esc:
			esc = false
		case c == '\\' && in:
			esc = true
		case c == '"':
			in = !in
		case c == ';' && !in:
			out = append(out, strings.TrimSpace(line[start:i+1]))
			start = i + 1
		}
	}
	if rest := strings.TrimSpace(line[start:]); rest != "" {
		out = append(out, rest)
	}
	return out
}

// UserJS is the module's user.js against the profile's.
type UserJS struct {
	Declared   string `json:"declared"`
	Present    bool   `json:"declared_present"`
	Valid      string `json:"declared_problem,omitempty"`
	Profile    string `json:"profile_user_js"`
	Placed     string `json:"placed"`
	Statements int    `json:"statements"`
	// Effective says whether the running Betterbird started after the file was placed.
	Effective string `json:"effective,omitempty"`
	Content   string `json:"content,omitempty"`
	Applied   bool   `json:"applied,omitempty"`
	// Accounts are how many accounts the setting accounts declares, and what is wrong with them.
	Accounts        int    `json:"declared_accounts"`
	AccountsProblem string `json:"accounts_problem,omitempty"`
	// Each is every declared account by its place: adopted, created or refused (with why), its keys, and the
	// preferences an apply changed for it, by name (novox/hq ADR 0317 §3, §4).
	Each []resolvedAccount `json:"accounts,omitempty"`
	// Lists says whether the keys of the accounts the module created were written into the account lists.
	Lists string `json:"account_lists,omitempty"`
	// Configuration is every declared identity, calendar, address book and tag by its name: adopted, created or
	// refused (with why), and its key (novox/hq ADR 0329 §7).
	Configuration        []resolvedEntry `json:"configuration,omitempty"`
	ConfigurationProblem string          `json:"configuration_problem,omitempty"`
	// Filters are the declared accounts' filter files.
	Filters []filterPlan `json:"filters,omitempty"`
	// Closed says what was written of what Betterbird rewrites itself, or why it waits (§9).
	Closed string `json:"written_while_closed,omitempty"`
	// Waiting counts, per kind, the changes that wait for Betterbird to close.
	Waiting map[string]int `json:"waiting_for_betterbird_to_close,omitempty"`
}

// refusedAccounts are the reasons of the declared accounts that were refused.
func (u UserJS) refusedAccounts() []string {
	var out []string
	for _, r := range u.Each {
		if r.How == refused {
			out = append(out, r.Why)
		}
	}
	return out
}

// userJS compares the user.js the module places — the declared file, then the declared accounts' fields under
// the keys each was adopted or created under (adoption.go) — with the profile's; apply places it when the
// profile has none or the module's, records what was adopted or created, and writes the keys of the accounts
// it created into the profile's account lists when Betterbird is not running. A refused account gets nothing;
// the others go ahead (novox/hq ADR 0317 §7).
func (m *Machine) userJS(apply bool) (UserJS, error) {
	a := UserJS{Declared: "~/" + prefsFile}
	declaredText, present, derr := m.declaredText()
	a.Present = present
	if derr != nil {
		a.Present, a.Valid = true, derr.Error()
	}
	p, perr := m.profile()
	if perr != nil {
		return a, perr
	}
	target := filepath.Join(p.path, "user.js")
	a.Profile = m.tilde(strings.TrimPrefix(target, m.Root))
	accounts, _, aerr := m.readAccounts()
	merged := mergedPrefs(p.path)
	if aerr != nil {
		a.AccountsProblem = aerr.Error()
		accounts = nil
	} else {
		a.Accounts = len(accounts)
		a.Each = resolveAccounts(accounts, merged, m.readRecord(p.Dir))
	}
	conf, cerr := m.readConfiguration()
	if cerr != nil {
		a.ConfigurationProblem = cerr.Error()
	}
	plan := m.planConfiguration(p, conf, accounts, a.Each, merged)
	a.Configuration, a.Filters = plan.Entries, plan.filters
	a.Waiting = m.waiting(p.path, plan, a.Each)
	var want []byte
	if a.Present && a.Valid == "" {
		if verr := validUserJS(declaredText); verr != nil {
			a.Valid = verr.Error()
		}
		want = []byte(composeUserJS(declaredText, accounts, a.Each) + plan.configBlock())
		a.Content = m.answeredUserJS(string(want))
		for _, line := range strings.Split(string(want), "\n") {
			for _, st := range splitStatements(strings.TrimSpace(line)) {
				if statement.MatchString(st) {
					a.Statements++
				}
			}
		}
	}
	have, herr := readBounded(target)
	switch {
	case herr != nil:
		a.Placed = "none: the profile has no user.js"
	case a.Present && bytes.Equal(have, want):
		a.Placed = "the module's, as declared"
	case bytes.HasPrefix(have, []byte(mark)):
		a.Placed = "the module's, from an earlier declaration"
	default:
		a.Placed = "not the module's: a user.js of the operator's, which the module never overwrites"
	}
	if apply {
		switch {
		case !a.Present:
			return a, fmt.Errorf("the setting prefs is not on this machine (~/%s): the module is not pushed here yet", prefsFile)
		case a.Valid != "":
			return a, fmt.Errorf("the module's user.js is refused: %s; change the setting prefs", a.Valid)
		case a.AccountsProblem != "":
			return a, fmt.Errorf("nothing was placed: %s", a.AccountsProblem)
		case a.ConfigurationProblem != "":
			return a, fmt.Errorf("nothing was placed: %s", a.ConfigurationProblem)
		}
		saved, _ := readBounded(filepath.Join(p.path, "prefs.js"))
		savedPrefs := readPrefs(saved)
		for i, r := range a.Each {
			if r.How != refused {
				a.Each[i].Changed = changedPrefs(r, accounts[r.name], savedPrefs)
			}
		}
		switch {
		case herr == nil && !bytes.HasPrefix(have, []byte(mark)):
			return a, fmt.Errorf("%s is not the module's: move it aside (its preferences belong in the setting prefs), then apply again", a.Profile)
		}
		if herr != nil || !bytes.Equal(have, want) {
			tmp := target + ".mesh-new"
			if err := os.WriteFile(tmp, want, 0o644); err != nil {
				return a, err
			}
			if err := os.Rename(tmp, target); err != nil {
				os.Remove(tmp)
				return a, err
			}
		}
		a.Applied, a.Placed = true, "the module's, as declared"
		if err := m.writeRecord(p.Dir, a.Each); err != nil {
			return a, fmt.Errorf("the user.js is placed, and what was adopted or created could not be recorded: %v", err)
		}
		if lists, err := m.writeAccountLists(p.path, a.Each); err != nil {
			a.Lists = "not written: " + err.Error()
		} else {
			a.Lists = lists
		}
		if closed, err := m.writeClosed(p, &plan); err != nil {
			a.Closed = "not written: " + err.Error()
		} else {
			a.Closed = closed
		}
		a.Filters = plan.filters
		if err := m.writeConfigRecord(p.Dir, plan.record); err != nil {
			return a, fmt.Errorf("the user.js is placed, and what was adopted or created could not be recorded: %v", err)
		}
		a.Waiting = m.waiting(p.path, plan, a.Each)
	}
	if ps := m.mainProcs(); len(ps) > 0 {
		if st, err := os.Stat(target); err == nil {
			if t, err := time.Parse(time.RFC3339, ps[0].Since); err == nil && st.ModTime().After(t) {
				a.Effective = "not yet: Betterbird reads user.js when it starts, and it started before the file was placed (betterbird_restart)"
			} else if err == nil {
				a.Effective = "yes: the running Betterbird started after the file was placed"
			}
		}
	}
	return a, nil
}

// answeredUserJS is the user.js as a tool answers it (novox/hq ADR 0316 §16, ADR 0317 §9): an account's
// preferences with their values withheld, every other value masked as betterbird_prefs masks it.
func (m *Machine) answeredUserJS(text string) string {
	lines := strings.Split(text, "\n")
	for i, line := range lines {
		for _, st := range splitStatements(strings.TrimSpace(line)) {
			mm := prefLine.FindStringSubmatch(st)
			if mm == nil {
				continue
			}
			value := m.maskValue(mm[2])
			if why := withheld(mm[1]); why != "" {
				value = `"<withheld>"`
			}
			lines[i] = fmt.Sprintf("user_pref(%s, %s);", quoted(mm[1]), value)
		}
	}
	return strings.Join(lines, "\n")
}

// Prepare is what the module's start runs before Betterbird (`betterbird-tools prepare`): with no Betterbird
// of this account running, the user.js placed and the declared accounts written into the account lists. With
// one running it does nothing, since the start then only hands over to it. Its error is said by the start,
// which starts Betterbird either way.
func (m *Machine) Prepare() error {
	if len(m.mainProcs()) > 0 {
		return nil
	}
	u, err := m.userJS(true)
	if err != nil {
		return err
	}
	var problems []string
	if strings.HasPrefix(u.Lists, "not written") {
		problems = append(problems, "the account lists were "+u.Lists)
	}
	problems = append(problems, u.refusedAccounts()...)
	if strings.HasPrefix(u.Closed, "not written") {
		problems = append(problems, "what Betterbird rewrites itself was "+u.Closed)
	}
	// Said by kind, name and key only: the start's words reach the journal, which holds no address (ADR 0329 §1).
	for _, e := range u.Configuration {
		if e.How == refused {
			problems = append(problems, fmt.Sprintf("%s %s is refused: %s", e.Kind, e.Name, e.Why))
		}
	}
	for _, fp := range u.Filters {
		if fp.Refused != "" {
			problems = append(problems, fmt.Sprintf("the filters of %s are refused: %s", fp.Account, fp.Refused))
		}
	}
	if len(problems) > 0 {
		return fmt.Errorf("%s", strings.Join(problems, "; "))
	}
	return nil
}

// Status is what betterbird_status answers.
type Status struct {
	Running  []Proc `json:"running"`
	Children int    `json:"children"`
	// Installation is what is installed and from where, the policy and Owl (novox/hq ADR 0316 §16).
	Installation Installation `json:"installation"`
	// Installed is the community package's version, while it is still there.
	Installed string   `json:"community_package,omitempty"`
	Foreign   bool     `json:"from_the_aur,omitempty"`
	Profile   *Profile `json:"profile,omitempty"`
	Problem   string   `json:"profile_problem,omitempty"`
	Output    []Stream `json:"output,omitempty"`
	UserJS    *UserJS  `json:"user_js,omitempty"`
	// Accounts are counts: the declared, those in the profile's list, the others, those without a sign-in.
	Accounts *AccountCount `json:"accounts,omitempty"`
	AddOns   []AddOn       `json:"add_ons,omitempty"`
	// Display are the preferences that decide how its windows and notifications look.
	Display []Pref `json:"display_prefs,omitempty"`
}

// AddOn is one add-on the operator installed: its id, and whether it is on.
type AddOn struct {
	ID      string `json:"id"`
	Version string `json:"version,omitempty"`
	Active  bool   `json:"active"`
}

// displayPrefs are the preferences betterbird_status answers: the reminder dialog, the theme, the
// renderer, and whether notifications go to the desktop's notifier.
var displayPrefs = []string{"calendar.alarms.playsound", "calendar.alarms.show", "calendar.alarms.showmissed",
	"calendar.alarms.indicator.show", "alerts.useSystemBackend", "mail.biff.use_system_alert",
	"extensions.activeThemeID", "browser.display.use_system_colors", "ui.systemUsesDarkTheme",
	"gfx.webrender.software", "gfx.webrender.all", "layers.acceleration.disabled", "widget.gtk.overlay-scrollbars.enabled",
	"mozilla.widget.use-argb-visuals"}

func (m *Machine) foreign(pkg string) bool {
	o := m.cmd(0, nil, "pacman", "-Qmq", pkg)
	return o.Err == nil && o.Code == 0 && strings.TrimSpace(o.Stdout) == pkg
}

// Status answers whether Betterbird runs, what is installed, its profile and the module's user.js.
func (m *Machine) Status() (Status, error) {
	s := Status{Running: m.mainProcs()}
	if s.Running == nil {
		s.Running = []Proc{}
	}
	s.Children = len(m.procs(mainComm)) - len(s.Running)
	if v, err := m.installed(packageFor); err == nil && v != "" {
		s.Installed, s.Foreign = v, m.foreign(packageFor)
	}
	if len(s.Running) > 0 {
		s.Output = m.streams(s.Running[0].PID)
	}
	p, err := m.profile()
	if err != nil {
		s.Installation = m.installation("")
		s.Problem = err.Error()
		return s, nil
	}
	s.Profile = &p
	s.Installation = m.installation(p.path)
	if u, err := m.userJS(false); err == nil {
		u.Content = ""
		s.UserJS = &u
	}
	c := m.countAccounts(p.path, p.Dir)
	s.Accounts = &c
	s.AddOns = addOns(p.path)
	if got, err := m.Prefs(displayPrefs); err == nil {
		s.Display = got["prefs"].([]Pref)
	}
	return s, nil
}

// emailAddress and credentialWord are masked in what Betterbird printed.
var (
	emailAddress   = regexp.MustCompile(`[A-Za-z0-9._%+\-]+@[A-Za-z0-9.\-]+\.[A-Za-z]{2,}`)
	credentialWord = regexp.MustCompile(`(?i)((?:password|passwd|token|secret|authorization|cookie)\S*\s*[=:]\s*)\S+`)
)

// userAtHost and hostName are what a preference's value may name a server by.
var (
	userAtHost = regexp.MustCompile(`[A-Za-z0-9._%+\-]+%40[A-Za-z0-9.\-]+@[A-Za-z0-9.\-]+`)
	// A host is a name of three labels or more, or of two whose last is a common top-level domain: a
	// file's name (a.pdf) is not one.
	hostName      = regexp.MustCompile(`(?i)\b(?:(?:[a-z0-9](?:[a-z0-9-]*[a-z0-9])?\.){2,}[a-z]{2,}|[a-z0-9](?:[a-z0-9-]*[a-z0-9])?\.(?:com|org|net|be|nl|de|fr|eu|io|uk|us|info|biz|dev|app|cloud|local|lan|internal|home|corp))\b`)
	bearerOrBasic = regexp.MustCompile(`(?i)\b(bearer|basic)\s+[A-Za-z0-9._~+/=\-]+`)
)

func mask(s string) string {
	s = userAtHost.ReplaceAllString(s, "<hidden>")
	s = emailAddress.ReplaceAllString(s, "<hidden>")
	s = bearerOrBasic.ReplaceAllString(s, "$1 <hidden>")
	return credentialWord.ReplaceAllString(s, "${1}<hidden>")
}

// maskValue is a preference's value as answered: the home as ~, and no address, user or host.
func (m *Machine) maskValue(v string) string {
	if m.Home != "" && m.Home != "/" {
		v = strings.ReplaceAll(v, strings.TrimSuffix(m.Home, "/")+"/", "~/")
	}
	v = mask(v)
	return hostName.ReplaceAllString(v, "<host>")
}

// mostAnswer bounds a log answer.
const mostAnswer = 128 << 10

// Log answers the last n lines Betterbird printed, from the journal of the unit the module starts it
// as; problems keeps the lines that say error, warning, crash or assertion. Addresses and credentials
// are masked.
func (m *Machine) Log(n int, problems bool, unit string) (map[string]any, error) {
	if unit == "" {
		unit = restartAs
	}
	if unit != restartAs && unit != testUnit {
		return nil, fmt.Errorf("unit is %s or %s", restartAs, testUnit)
	}
	o := m.cmd(0, m.bus(), "journalctl", "--user", "--unit", unit+".service", "--no-pager", "--output", "short-iso", "-n", strconv.Itoa(n))
	if err := failed(o, "journalctl"); err != nil {
		return nil, err
	}
	var lines []string
	size := 0
	keep := regexp.MustCompile(`(?i)(error|warn|crash|assert|fail|exception)`)
	for _, l := range strings.Split(strings.TrimRight(o.Stdout, "\n"), "\n") {
		if l == "" || (problems && !keep.MatchString(l)) {
			continue
		}
		l = mask(l)
		if size+len(l) > mostAnswer {
			break
		}
		size += len(l) + 1
		lines = append(lines, l)
	}
	if lines == nil {
		lines = []string{}
	}
	a := map[string]any{"unit": unit + ".service", "lines": lines}
	if ps := m.mainProcs(); len(ps) > 0 && !strings.HasPrefix(ps[0].StartedIn, restartAs) && unit == restartAs {
		a["note"] = fmt.Sprintf("the running Betterbird (pid %d) was not started by the module: it runs in %s, "+
			"and its output goes where status says (output), not to this journal; betterbird_restart starts it as %s",
			ps[0].PID, ps[0].StartedIn, restartAs)
	}
	return a, nil
}

// RestartAnswer is what betterbird_restart answers.
type RestartAnswer struct {
	// Composers are the message windows that were open: what was unsent and is gone.
	Composers int     `json:"composers_discarded,omitempty"`
	Note      string  `json:"note,omitempty"`
	Ended     []int   `json:"ended"`
	Killed    []int   `json:"killed,omitempty"`
	UserJS    string  `json:"user_js"`
	Running   []Proc  `json:"running"`
	Session   Session `json:"session"`
	Unit      string  `json:"unit"`
}

// composerClass is the X instance of a message being written, whatever the class beside it.
var composerClass = regexp.MustCompile(`\("Msgcompose" "[^"]*"\)`)

// composers counts the open message windows, from the X window tree.
func (m *Machine) composers(s Session) (int, error) {
	o := m.cmd(3*time.Second, s.Env(), "xwininfo", "-root", "-tree")
	if err := failed(o, "xwininfo", "-root", "-tree"); err != nil {
		return 0, err
	}
	return len(composerClass.FindAllString(o.Stdout, -1)), nil
}

// Restart ends Betterbird (asked to quit, forced after 10 s), places the module's user.js when it can,
// and starts Betterbird again in the operator's session under the account's service manager, its output
// in the journal. While a message is being written it refuses, unless discard says the unsent may go.
// It answers within the runtime's 30 s: a start that has not shown after 4 s is said, and
// betterbird_status sees it come.
func (m *Machine) Restart(discard bool) (RestartAnswer, error) {
	s, err := m.session()
	if err != nil {
		return RestartAnswer{}, err
	}
	a := RestartAnswer{Session: s, Unit: restartAs + ".service"}
	n, err := m.composers(s)
	switch {
	case err != nil && !discard:
		return a, fmt.Errorf("whether a message is being written cannot be told (%v): restart with discard_unsent only if none is", err)
	case n > 0 && !discard:
		return a, fmt.Errorf("%d composer window(s) are open: an unsent message would be lost. Send or save it, or ask the operator, then call again with discard_unsent", n)
	}
	a.Composers = n
	a.Ended, a.Killed = m.stopProcs(8*time.Second, m.mainProcs())
	if u, err := m.userJS(true); err != nil {
		a.UserJS = "not placed: " + err.Error()
	} else {
		a.UserJS = fmt.Sprintf("placed: %d preference(s) from ~/%s, %d declared account(s); account lists %s",
			u.Statements, prefsFile, u.Accounts, u.Lists)
	}
	// As detach does, with shorter bounds: the worst case of the whole call (3 s for the window tree,
	// 8 s to quit, 3 + 5 s to start, 4 s to see it) stays under the runtime's 30 s.
	_ = m.cmd(3*time.Second, s.Env(), "systemctl", "--user", "stop", restartAs+".service")
	call := []string{"--user", "--collect", "--quiet", "--unit=" + restartAs}
	for _, kv := range [][2]string{{"DISPLAY", s.Display}, {"XAUTHORITY", s.XAuthority}} {
		if kv[1] != "" {
			call = append(call, "--setenv="+kv[0]+"="+kv[1])
		}
	}
	call = append(call, "--", startBin)
	if err := failed(m.cmd(5*time.Second, s.Env(), "systemd-run", call...), "systemd-run", call...); err != nil {
		return a, err
	}
	for waited := time.Duration(0); waited < 4*time.Second && len(m.mainProcs()) == 0; waited += 250 * time.Millisecond {
		m.Sleep(250 * time.Millisecond)
	}
	a.Running = m.mainProcs()
	if a.Running == nil {
		a.Running = []Proc{}
	}
	if len(a.Running) == 0 {
		a.Note = "started as " + a.Unit + "; no main process yet after 4 s: betterbird_status shows it once it runs, betterbird_log says why if it does not"
	}
	return a, nil
}

// CheckAnswer is what betterbird_check answers.
type CheckAnswer struct {
	OK       bool      `json:"ok"`
	Findings []Finding `json:"findings"`
}

// Check verifies what the module promises: Betterbird installed, one running in the session, its
// profile found, the module's user.js declared, valid, placed and in effect, its output read, and no ask at its
// start to be the default mail client (novox/hq issue 345).
func (m *Machine) Check() (CheckAnswer, error) {
	a := CheckAnswer{Findings: []Finding{}}
	add := func(what, do string) { a.Findings = append(a.Findings, Finding{what, do}) }
	if _, err := m.installed(packageFor); err != nil {
		return a, err
	}
	p, perr := m.profile()
	profilePath := ""
	if perr == nil {
		profilePath = p.path
	}
	in := m.installation(profilePath)
	switch {
	case in.From == "not installed":
		add("Betterbird is not installed in "+installDir, "the module's push places the mesh's release; it waits for the operator's push (upgrade policy record)")
	case in.From != "the mesh":
		add("Betterbird is "+in.From, "quit Betterbird, then push the module to this node: the push removes the community package and places the mesh's release")
	}
	if in.Behind {
		add("Betterbird "+in.Version+" (build "+in.Build+") is behind the newest published build "+in.Newest,
			"move the module's pin in a reviewed change (novox/hq ADR 0316 §5)")
	}
	if in.From == "the mesh" && !strings.HasPrefix(in.Policy, "placed:") {
		add("the enterprise policy is "+in.Policy, "push the module to this node")
	}
	if in.From == "the mesh" && !exists(m.path(meshDefaults)) {
		add("the module's defaults are not placed ("+meshDefaults+")", "push the module to this node")
	}
	if c, _, err := m.config(); err == nil {
		var cleared []string
		for _, cp := range c.Prefs {
			switch {
			case cp.Drift == "":
			case cp.Remedy != "":
				add(cp.Name+" drifts from the module's: "+cp.Drift, cp.Remedy)
			default:
				cleared = append(cleared, cp.Name)
			}
		}
		if len(cleared) > 0 {
			add(fmt.Sprintf("%d managed preference(s) drift from the module's defaults: the profile saved its own value and no "+
				"node layer says so (%s)", len(cleared), strings.Join(cleared, ", ")),
				"betterbird_config shows each; to keep a value on this node, betterbird_config with keep proposes a node "+
					"layer; otherwise quit Betterbird and call betterbird_config_apply with apply")
		}
		if len(c.Refused) > 0 {
			add(fmt.Sprintf("the module's defaults list %d preference(s) the tools never manage (%s)", len(c.Refused), strings.Join(c.Refused, "; ")),
				"take them out of install/defaults-pref/mesh-defaults.js in a reviewed catalogue change: an account is declared in the setting accounts")
		}
	}
	if why := m.defaultClientAsk(profilePath); why != "" && in.From != "not installed" {
		add("Betterbird asks at its start to be the default mail client: "+why,
			"push the module to this node: its policy locks "+defaultClientPref+" off (novox/hq issue 345)")
	}
	switch {
	case in.From != "the mesh":
	case in.Owl == nil || in.Owl.Pinned == "":
		add("Owl's xpi is not placed ("+owlFile+")", "push the module to this node")
	case in.Owl.InProfile == "":
		add("Owl is not installed in the profile yet", "Betterbird installs it from the policy at its next start")
	case !in.Owl.AtThePin && !newerVersion(in.Owl.InProfile, in.Owl.Pinned):
		add("Owl "+in.Owl.InProfile+" runs in the profile, older than the mesh's pin "+in.Owl.Pinned,
			"Owl is installed by policy and cannot be removed in Betterbird: move the module's Owl pin in a reviewed change (novox/hq ADR 0316 §6)")
	case !in.Owl.Active:
		add("Owl is installed and not active", "turn it on in Betterbird's add-ons")
	}
	if perr == nil {
		c := m.countAccounts(p.path, p.Dir)
		for _, r := range c.Each {
			if r.How == refused {
				add("refused: "+r.Why, "as it says; the other declared accounts go ahead")
			}
		}
		switch {
		case c.InList < c.Declared:
			add(fmt.Sprintf("%d of %d declared account(s) are not in the profile's account list yet", c.Declared-c.InList, c.Declared),
				"quit Betterbird and start it from the launcher, or betterbird_restart")
		}
		if c.WithoutSignIn > 0 {
			add(fmt.Sprintf("%d declared account(s) have not signed in yet", c.WithoutSignIn),
				"a person signs in once, in Betterbird's own window, when it asks (novox/hq ADR 0316 §14)")
		}
		// The five kinds of ADR 0329: drift and what waits for Betterbird to close, per kind (§9, §11, §12).
		if _, err := m.readConfiguration(); err != nil {
			add(err.Error(), "change the setting at the controller's terminal, or derive and propose it (betterbird_derive)")
		} else if v, err := m.view(); err == nil {
			drift := m.driftCounts(v)
			waiting := m.waiting(v.p.path, v.plan, v.ra)
			for _, k := range allKinds {
				if n := drift[k]; n > 0 {
					add(fmt.Sprintf("%d declared %s differ from the profile, are not in it yet or are refused", n, k),
						"betterbird_drift with kind "+k+" says which and how; the module's user.js holds them from Betterbird's next start (betterbird_restart)")
				}
				if n := waiting[k]; n > 0 {
					add(fmt.Sprintf("%d change(s) to %s are not written yet: they are written only while Betterbird is closed", n, k),
						"quit Betterbird and start it from the launcher, or betterbird_restart (novox/hq ADR 0329 §9)")
				}
			}
		}
	}
	ps := m.mainProcs()
	switch {
	case len(ps) == 0:
		add("Betterbird is not running", "start it from the launcher, or betterbird_restart")
	case len(ps) > 1:
		add(fmt.Sprintf("%d Betterbird main processes run", len(ps)), "one profile is held by one process: betterbird_restart")
	}
	u, err := m.userJS(false)
	switch {
	case err != nil:
		add("no profile: "+err.Error(), "start Betterbird once")
	case !u.Present:
		add("the setting prefs is not on this machine (~/"+prefsFile+")", "the module is not pushed here: the controller's push")
	case u.Valid != "":
		add("the module's user.js is refused: "+u.Valid, "change the module's setting prefs")
	case u.AccountsProblem != "":
		add("the module's user.js is not placed: "+u.AccountsProblem, "change the module's setting accounts at the controller's terminal, or propose it (betterbird_accounts)")
	case !strings.HasPrefix(u.Placed, "the module's, as declared"):
		add("the profile's user.js is "+u.Placed, "betterbird_user_js with apply, then betterbird_restart")
	case strings.HasPrefix(u.Effective, "not yet"):
		add("the module's user.js is placed but not in effect: "+u.Effective, "betterbird_restart")
	}
	if len(ps) > 0 {
		for _, s := range m.streams(ps[0].PID) {
			if s.Dead {
				add(fmt.Sprintf("Betterbird's fd %d is a pipe nobody reads (a session started before its output went to the journal): what it prints is lost", s.FD),
					"betterbird_restart, which starts it with its output in the journal")
			}
		}
	}
	a.OK = len(a.Findings) == 0
	return a, nil
}