Compare commits
67
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
b1f2776693 | ||
|
|
f35b067504 | ||
|
|
93d58f3740 | ||
|
|
5cd608ba88 | ||
|
|
2f5e97030a | ||
|
|
bc3b59227f | ||
|
|
eb42806379 | ||
|
|
fe2040c65e | ||
|
|
53e5186496 | ||
|
|
e0bc919d49 | ||
|
|
f88e3452f8 | ||
|
|
827b89dbf1 | ||
|
|
c9a711c259 | ||
|
|
1835f0efad | ||
|
|
39e7a06e62 | ||
|
|
84911b94f3 | ||
|
|
4193eacec4 | ||
|
|
8f80cd4716 | ||
|
|
43a9155d5a | ||
|
|
c4eebce598 | ||
|
|
14999f1d3e | ||
|
|
e4e0ce7619 | ||
|
|
9446bec2f9 | ||
|
|
9c9a76d220 | ||
|
|
ac2d4dc46f | ||
|
|
d09dbe33ac | ||
|
|
005f802504 | ||
|
|
eb5f33f2b4 | ||
|
|
c4b4bac757 | ||
|
|
0863ec2a05 | ||
|
|
854c85393c | ||
|
|
2fe5601397 | ||
|
|
01d2ff7a46 | ||
|
|
961e57f7e4 | ||
|
|
22071dabfc | ||
|
|
481e44cdaa | ||
|
|
23ea0f431f | ||
|
|
41b6e10077 | ||
|
|
2f55adc57c | ||
|
|
434506f9b1 | ||
|
|
82cca650a2 | ||
|
|
cd2b6aacd8 | ||
|
|
aed7e0c968 | ||
|
|
43df7ebc5a | ||
|
|
c5e1bc7bb4 | ||
|
|
6190b40175 | ||
|
|
edf39a3b74 | ||
|
|
15983e2f96 | ||
|
|
263c86d2e5 | ||
|
|
ec4ab7afe7 | ||
|
|
0e6f382bb7 | ||
|
|
d745a4d31e | ||
|
|
994cab533b | ||
|
|
2772120758 | ||
|
|
13a7d40ac5 | ||
|
|
a3ad3c5061 | ||
|
|
5b75499ea6 | ||
|
|
0366208274 | ||
|
|
9f893b6305 | ||
|
|
d1baca9b94 | ||
|
|
3a5c86b5d6 | ||
|
|
befe451f6d | ||
|
|
7ad54356fd | ||
|
|
69f76a53e7 | ||
|
|
7e27259a64 | ||
|
|
f067fab9db | ||
|
|
4fdb047cc8 |
@@ -19,3 +19,5 @@ modules/mesh-issues/cmd/mesh-issues/mesh-issues
|
||||
modules/mounts/cmd/mounts/mounts
|
||||
modules/mesh-knowledge/cmd/mesh-knowledge/mesh-knowledge
|
||||
modules/mesh-knowledge/mesh-knowledge
|
||||
modules/mesh-reads/mesh-reads
|
||||
modules/mesh-reads/cmd/mesh-reads/mesh-reads
|
||||
|
||||
@@ -0,0 +1,3 @@
|
||||
module git.novox.be/novox/mesh-catalog/checks/secrets
|
||||
|
||||
go 1.22
|
||||
@@ -0,0 +1,4 @@
|
||||
# Secrets nested below a key a mergeable file takes for its other values, which the controller cannot
|
||||
# protect alone yet (it protects top-level keys): "<module> <resource> <dotted key>". This list only shrinks:
|
||||
# a line that no longer names a finding is refused, and so is one the trunk's copy does not hold (hq issue 342).
|
||||
searxng settings server.secret_key
|
||||
@@ -0,0 +1,244 @@
|
||||
// Command secrets refuses, in every module's manifest, a mergeable file that a setting could fill with a
|
||||
// password, a token or an API key in place of the vault's (novox/hq issue 342).
|
||||
//
|
||||
// A password is not a setting: it comes from the key vault, and it can be rotated there. A file that says
|
||||
// `"merge": "json"` takes the keys of the mesh-wide and the node's layers over its own content, so a key the
|
||||
// content fills with `${secret:<name>}` is one a layer could replace, and the vault's value would lose to it
|
||||
// until somebody removed it by hand. The controller refuses a setting of a key the file keeps as `protected`,
|
||||
// naming the layer and the key, never the value. This check holds every such file to that: each top-level key
|
||||
// whose value holds `${secret:…}`, anywhere below it, is either listed in the file's `protected`, or left out
|
||||
// of the file's `takes-settings` (novox/hq ADR 0326), so that no layer reaches it.
|
||||
//
|
||||
// A secret nested below a key the file takes for its other values (`server.secret_key` below `server`) cannot
|
||||
// be protected alone today: the controller protects top-level keys. Such a file is named in the file `known`
|
||||
// beside this check, one line per key, until the controller refuses a nested key. The list only shrinks: a
|
||||
// line there that no longer names a finding is refused, and so is a line the trunk's copy (the merge base's,
|
||||
// given with -base) does not hold. While the trunk holds no copy, the lines in seed are the trunk's.
|
||||
//
|
||||
// go run . [-base <the trunk's known>] <catalogue root>
|
||||
//
|
||||
// Exits 1 on a finding.
|
||||
package main
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"encoding/json"
|
||||
"flag"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Finding is one key of one mergeable file a layer could fill in place of the vault.
|
||||
type Finding struct {
|
||||
Module, ID, Key string
|
||||
}
|
||||
|
||||
func (f Finding) String() string { return f.Module + " " + f.ID + " " + f.Key }
|
||||
|
||||
// Check reads every manifest under root/modules and answers each secret-bearing key a layer could replace.
|
||||
func Check(root string) ([]Finding, error) {
|
||||
manifests, err := filepath.Glob(filepath.Join(root, "modules", "*", "module.json"))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
sort.Strings(manifests)
|
||||
var out []Finding
|
||||
for _, path := range manifests {
|
||||
raw, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var m struct {
|
||||
Module string `json:"module"`
|
||||
Resources []map[string]any `json:"resources"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &m); err != nil {
|
||||
return nil, fmt.Errorf("%s: %w", path, err)
|
||||
}
|
||||
for _, r := range m.Resources {
|
||||
if how, _ := r["merge"].(string); how == "" {
|
||||
continue
|
||||
}
|
||||
id, _ := r["id"].(string)
|
||||
content, _ := r["content"].(string)
|
||||
var doc map[string]any
|
||||
if strings.TrimSpace(content) == "" {
|
||||
continue
|
||||
}
|
||||
if err := json.Unmarshal([]byte(content), &doc); err != nil {
|
||||
return nil, fmt.Errorf("%s: %s merges as JSON and its content is not JSON: %w", m.Module, id, err)
|
||||
}
|
||||
protected := set(r["protected"])
|
||||
takes, names := r["takes-settings"]
|
||||
taken := set(takes)
|
||||
for _, key := range sortedKeys(doc) {
|
||||
for _, at := range secretsBelow(key, doc[key]) {
|
||||
switch {
|
||||
case protected[key]:
|
||||
case names && !taken[key]:
|
||||
default:
|
||||
out = append(out, Finding{m.Module, id, at})
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// secretsBelow answers the dotted path of every string at or below value that names a vault secret.
|
||||
func secretsBelow(at string, value any) []string {
|
||||
switch v := value.(type) {
|
||||
case string:
|
||||
if strings.Contains(v, "${secret:") {
|
||||
return []string{at}
|
||||
}
|
||||
case map[string]any:
|
||||
var out []string
|
||||
for _, k := range sortedKeys(v) {
|
||||
out = append(out, secretsBelow(at+"."+k, v[k])...)
|
||||
}
|
||||
return out
|
||||
case []any:
|
||||
var out []string
|
||||
for i, e := range v {
|
||||
out = append(out, secretsBelow(fmt.Sprintf("%s[%d]", at, i), e)...)
|
||||
}
|
||||
return out
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func set(v any) map[string]bool {
|
||||
out := map[string]bool{}
|
||||
list, _ := v.([]any)
|
||||
for _, e := range list {
|
||||
if s, ok := e.(string); ok {
|
||||
out[s] = true
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func sortedKeys(m map[string]any) []string {
|
||||
keys := make([]string, 0, len(m))
|
||||
for k := range m {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
return keys
|
||||
}
|
||||
|
||||
// Known reads the lines of the known list: "<module> <resource> <key>", # comments and blank lines skipped.
|
||||
func Known(path string) (map[string]bool, error) {
|
||||
f, err := os.Open(path)
|
||||
if os.IsNotExist(err) {
|
||||
return map[string]bool{}, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer f.Close()
|
||||
out := map[string]bool{}
|
||||
s := bufio.NewScanner(f)
|
||||
for s.Scan() {
|
||||
line := strings.TrimSpace(s.Text())
|
||||
if line == "" || strings.HasPrefix(line, "#") {
|
||||
continue
|
||||
}
|
||||
out[strings.Join(strings.Fields(line), " ")] = true
|
||||
}
|
||||
return out, s.Err()
|
||||
}
|
||||
|
||||
// seed is the known list as it was first written, standing for the trunk's copy until the trunk holds one.
|
||||
var seed = map[string]bool{
|
||||
"searxng settings server.secret_key": true,
|
||||
}
|
||||
|
||||
// Added answers the lines of known that base does not hold: a line may leave the list, never join it.
|
||||
func Added(known, base map[string]bool) []string {
|
||||
var out []string
|
||||
for line := range known {
|
||||
if !base[line] {
|
||||
out = append(out, line)
|
||||
}
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// Judge splits the findings into those refused and the known lines that no longer name one.
|
||||
func Judge(found []Finding, known map[string]bool) (refused []Finding, stale []string) {
|
||||
seen := map[string]bool{}
|
||||
for _, f := range found {
|
||||
seen[f.String()] = true
|
||||
if !known[f.String()] {
|
||||
refused = append(refused, f)
|
||||
}
|
||||
}
|
||||
for line := range known {
|
||||
if !seen[line] {
|
||||
stale = append(stale, line)
|
||||
}
|
||||
}
|
||||
sort.Strings(stale)
|
||||
return refused, stale
|
||||
}
|
||||
|
||||
// remedy says how a finding is put right.
|
||||
func remedy(key string) string {
|
||||
top, _, nested := strings.Cut(key, ".")
|
||||
if !nested {
|
||||
return fmt.Sprintf("list %q in the file's \"protected\", so the controller refuses a setting of it", top)
|
||||
}
|
||||
return fmt.Sprintf("the controller protects a top-level key only; list %q in the file's \"protected\" if "+
|
||||
"nothing else below it is the operator's, or leave it out of the file's \"takes-settings\"", top)
|
||||
}
|
||||
|
||||
func main() {
|
||||
basePath := flag.String("base", "", "the trunk's copy of known (the merge base's); seed when absent")
|
||||
flag.Parse()
|
||||
root := "."
|
||||
if flag.NArg() > 0 {
|
||||
root = flag.Arg(0)
|
||||
}
|
||||
found, err := Check(root)
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
os.Exit(1)
|
||||
}
|
||||
known, err := Known("known")
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
os.Exit(1)
|
||||
}
|
||||
base := seed
|
||||
if *basePath != "" {
|
||||
if base, err = Known(*basePath); err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
added := Added(known, base)
|
||||
for _, line := range added {
|
||||
fmt.Printf("checks/secrets/known gains %q, which the trunk's copy does not hold: the list only shrinks — "+
|
||||
"protect the key instead (novox/hq issue 342)\n", line)
|
||||
}
|
||||
refused, stale := Judge(found, known)
|
||||
for _, f := range refused {
|
||||
fmt.Printf("%s: %s fills %s with a vault secret, and a setting could replace it (novox/hq issue 342): %s\n",
|
||||
f.Module, f.ID, f.Key, remedy(f.Key))
|
||||
}
|
||||
for _, line := range stale {
|
||||
fmt.Printf("checks/secrets/known names %q, which no longer holds a secret a setting could replace: "+
|
||||
"remove the line\n", line)
|
||||
}
|
||||
if len(refused) > 0 || len(stale) > 0 || len(added) > 0 {
|
||||
os.Exit(1)
|
||||
}
|
||||
fmt.Printf("secrets: no setting can replace a vault secret in a mergeable file (%d known, nested)\n", len(known))
|
||||
}
|
||||
@@ -0,0 +1,125 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func manifest(t *testing.T, root, module, resources string) {
|
||||
t.Helper()
|
||||
dir := filepath.Join(root, "modules", module)
|
||||
if err := os.MkdirAll(dir, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
body := `{"module":"` + module + `","resources":[` + resources + `]}`
|
||||
if err := os.WriteFile(filepath.Join(dir, "module.json"), []byte(body), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestASecretASettingCouldReplaceIsRefused(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
// Refused: the key holds the vault's secret and nothing keeps a layer off it.
|
||||
manifest(t, root, "a", `{"id":"cfg","type":"file","merge":"json","content":"{\"password\": \"${secret:admin}\"}"}`)
|
||||
// Refused: nested below a key the file takes.
|
||||
manifest(t, root, "b", `{"id":"cfg","type":"file","merge":"json","takes-settings":["server"],`+
|
||||
`"content":"{\"server\": {\"secret_key\": \"${secret:s}\", \"port\": 1}}"}`)
|
||||
// Kept: protected.
|
||||
manifest(t, root, "c", `{"id":"cfg","type":"file","merge":"json","protected":["password"],`+
|
||||
`"content":"{\"password\": \"${secret:admin}\", \"user\": \"admin\"}"}`)
|
||||
// Kept: the file names its keys, and not this one.
|
||||
manifest(t, root, "d", `{"id":"cfg","type":"file","merge":"json","takes-settings":["ui"],`+
|
||||
`"content":"{\"token\": \"${secret:t}\", \"ui\": {}}"}`)
|
||||
// Kept: a file that does not merge takes no setting.
|
||||
manifest(t, root, "e", `{"id":"cfg","type":"file","content":"{\"token\": \"${secret:t}\"}"}`)
|
||||
// Refused: protecting another key does not protect this one; a list element counts too.
|
||||
manifest(t, root, "f", `{"id":"cfg","type":"file","merge":"json","protected":["user"],`+
|
||||
`"content":"{\"users\": [{\"password\": \"${secret:p}\"}]}"}`)
|
||||
found, err := Check(root)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := []string{"a cfg password", "b cfg server.secret_key", "f cfg users[0].password"}
|
||||
if len(found) != len(want) {
|
||||
t.Fatalf("found %v, want %v", found, want)
|
||||
}
|
||||
for i, f := range found {
|
||||
if f.String() != want[i] {
|
||||
t.Fatalf("found %v, want %v", found, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheKnownListOnlyShrinks(t *testing.T) {
|
||||
found := []Finding{{"b", "cfg", "server.secret_key"}, {"a", "cfg", "password"}}
|
||||
refused, stale := Judge(found, map[string]bool{"b cfg server.secret_key": true, "gone cfg token": true})
|
||||
if len(refused) != 1 || refused[0].String() != "a cfg password" {
|
||||
t.Fatalf("refused %v", refused)
|
||||
}
|
||||
if len(stale) != 1 || stale[0] != "gone cfg token" {
|
||||
t.Fatalf("stale %v", stale)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheCatalogueHoldsNoSecretASettingCouldReplace(t *testing.T) {
|
||||
found, err := Check("../..")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
known, err := Known("known")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
refused, stale := Judge(found, known)
|
||||
if len(refused) > 0 || len(stale) > 0 {
|
||||
t.Fatalf("refused %v, stale %v", refused, stale)
|
||||
}
|
||||
}
|
||||
|
||||
func TestALineTheTrunkDoesNotHoldIsRefused(t *testing.T) {
|
||||
trunk := map[string]bool{"b cfg server.secret_key": true}
|
||||
// A key unprotected on the branch, and its line added to the list: refused, though it matches a finding.
|
||||
branch := map[string]bool{"b cfg server.secret_key": true, "a cfg password": true}
|
||||
if added := Added(branch, trunk); len(added) != 1 || added[0] != "a cfg password" {
|
||||
t.Fatalf("added %v", added)
|
||||
}
|
||||
// A line removed is no addition.
|
||||
if added := Added(map[string]bool{}, trunk); len(added) != 0 {
|
||||
t.Fatalf("added %v", added)
|
||||
}
|
||||
// Until the trunk holds a copy, the seed stands for it, and the list as committed is within it.
|
||||
known, err := Known("known")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if added := Added(known, seed); len(added) != 0 {
|
||||
t.Fatalf("known holds lines the seed does not: %v", added)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheCheckRefusesALineAddedAgainstTheTrunk(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
manifest(t, root, "a", `{"id":"cfg","type":"file","merge":"json","content":"{\"password\": \"${secret:admin}\"}"}`)
|
||||
dir := t.TempDir()
|
||||
base := filepath.Join(dir, "base")
|
||||
if err := os.WriteFile(base, []byte("# the trunk's\n"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
branch := map[string]bool{"a cfg password": true}
|
||||
found, err := Check(root)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
trunk, err := Known(base)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
refused, stale := Judge(found, branch)
|
||||
if len(refused) != 0 || len(stale) != 0 {
|
||||
t.Fatalf("the line hides the finding from Judge alone: %v %v", refused, stale)
|
||||
}
|
||||
if added := Added(branch, trunk); len(added) != 1 {
|
||||
t.Fatalf("an added line passed: %v", added)
|
||||
}
|
||||
}
|
||||
+1
-1
@@ -1,4 +1,4 @@
|
||||
69
|
||||
66
|
||||
The long-running resources of this catalogue that do not say how they are ready (`health`, novox/hq ADR 0240
|
||||
rule 8), as `module check` counts them. It may only go down: merge-check.sh fails a change that raises it, and
|
||||
one that lowers it writes the new number on the first line. From 2026-11-18, or once it is 0, a long-running
|
||||
|
||||
@@ -30,11 +30,27 @@
|
||||
# 5. the agent's home (novox/hq ADR 0266): no module unpacks an archive, or writes into a file, under
|
||||
# ${machine:agent-home} — the node-engine does both after checking the path, and the agent account owns
|
||||
# that home and could swap a link in between.
|
||||
# 6. the secrets (novox/hq issue 342): no mergeable file fills a key with a vault secret that a setting could
|
||||
# replace — the key is `protected`, or left out of the file's `takes-settings`. checks/secrets/known names
|
||||
# the nested ones the controller cannot protect alone yet, and only shrinks.
|
||||
set -eu
|
||||
|
||||
(cd checks/words && go test -count=1 . && go run . ../..)
|
||||
# The agent's home (novox/hq ADR 0266): no archive, and no file written into, under it.
|
||||
(cd checks/agenthome && go test -count=1 . && go run . ../..)
|
||||
# The secrets (novox/hq issue 342): a password is not a setting, so no setting can replace the vault's. The
|
||||
# known list only shrinks: judged against the trunk's copy at the merge base, a line it does not hold is refused.
|
||||
secrets_base=""
|
||||
secrets_ref="origin/${MESH_CHECK_BASE:-main}"
|
||||
if merge_base=$(git merge-base HEAD "$secrets_ref" 2>/dev/null); then
|
||||
if git cat-file -e "$merge_base:checks/secrets/known" 2>/dev/null; then
|
||||
secrets_base=$(mktemp)
|
||||
git show "$merge_base:checks/secrets/known" >"$secrets_base"
|
||||
fi
|
||||
else
|
||||
echo "NOT COMPARED WITH THE TRUNK: no merge base with $secrets_ref; checks/secrets/known is held to the list it was first written with"
|
||||
fi
|
||||
(cd checks/secrets && go test -count=1 . && if [ -n "$secrets_base" ]; then go run . -base "$secrets_base" ../..; else go run . ../..; fi)
|
||||
# The lists that narrow the paths check may not grow unseen: the base branch's copies are what they are
|
||||
# judged against (MESH_CHECK_BASE, main when unset), read from the clone the check runs in.
|
||||
base_lists=""
|
||||
|
||||
@@ -1,7 +1,8 @@
|
||||
# artifact-store-tools
|
||||
|
||||
Tools that say what the mesh's artifact store holds, and what the controller's records say of it
|
||||
(novox/hq ADR 0251 §1–3, to-be 51). It declares no resources and changes nothing on its machine.
|
||||
(novox/hq ADR 0251 §1–3, to-be 51). It changes nothing on its machine; its only resources are its state
|
||||
directory and the file the mesh writes there from its setting `older-store-names`.
|
||||
|
||||
## Why a module beside the store
|
||||
|
||||
@@ -18,7 +19,7 @@ on that machine.
|
||||
|---|---|---|
|
||||
| `artifact_store_repositories` | r | every repository: tags and what each names, how many manifests (tagged or not), size |
|
||||
| `artifact_store_usage` | r | the store's size, the largest repositories, bytes shared between repositories, bytes no manifest marks (what the nightly collector frees next) |
|
||||
| `artifact_store_references` | r | what the controller's records say of each manifest, counted and sized by state and repository; each manifest listed when one repository is asked; what the records keep that the store does not hold |
|
||||
| `artifact_store_references` | r | what the controller's records and the catalogue's definitions say of each manifest, counted and sized by state and repository; each manifest listed when one repository is asked; what the records keep that the store does not hold |
|
||||
| `artifact_store_collect` | a | a dry run unless `dry_run` is false: what the controller would let go of, and the bytes the nightly collector would free then and now. A real run needs `why` and asks the controller's `collect` |
|
||||
|
||||
### How the store is read
|
||||
@@ -43,6 +44,7 @@ it lists. The store's collector removes every blob no manifest marks, so these n
|
||||
| state | what | removable |
|
||||
|---|---|---|
|
||||
| `kept` | a definition names it, or one of the five most recent builds of a module the mesh holds; `why` says which | no |
|
||||
| `named-by-a-definition` | no record names it, and a definition in the catalogue pins it by digest, under the store's current name or an older one; `why` names the module (hq issue 323) | no |
|
||||
| `holder-of-kept-archive` | the manifest that keeps a kept archive's blob (hq issue 253) | no |
|
||||
| `eligible` | the mesh made it and keeps it for no reason | through the controller's `collect` |
|
||||
| `holder-of-eligible-archive` | the manifest that keeps an eligible archive's blob | with its archive |
|
||||
@@ -56,6 +58,20 @@ its own right; the controller lets an index go with the platform manifests only
|
||||
counted as the index's. Two indexes naming one platform give it the more keeping state. An index no
|
||||
record names leaves its platform manifests unrecorded with it.
|
||||
|
||||
**What the definitions pin** (hq issue 323). A definition may pin an image of the store the mesh did not
|
||||
make: pushed before the mesh built it, and pinned under the store's current name or a name it had before.
|
||||
No record of the controller names it, yet a machine runs it, so `artifact_store_references` reads every
|
||||
definition the catalogue holds (`mesh-catalog.catalog_modules`, then `catalog_module` for each, eight at
|
||||
a time and within twenty seconds) and finds every `<name>/<repository>@sha256:…` in any of its strings,
|
||||
under each of the store's names: `artifact-store://`, the store's address as the controller answers it,
|
||||
and the names it had before. Those are the mesh's setting `older-store-names` (space-separated, `none` by
|
||||
default, set with the controller's `settings`), which the mesh writes to the module's `store-names` file: a
|
||||
catalogue names no installation. When they cannot be read, the answer's first line says so. A manifest no record
|
||||
names that a definition pins is `named-by-a-definition`; one the records name keeps the records' state. A
|
||||
definition read at its commit is kept, so the next call asks only for the list. Definitions that could
|
||||
not be read are listed under `definitions.unread`, and the answer's first line says that what they pin
|
||||
reads as unrecorded until they are.
|
||||
|
||||
**Unrecorded copies of upstream images** are listed in `unrecorded_mirrors`: each manifest at the top of
|
||||
a repository only the build's mirror writes (`upstream/<host>/<path>`, or a module's `<module>/on-<argument>`
|
||||
from before hq ADR 0257) that no record names, with what it and its platforms mark alone. They are the
|
||||
@@ -88,6 +104,8 @@ with slashes, a cut listing refused), manifests read with one absent or unreadab
|
||||
digest never becoming a path outside the blobs, the mark (an index, a shared blob, a blob nothing marks),
|
||||
every state, what the records keep that the store lacks, bytes freed counting a blob shared with a kept
|
||||
manifest as kept, a dry run never confirming, a real run without `why` refused before anything is asked,
|
||||
the fall-back when the references let go of cannot be carried, escalation through `sudo -n`, and that the
|
||||
tools served are exactly the manifest's `tools` and its `invokes` exactly the two controller verbs.
|
||||
the fall-back when the references let go of cannot be carried, a manifest a definition pins under the
|
||||
store's current or an older name, a definition not read said first, escalation through `sudo -n`, and that the
|
||||
tools served are exactly the manifest's `tools` and its `invokes` exactly the two controller verbs and the
|
||||
catalogue's two tools.
|
||||
Any command other than the two read-only scripts fails the test.
|
||||
|
||||
@@ -7,12 +7,15 @@ import (
|
||||
|
||||
// The states a manifest the store holds can be in, as the records see it (novox/hq to-be 51).
|
||||
const (
|
||||
StateKept = "kept"
|
||||
StateHolderKept = "holder-of-kept-archive"
|
||||
StateEligible = "eligible"
|
||||
StateHolderEligible = "holder-of-eligible-archive"
|
||||
StateCollectedPresent = "let-go-yet-present"
|
||||
StateNamedDocument = "named-document"
|
||||
StateKept = "kept"
|
||||
StateHolderKept = "holder-of-kept-archive"
|
||||
StateEligible = "eligible"
|
||||
StateHolderEligible = "holder-of-eligible-archive"
|
||||
StateCollectedPresent = "let-go-yet-present"
|
||||
StateNamedDocument = "named-document"
|
||||
// StateNamedByDefinition is a manifest no record names that a definition pins by digest, under the
|
||||
// store's current name or an older one (novox/hq issue 323): a machine may run it, so it is kept.
|
||||
StateNamedByDefinition = "named-by-a-definition"
|
||||
StateUnrecorded = "unrecorded"
|
||||
collectionRemovesThese = "eligible and holder-of-eligible-archive, through the controller's collect; nothing else"
|
||||
)
|
||||
@@ -45,8 +48,8 @@ var mirrorRepository = regexp.MustCompile(`^(upstream/[a-z0-9][a-z0-9._/-]*|[a-z
|
||||
// keeping orders the states a platform manifest may take from the indexes naming it: the most keeping
|
||||
// wins, so a platform two indexes name is as kept as the more kept of them.
|
||||
var keeping = map[string]int{
|
||||
StateKept: 0, StateNamedDocument: 1, StateHolderKept: 2, StateEligible: 3, StateHolderEligible: 4,
|
||||
StateCollectedPresent: 5,
|
||||
StateKept: 0, StateNamedByDefinition: 1, StateNamedDocument: 2, StateHolderKept: 3, StateEligible: 4,
|
||||
StateHolderEligible: 5, StateCollectedPresent: 6,
|
||||
}
|
||||
|
||||
// recordIndex finds a record by repository and digest.
|
||||
@@ -64,8 +67,9 @@ func indexRecords(r *Records) recordIndex {
|
||||
return idx
|
||||
}
|
||||
|
||||
// Classify gives every manifest the store holds one state.
|
||||
func Classify(v *View, recs *Records) []Entry {
|
||||
// Classify gives every manifest the store holds one state. A manifest no record names that a definition
|
||||
// pins is named by a definition; defs may be nil, when the definitions were not read.
|
||||
func Classify(v *View, recs *Records, defs *Definitions) []Entry {
|
||||
idx := indexRecords(recs)
|
||||
var out []Entry
|
||||
for _, name := range v.L.RepoNames() {
|
||||
@@ -99,6 +103,12 @@ func Classify(v *View, recs *Records) []Entry {
|
||||
e.State = StateNamedDocument
|
||||
}
|
||||
}
|
||||
if e.State == StateUnrecorded && defs != nil {
|
||||
for _, p := range defs.Pins[Key(name, d)] {
|
||||
e.State = StateNamedByDefinition
|
||||
e.Why = append(e.Why, p.why())
|
||||
}
|
||||
}
|
||||
out = append(out, e)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,343 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// What the definitions pin in the store (novox/hq issue 323). A definition may pin an image of the store by
|
||||
// digest that the mesh did not make: one pushed before the mesh built it, pinned under the store's current
|
||||
// name or under a name the store had before. No record of the controller names such a manifest, and yet a
|
||||
// machine runs it, so it is said to be named by a definition and never unrecorded.
|
||||
|
||||
// NamesFileEnv names the file the mesh writes from the setting older-store-names: a line `older=` with the
|
||||
// names the store had before, as a definition may still write them — host names, or host:port, separated by
|
||||
// spaces — or `none`. A catalogue names no installation, so the names are the mesh's setting, not the
|
||||
// manifest's.
|
||||
const NamesFileEnv = "MESH_ARTIFACT_STORE_NAMES_FILE"
|
||||
|
||||
// CatalogueModules and CatalogueModule are the catalogue's tools these definitions are read through.
|
||||
const (
|
||||
CatalogueModules = "mesh-catalog.catalog_modules"
|
||||
CatalogueModule = "mesh-catalog.catalog_module"
|
||||
)
|
||||
|
||||
// DefinitionsTimeout bounds reading every definition, below the runtime's thirty-second call limit; what
|
||||
// was read is kept, so a call that runs out reads only the rest the next time.
|
||||
const DefinitionsTimeout = 20 * time.Second
|
||||
|
||||
// definitionReaders is how many definitions are asked at once.
|
||||
const definitionReaders = 8
|
||||
|
||||
// StoreNames are the names a definition may pin an image of the store under: the scheme the mesh writes
|
||||
// (artifact-store://), the store's address now, and the names it had before.
|
||||
type StoreNames struct {
|
||||
Current []string
|
||||
Older []string
|
||||
}
|
||||
|
||||
// NamesOf is the store's names: its address as the controller answers it, and the older ones given.
|
||||
func NamesOf(address, older string) StoreNames {
|
||||
n := StoreNames{Current: []string{"artifact-store://"}}
|
||||
if a := strings.Trim(strings.TrimSpace(address), "/"); a != "" {
|
||||
n.Current = append(n.Current, a+"/")
|
||||
}
|
||||
for _, o := range strings.FieldsFunc(older, func(r rune) bool { return r == ',' || r == ' ' || r == '\n' || r == '\t' }) {
|
||||
if o = strings.Trim(o, "/"); o != "" && o != "none" {
|
||||
n.Older = append(n.Older, o+"/")
|
||||
}
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
// OlderNames reads the older names from the file the mesh writes, and says why when it cannot.
|
||||
func OlderNames() (string, string) {
|
||||
path := os.Getenv(NamesFileEnv)
|
||||
if path == "" {
|
||||
return "", NamesFileEnv + " is not set, so no older name of the store was read"
|
||||
}
|
||||
raw, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return "", "the store's older names could not be read: " + err.Error()
|
||||
}
|
||||
for _, line := range strings.Split(string(raw), "\n") {
|
||||
if v, ok := strings.CutPrefix(strings.TrimSpace(line), "older="); ok {
|
||||
return v, ""
|
||||
}
|
||||
}
|
||||
return "", path + " holds no line older=, so no older name of the store was read"
|
||||
}
|
||||
|
||||
// Pin is one definition pinning a manifest of the store by digest.
|
||||
type Pin struct {
|
||||
Module string `json:"module"`
|
||||
// Name is the store's name it is pinned under, as written: "artifact-store://", an address, or an older name.
|
||||
Name string `json:"name"`
|
||||
Older bool `json:"older,omitempty"`
|
||||
}
|
||||
|
||||
// pinned is one pin found in a manifest: where in the store, and under which name.
|
||||
type pinned struct {
|
||||
Repository, Digest, Name string
|
||||
Older bool
|
||||
}
|
||||
|
||||
var pinShapes sync.Map // name -> *regexp.Regexp
|
||||
|
||||
// pinShape is `<name><repository>@sha256:<hex>`, the name standing on its own: not the tail of a longer host.
|
||||
func pinShape(name string) *regexp.Regexp {
|
||||
if re, ok := pinShapes.Load(name); ok {
|
||||
return re.(*regexp.Regexp)
|
||||
}
|
||||
re := regexp.MustCompile(`(?:^|[^A-Za-z0-9.-])` + regexp.QuoteMeta(name) +
|
||||
`([a-z0-9][a-z0-9._/-]*)@(sha256:[0-9a-f]{64})`)
|
||||
pinShapes.Store(name, re)
|
||||
return re
|
||||
}
|
||||
|
||||
// PinsIn finds every image of the store a manifest pins by digest, in any string it holds: a resource's
|
||||
// image, a build's base, a file's content.
|
||||
func PinsIn(manifest json.RawMessage, names StoreNames) []pinned {
|
||||
var doc any
|
||||
if json.Unmarshal(manifest, &doc) != nil {
|
||||
return nil
|
||||
}
|
||||
seen := map[string]bool{}
|
||||
var out []pinned
|
||||
var walk func(any)
|
||||
walk = func(x any) {
|
||||
switch v := x.(type) {
|
||||
case map[string]any:
|
||||
for _, e := range v {
|
||||
walk(e)
|
||||
}
|
||||
case []any:
|
||||
for _, e := range v {
|
||||
walk(e)
|
||||
}
|
||||
case string:
|
||||
for _, group := range []struct {
|
||||
names []string
|
||||
older bool
|
||||
}{{names.Current, false}, {names.Older, true}} {
|
||||
for _, name := range group.names {
|
||||
for _, m := range pinShape(name).FindAllStringSubmatch(v, -1) {
|
||||
p := pinned{Repository: strings.Trim(m[1], "/"), Digest: m[2], Name: name, Older: group.older}
|
||||
if k := p.Repository + "@" + p.Digest + " " + p.Name; !seen[k] {
|
||||
seen[k] = true
|
||||
out = append(out, p)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
walk(doc)
|
||||
return out
|
||||
}
|
||||
|
||||
// Definitions is what the catalogue's definitions pin in the store, and what of them could not be read.
|
||||
type Definitions struct {
|
||||
// Pins are by Key(repository, digest).
|
||||
Pins map[string][]Pin
|
||||
// Read is how many definitions were read; Unread says, per module, why its definition was not.
|
||||
Read int
|
||||
Unread map[string]string
|
||||
// NotAsked says why the catalogue could not be asked at all, when it could not.
|
||||
NotAsked string
|
||||
// NamesNotRead says why the store's older names could not be read, when they could not.
|
||||
NamesNotRead string
|
||||
Names StoreNames
|
||||
}
|
||||
|
||||
// Catalogue reads the definitions through the catalogue's tools, and keeps each definition it read by its
|
||||
// module and commit, which never change together.
|
||||
type Catalogue struct {
|
||||
Ask Asking
|
||||
|
||||
mu sync.Mutex
|
||||
kept map[string][]pinned // "<module>@<commit>|<names>" -> its pins
|
||||
}
|
||||
|
||||
type catalogueEntry struct {
|
||||
Module string `json:"module"`
|
||||
Commit string `json:"commit"`
|
||||
}
|
||||
|
||||
// Read asks the catalogue for every module and reads each definition, bounded by DefinitionsTimeout.
|
||||
func (c *Catalogue) Read(ctx context.Context, names StoreNames) *Definitions {
|
||||
defs := &Definitions{Pins: map[string][]Pin{}, Unread: map[string]string{}, Names: names}
|
||||
if c == nil || c.Ask == nil {
|
||||
defs.NotAsked = "this bundle has no way to ask the catalogue"
|
||||
return defs
|
||||
}
|
||||
raw, err := c.Ask(CatalogueModules, map[string]any{})
|
||||
if err != nil {
|
||||
defs.NotAsked = "the catalogue's modules could not be asked: " + err.Error()
|
||||
return defs
|
||||
}
|
||||
answer, output, ok := answerOf(raw)
|
||||
if !ok {
|
||||
defs.NotAsked = "the catalogue refused its modules: " + lastLine(output)
|
||||
return defs
|
||||
}
|
||||
var list struct {
|
||||
Modules []catalogueEntry `json:"modules"`
|
||||
Error string `json:"error"`
|
||||
}
|
||||
if err := json.Unmarshal(answer, &list); err != nil || list.Error != "" || list.Modules == nil {
|
||||
defs.NotAsked = "the catalogue's modules answer is not readable: " + firstOf(list.Error, errText(err), lastLine(output))
|
||||
return defs
|
||||
}
|
||||
c.mu.Lock()
|
||||
if c.kept == nil {
|
||||
c.kept = map[string][]pinned{}
|
||||
}
|
||||
c.mu.Unlock()
|
||||
nameKey := strings.Join(names.Current, ",") + ";" + strings.Join(names.Older, ",")
|
||||
|
||||
ctx, cancel := context.WithTimeout(ctx, DefinitionsTimeout)
|
||||
defer cancel()
|
||||
type result struct {
|
||||
module string
|
||||
pins []pinned
|
||||
why string
|
||||
}
|
||||
results := make(chan result, len(list.Modules))
|
||||
work := make(chan catalogueEntry)
|
||||
var wg sync.WaitGroup
|
||||
for i := 0; i < definitionReaders; i++ {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
for m := range work {
|
||||
pins, why := c.definition(m, names, nameKey)
|
||||
results <- result{m.Module, pins, why}
|
||||
}
|
||||
}()
|
||||
}
|
||||
go func() {
|
||||
defer close(work)
|
||||
for _, m := range list.Modules {
|
||||
select {
|
||||
case work <- m:
|
||||
case <-ctx.Done():
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
go func() { wg.Wait(); close(results) }()
|
||||
|
||||
got := map[string]bool{}
|
||||
collect:
|
||||
for {
|
||||
select {
|
||||
case r, open := <-results:
|
||||
if !open {
|
||||
break collect
|
||||
}
|
||||
got[r.module] = true
|
||||
if r.why != "" {
|
||||
defs.Unread[r.module] = r.why
|
||||
continue
|
||||
}
|
||||
defs.Read++
|
||||
for _, p := range r.pins {
|
||||
k := Key(p.Repository, p.Digest)
|
||||
defs.Pins[k] = append(defs.Pins[k], Pin{Module: r.module, Name: p.Name, Older: p.Older})
|
||||
}
|
||||
case <-ctx.Done():
|
||||
break collect
|
||||
}
|
||||
}
|
||||
for _, m := range list.Modules {
|
||||
if !got[m.Module] {
|
||||
defs.Unread[m.Module] = fmt.Sprintf("not read within %d s", int(DefinitionsTimeout/time.Second))
|
||||
}
|
||||
}
|
||||
for k := range defs.Pins {
|
||||
sort.Slice(defs.Pins[k], func(i, j int) bool { return defs.Pins[k][i].Module < defs.Pins[k][j].Module })
|
||||
}
|
||||
return defs
|
||||
}
|
||||
|
||||
// definition reads one module's definition, or answers it from what was read before.
|
||||
func (c *Catalogue) definition(m catalogueEntry, names StoreNames, nameKey string) ([]pinned, string) {
|
||||
key := m.Module + "@" + m.Commit + "|" + nameKey
|
||||
if m.Commit != "" {
|
||||
c.mu.Lock()
|
||||
pins, had := c.kept[key]
|
||||
c.mu.Unlock()
|
||||
if had {
|
||||
return pins, ""
|
||||
}
|
||||
}
|
||||
args := map[string]any{"module": m.Module}
|
||||
if m.Commit != "" {
|
||||
args["commit"] = m.Commit
|
||||
}
|
||||
raw, err := c.Ask(CatalogueModule, args)
|
||||
if err != nil {
|
||||
return nil, "could not be asked: " + err.Error()
|
||||
}
|
||||
answer, output, ok := answerOf(raw)
|
||||
if !ok {
|
||||
return nil, "refused: " + lastLine(output)
|
||||
}
|
||||
var one struct {
|
||||
Module *struct {
|
||||
Manifest json.RawMessage `json:"manifest"`
|
||||
} `json:"module"`
|
||||
Error string `json:"error"`
|
||||
}
|
||||
if err := json.Unmarshal(answer, &one); err != nil || one.Error != "" || one.Module == nil || len(one.Module.Manifest) == 0 {
|
||||
return nil, "not readable: " + firstOf(one.Error, errText(err), "no manifest in the answer")
|
||||
}
|
||||
pins := PinsIn(one.Module.Manifest, names)
|
||||
if m.Commit != "" {
|
||||
c.mu.Lock()
|
||||
c.kept[key] = pins
|
||||
c.mu.Unlock()
|
||||
}
|
||||
return pins, ""
|
||||
}
|
||||
|
||||
// why says which definitions name a manifest, and under which of the store's names.
|
||||
func (p Pin) why() string {
|
||||
if p.Older {
|
||||
return fmt.Sprintf("a definition names it: %s, under %s, an older name of the store", p.Module, shownName(p.Name))
|
||||
}
|
||||
return "a definition names it: " + p.Module
|
||||
}
|
||||
|
||||
func firstOf(s ...string) string {
|
||||
for _, x := range s {
|
||||
if strings.TrimSpace(x) != "" {
|
||||
return x
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func errText(err error) string {
|
||||
if err == nil {
|
||||
return ""
|
||||
}
|
||||
return err.Error()
|
||||
}
|
||||
|
||||
// shownName is a name of the store as a person reads it: a host without the slash a pin puts after it, the
|
||||
// scheme as written.
|
||||
func shownName(name string) string {
|
||||
if strings.HasSuffix(name, "://") {
|
||||
return name
|
||||
}
|
||||
return strings.TrimSuffix(name, "/")
|
||||
}
|
||||
@@ -0,0 +1,259 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// A definition may pin an image of the store the mesh did not make, under a name the store had before
|
||||
// (novox/hq issue 323): three apps the anchor runs are pinned so. No record names them; a definition does.
|
||||
|
||||
const oldName = "registry-api.example.org"
|
||||
|
||||
// container is a definition whose one container runs image.
|
||||
func container(image string) string {
|
||||
return fmt.Sprintf(`{"module":"x","resources":[{"id":"server","type":"container","image":%q}]}`, image)
|
||||
}
|
||||
|
||||
func TestAnImageADefinitionPinsUnderAnOlderNameOfTheStoreIsNamedByADefinition(t *testing.T) {
|
||||
v := view(t, world())
|
||||
names := NamesOf("store:5000", oldName)
|
||||
defs := &Definitions{Pins: map[string][]Pin{}}
|
||||
for module, manifest := range map[string]string{
|
||||
// m-stranger: no record names it; the invoicing app pins it under the store's older name.
|
||||
"invoicing": container(oldName + "/app/server@" + d("m-stranger")),
|
||||
// m-old: the records say eligible; a definition pinning it does not change what the records say.
|
||||
"website": container("store:5000/app/server@" + d("m-old")),
|
||||
} {
|
||||
for _, p := range PinsIn(json.RawMessage(manifest), names) {
|
||||
defs.Pins[Key(p.Repository, p.Digest)] = append(defs.Pins[Key(p.Repository, p.Digest)], Pin{Module: module, Name: p.Name, Older: p.Older})
|
||||
}
|
||||
}
|
||||
got := stateByName(Classify(v, withFill(records()), defs))
|
||||
stranger := got["app/server "+d("m-stranger")]
|
||||
if stranger.State != StateNamedByDefinition {
|
||||
t.Fatalf("a manifest a definition pins under an older name reads %q, want %q", stranger.State, StateNamedByDefinition)
|
||||
}
|
||||
if want := []string{"a definition names it: invoicing, under " + oldName + ", an older name of the store"}; !reflect.DeepEqual(stranger.Why, want) {
|
||||
t.Errorf("why %q, want %q", stranger.Why, want)
|
||||
}
|
||||
if s := got["app/server "+d("m-old")].State; s != StateEligible {
|
||||
t.Errorf("a recorded manifest a definition pins reads %q: the records decide what the mesh made", s)
|
||||
}
|
||||
// Without the definitions it is what it was: unrecorded.
|
||||
if s := stateByName(Classify(v, withFill(records()), nil))["app/server "+d("m-stranger")].State; s != StateUnrecorded {
|
||||
t.Errorf("without definitions %q", s)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPinsAreFoundInEveryStringAndOnlyUnderTheStoresNames(t *testing.T) {
|
||||
names := NamesOf("store:5000", oldName+", older.example/")
|
||||
manifest := `{"resources":[
|
||||
{"type":"container","image":"` + oldName + `/team/www@` + d("www") + `"},
|
||||
{"type":"file","content":"IMAGE=older.example/team/tool@` + d("tool") + `\n"},
|
||||
{"type":"container","image":"artifact-store://mod/art@` + d("built") + `"},
|
||||
{"type":"container","image":"store:5000/mod/art@` + d("addressed") + `"},
|
||||
{"type":"container","image":"not` + oldName + `/team/www@` + d("lookalike") + `"},
|
||||
{"type":"container","image":"docker.io/library/nginx@` + d("upstream") + `"},
|
||||
{"type":"container","image":"` + oldName + `/team/www:latest"}],
|
||||
"build":{"on":{"base":"` + oldName + `/team/www@` + d("www") + `"}}}`
|
||||
var got []string
|
||||
for _, p := range PinsIn(json.RawMessage(manifest), names) {
|
||||
got = append(got, fmt.Sprintf("%s@%s %s %v", p.Repository, p.Digest, p.Name, p.Older))
|
||||
}
|
||||
want := map[string]bool{
|
||||
"team/www@" + d("www") + " " + oldName + "/ true": true,
|
||||
"team/tool@" + d("tool") + " older.example/ true": true,
|
||||
"mod/art@" + d("built") + " artifact-store:// false": true,
|
||||
"mod/art@" + d("addressed") + " store:5000/ false": true,
|
||||
}
|
||||
if len(got) != len(want) {
|
||||
t.Fatalf("found %d pins, want %d (one per image and name; not a longer host's tail, not another registry, not a tag): %v", len(got), len(want), got)
|
||||
}
|
||||
for _, g := range got {
|
||||
if !want[g] {
|
||||
t.Errorf("found %s", g)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAPlatformTakesTheStateOfAnIndexADefinitionNames(t *testing.T) {
|
||||
v := mirrorWorld()
|
||||
defs := &Definitions{Pins: map[string][]Pin{
|
||||
Key("route-proxy/on-go_base", d("i-pre")): {{Module: "route-proxy", Name: oldName + "/", Older: true}},
|
||||
}}
|
||||
got := stateByName(Classify(v, withFill(mirrorRecords()), defs))
|
||||
for _, name := range []string{"i-pre", "p-pre"} {
|
||||
if s := got["route-proxy/on-go_base "+d(name)].State; s != StateNamedByDefinition {
|
||||
t.Errorf("%s reads %q, want %q", name, s, StateNamedByDefinition)
|
||||
}
|
||||
}
|
||||
if e := got["route-proxy/on-go_base "+d("p-pre")]; e.Index != d("i-pre") {
|
||||
t.Errorf("p-pre takes its state from %q", e.Index)
|
||||
}
|
||||
if s := got["route-proxy/on-go_base "+d("p-lost")].State; s != StateUnrecorded {
|
||||
t.Errorf("a platform nothing names reads %q", s)
|
||||
}
|
||||
for _, um := range UnrecordedMirrors(v, Classify(v, withFill(mirrorRecords()), defs)) {
|
||||
if strings.Contains(um.Reference, hexOf(d("i-pre"))) {
|
||||
t.Errorf("a copy a definition names is offered to the mirrors verb: %v", um)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// olderNames writes the file the mesh writes from the setting older-store-names.
|
||||
func olderNames(t *testing.T, names string) {
|
||||
t.Helper()
|
||||
path := filepath.Join(t.TempDir(), "store-names")
|
||||
if err := os.WriteFile(path, []byte("# written by the mesh\nolder="+names+"\n"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Setenv(NamesFileEnv, path)
|
||||
}
|
||||
|
||||
func TestTheOlderNamesAreReadFromTheFileTheMeshWrites(t *testing.T) {
|
||||
olderNames(t, oldName+" older.example")
|
||||
got, why := OlderNames()
|
||||
if why != "" || !reflect.DeepEqual(NamesOf("", got).Older, []string{oldName + "/", "older.example/"}) {
|
||||
t.Errorf("read %q (%s)", got, why)
|
||||
}
|
||||
olderNames(t, "none")
|
||||
if got, why := OlderNames(); why != "" || len(NamesOf("", got).Older) != 0 {
|
||||
t.Errorf("none read as %q (%s)", got, why)
|
||||
}
|
||||
t.Setenv(NamesFileEnv, filepath.Join(t.TempDir(), "absent"))
|
||||
if _, why := OlderNames(); why == "" {
|
||||
t.Error("an absent file was not said")
|
||||
}
|
||||
}
|
||||
|
||||
func TestOlderNamesNotReadAreSaidFirst(t *testing.T) {
|
||||
t.Setenv(NamesFileEnv, "")
|
||||
a := &asked{definitions: map[string]string{"invoicing": container(oldName + "/app/server@" + d("m-stranger"))}}
|
||||
out, err := references(t, a)(map[string]any{"repository": "app/server"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m := out.(map[string]any)
|
||||
if w := m["said"].([]string)[0]; !strings.Contains(w, NamesFileEnv) {
|
||||
t.Errorf("first said %q", w)
|
||||
}
|
||||
if s := manifestsOf(t, out)["app/server "+d("m-stranger")].State; s != StateUnrecorded {
|
||||
t.Errorf("m-stranger reads %q without the older names", s)
|
||||
}
|
||||
}
|
||||
|
||||
// references is one artifact_store_references tool, kept across calls as the runtime keeps the bundle.
|
||||
func references(t *testing.T, a *asked) func(map[string]any) (any, error) {
|
||||
f := world()
|
||||
s := &Store{Container: "mesh-registry", Run: f.run(t)}
|
||||
for _, tl := range Tools(s, Controller{Ask: a.ask}, &Catalogue{Ask: a.ask}) {
|
||||
if tl.Name == "artifact_store_references" {
|
||||
return tl.Run
|
||||
}
|
||||
}
|
||||
t.Fatal("no artifact_store_references")
|
||||
return nil
|
||||
}
|
||||
|
||||
func manifestsOf(t *testing.T, out any) map[string]Entry {
|
||||
t.Helper()
|
||||
entries, ok := out.(map[string]any)["manifests"].([]Entry)
|
||||
if !ok {
|
||||
t.Fatalf("no manifests in %v", out)
|
||||
}
|
||||
return stateByName(entries)
|
||||
}
|
||||
|
||||
func TestReferencesReadTheDefinitionsUnderTheStoresOlderNames(t *testing.T) {
|
||||
olderNames(t, oldName)
|
||||
a := &asked{definitions: map[string]string{
|
||||
"invoicing": container(oldName + "/app/server@" + d("m-stranger")),
|
||||
"grafana": container("grafana/grafana@" + d("upstream")),
|
||||
}}
|
||||
run := references(t, a)
|
||||
out, err := run(map[string]any{"repository": "app/server"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
e := manifestsOf(t, out)["app/server "+d("m-stranger")]
|
||||
if e.State != StateNamedByDefinition || len(e.Why) != 1 || !strings.Contains(e.Why[0], "invoicing") || !strings.Contains(e.Why[0], oldName) {
|
||||
t.Errorf("m-stranger: %+v", e)
|
||||
}
|
||||
said := out.(map[string]any)["definitions"].(map[string]any)
|
||||
if said["read"] != 2 || !reflect.DeepEqual(said["older_store_names"], []string{oldName}) ||
|
||||
!reflect.DeepEqual(said["store_names"], []string{"artifact-store://", "store:5000"}) {
|
||||
t.Errorf("definitions said %v", said)
|
||||
}
|
||||
if w := out.(map[string]any)["said"].([]string)[0]; strings.Contains(w, "could not be read") {
|
||||
t.Errorf("a warning with every definition read: %s", w)
|
||||
}
|
||||
// A definition read at its commit is kept: the next call asks only for the list.
|
||||
before := 0
|
||||
for _, c := range a.calls {
|
||||
if c == CatalogueModule {
|
||||
before++
|
||||
}
|
||||
}
|
||||
if _, err := run(map[string]any{"repository": "app/server"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
after := 0
|
||||
for _, c := range a.calls {
|
||||
if c == CatalogueModule {
|
||||
after++
|
||||
}
|
||||
}
|
||||
if before != 2 || after != 2 {
|
||||
t.Errorf("asked catalog_module %d times, then %d: each definition at its commit is read once", before, after)
|
||||
}
|
||||
}
|
||||
|
||||
func TestADefinitionNotReadIsSaidFirstAndLeavesWhatItPinsUnrecorded(t *testing.T) {
|
||||
olderNames(t, oldName)
|
||||
a := &asked{
|
||||
definitions: map[string]string{"invoicing": container(oldName + "/app/server@" + d("m-stranger")), "grafana": container("x")},
|
||||
failing: map[string]bool{"invoicing": true},
|
||||
}
|
||||
out, err := references(t, a)(map[string]any{"repository": "app/server"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m := out.(map[string]any)
|
||||
if s := manifestsOf(t, out)["app/server "+d("m-stranger")].State; s != StateUnrecorded {
|
||||
t.Errorf("m-stranger reads %q with its definition unread", s)
|
||||
}
|
||||
if w := m["said"].([]string)[0]; !strings.Contains(w, "1 definitions could not be read") {
|
||||
t.Errorf("first said %q", w)
|
||||
}
|
||||
if u := m["definitions"].(map[string]any)["unread"].(map[string]string); u["invoicing"] == "" || len(u) != 1 {
|
||||
t.Errorf("unread %v", u)
|
||||
}
|
||||
|
||||
// The catalogue not answering at all: the records are still answered, and the warning stands first.
|
||||
a = &asked{noCatalogue: true}
|
||||
out, err = references(t, a)(map[string]any{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m = out.(map[string]any)
|
||||
if w := m["said"].([]string)[0]; !strings.Contains(w, "the definitions could not be read") || !strings.Contains(w, "no responders") {
|
||||
t.Errorf("first said %q", w)
|
||||
}
|
||||
if m["definitions"].(map[string]any)["not_asked"] == nil {
|
||||
t.Errorf("definitions %v", m["definitions"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestNoCatalogueIsSaidAndNothingIsAsked(t *testing.T) {
|
||||
defs := (&Catalogue{}).Read(context.Background(), NamesOf("", ""))
|
||||
if defs.NotAsked == "" || defs.Read != 0 {
|
||||
t.Errorf("%+v", defs)
|
||||
}
|
||||
}
|
||||
@@ -24,7 +24,7 @@ func main() {
|
||||
UID: os.Getuid(),
|
||||
}
|
||||
// An empty name serves as the module the runtime names (MESH_SERVED_MODULE): artifact-store-tools.
|
||||
if err := stdio.Serve("", Tools(s, Controller{Ask: stdio.Ask})); err != nil {
|
||||
if err := stdio.Serve("", Tools(s, Controller{Ask: stdio.Ask}, &Catalogue{Ask: stdio.Ask})); err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
@@ -65,7 +65,7 @@ func TestAPlatformManifestTakesTheStateOfTheIndexNamingIt(t *testing.T) {
|
||||
for i := range recs.References {
|
||||
fill(&recs.References[i])
|
||||
}
|
||||
got := stateByName(Classify(v, recs))
|
||||
got := stateByName(Classify(v, recs, nil))
|
||||
up := "upstream/docker.io/library/golang "
|
||||
for name, want := range map[string]string{
|
||||
up + d("p-amd"): StateEligible, // only the eligible index names it
|
||||
@@ -92,7 +92,7 @@ func TestAPlatformManifestTakesTheStateOfTheIndexNamingIt(t *testing.T) {
|
||||
if len(mirrors) != 2 || !got["route-proxy/on-go_base "+d("i-pre")].Mirror || !got["route-proxy/on-go_base "+d("p-lost")].Mirror {
|
||||
t.Errorf("offered as unrecorded copies: %v", mirrors)
|
||||
}
|
||||
um := UnrecordedMirrors(v, Classify(v, recs))
|
||||
um := UnrecordedMirrors(v, Classify(v, recs, nil))
|
||||
if len(um) != 2 || um[0].Reference != "artifact-store://route-proxy/on-go_base@"+d("i-pre") || um[0].OnlyTheir != 1+1+1000 {
|
||||
t.Errorf("unrecorded copies: %+v", um)
|
||||
}
|
||||
@@ -104,7 +104,7 @@ func TestLettingAnIndexGoTakesItsOwnPlatformsAndSaysWhatStaysAndNamesOne(t *test
|
||||
for i := range recs.References {
|
||||
fill(&recs.References[i])
|
||||
}
|
||||
entries := Classify(v, recs)
|
||||
entries := Classify(v, recs, nil)
|
||||
old := "artifact-store://upstream/docker.io/library/golang@" + d("i-old")
|
||||
gone := LetGoKeys(v, []string{old}, Spared(v, entries))
|
||||
up := "upstream/docker.io/library/golang"
|
||||
|
||||
@@ -215,7 +215,7 @@ func TestTheMarkIsTheCollectorsIncludingAnIndexAndSharedBlobs(t *testing.T) {
|
||||
func TestEveryStateIsGiven(t *testing.T) {
|
||||
v := view(t, world())
|
||||
got := map[string]string{}
|
||||
for _, e := range Classify(v, withFill(records())) {
|
||||
for _, e := range Classify(v, withFill(records()), nil) {
|
||||
got[e.Repository+" "+e.Digest] = e.State
|
||||
}
|
||||
want := map[string]string{
|
||||
@@ -239,7 +239,7 @@ func TestALetGoReferenceStillPresentIsSaidSo(t *testing.T) {
|
||||
v := view(t, world())
|
||||
r := records()
|
||||
r.References = append(r.References, Record{Reference: "artifact-store://app/server@" + d("m-stranger"), State: "collected"})
|
||||
for _, e := range Classify(v, withFill(r)) {
|
||||
for _, e := range Classify(v, withFill(r), nil) {
|
||||
if e.Digest == d("m-stranger") && e.State != StateCollectedPresent {
|
||||
t.Errorf("state %s", e.State)
|
||||
}
|
||||
@@ -270,7 +270,7 @@ func TestBytesFreedCountASharedBlobAsKept(t *testing.T) {
|
||||
|
||||
func TestOnlyTheirBytesLeaveOutWhatAnotherStateMarks(t *testing.T) {
|
||||
v := view(t, world())
|
||||
entries := Classify(v, withFill(records()))
|
||||
entries := Classify(v, withFill(records()), nil)
|
||||
sums := Summarise(v, entries, entries)
|
||||
el := sums[StateEligible]
|
||||
if el.OnlyTheirBytes != 500+v.L.Blobs[d("m-old")] {
|
||||
@@ -284,6 +284,11 @@ type asked struct {
|
||||
calls []string
|
||||
args []map[string]any
|
||||
fail map[string]bool
|
||||
// definitions are the catalogue's modules and their manifests; failing, the modules whose
|
||||
// definition cannot be read; noCatalogue, the catalogue not answering at all.
|
||||
definitions map[string]string
|
||||
failing map[string]bool
|
||||
noCatalogue bool
|
||||
}
|
||||
|
||||
func (a *asked) ask(key string, body any) (json.RawMessage, error) {
|
||||
@@ -299,6 +304,31 @@ func (a *asked) ask(key string, body any) (json.RawMessage, error) {
|
||||
switch key {
|
||||
case "seat:mesh-controller.artifacts":
|
||||
answer = records()
|
||||
case CatalogueModules:
|
||||
if a.noCatalogue {
|
||||
return nil, fmt.Errorf("no responders")
|
||||
}
|
||||
ms := []map[string]any{}
|
||||
for name := range a.definitions {
|
||||
ms = append(ms, map[string]any{"module": name, "commit": "c-" + name, "path": "modules/" + name})
|
||||
}
|
||||
answer = map[string]any{"modules": ms}
|
||||
case CatalogueModule:
|
||||
name, _ := args["module"].(string)
|
||||
if a.failing[name] {
|
||||
return nil, fmt.Errorf("no answer")
|
||||
}
|
||||
if args["commit"] != "c-"+name {
|
||||
return nil, fmt.Errorf("asked %s at %v", name, args["commit"])
|
||||
}
|
||||
manifest, has := a.definitions[name]
|
||||
if !has {
|
||||
answer = map[string]any{"error": "the catalogue holds no " + name}
|
||||
break
|
||||
}
|
||||
// The catalogue's tool answers through the runtime as the protocol's content list.
|
||||
raw, _ := json.Marshal(map[string]any{"module": map[string]any{"module": name, "manifest": json.RawMessage(manifest)}})
|
||||
return json.Marshal(map[string]any{"content": []map[string]any{{"type": "text", "text": string(raw)}}})
|
||||
case "seat:mesh-controller.collect":
|
||||
if args["confirm"] == "true" {
|
||||
answer = CollectAnswer{LetGo: []string{"artifact-store://app/server@" + d("m-old")}}
|
||||
@@ -315,7 +345,7 @@ func (a *asked) ask(key string, body any) (json.RawMessage, error) {
|
||||
func tool(t *testing.T, a *asked, name string) func(map[string]any) (any, error) {
|
||||
f := world()
|
||||
s := &Store{Container: "mesh-registry", Run: f.run(t)}
|
||||
for _, tl := range Tools(s, Controller{Ask: a.ask}) {
|
||||
for _, tl := range Tools(s, Controller{Ask: a.ask}, &Catalogue{Ask: a.ask}) {
|
||||
if tl.Name == name {
|
||||
return tl.Run
|
||||
}
|
||||
@@ -386,7 +416,7 @@ func TestTheToolsServedAreTheToolsTheManifestNames(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var served []string
|
||||
for _, tl := range Tools(&Store{}, Controller{}) {
|
||||
for _, tl := range Tools(&Store{}, Controller{}, &Catalogue{}) {
|
||||
if !strings.HasPrefix(tl.Name, "artifact_store_") || tl.Description == "" || tl.Run == nil {
|
||||
t.Errorf("tool %q", tl.Name)
|
||||
}
|
||||
@@ -398,7 +428,7 @@ func TestTheToolsServedAreTheToolsTheManifestNames(t *testing.T) {
|
||||
if !reflect.DeepEqual(served, listed) {
|
||||
t.Fatalf("served %v, manifest %v", served, listed)
|
||||
}
|
||||
want := []string{"seat:mesh-controller.artifacts", "seat:mesh-controller.collect"}
|
||||
want := []string{"seat:mesh-controller.artifacts", "seat:mesh-controller.collect", CatalogueModules, CatalogueModule}
|
||||
if !reflect.DeepEqual(m.Invokes, want) {
|
||||
t.Errorf("invokes %v", m.Invokes)
|
||||
}
|
||||
|
||||
@@ -13,7 +13,7 @@ import (
|
||||
|
||||
// Tools are the artifact store's tools (novox/hq ADR 0251 §1). The first three only read; artifact_store_collect
|
||||
// only reads too, unless it is a real run, and then it asks the controller, which decides and deletes.
|
||||
func Tools(s *Store, c Controller) []stdio.Tool {
|
||||
func Tools(s *Store, c Controller, cat *Catalogue) []stdio.Tool {
|
||||
ctx := context.Background()
|
||||
repoArg := map[string]any{"type": "string", "description": "one repository, as the store names it (<module>/<artifact>)"}
|
||||
return []stdio.Tool{
|
||||
@@ -53,7 +53,9 @@ func Tools(s *Store, c Controller) []stdio.Tool {
|
||||
{
|
||||
Name: "artifact_store_references",
|
||||
Description: "What the controller's records say of each manifest the artifact store holds: kept (a definition " +
|
||||
"names it, or one of the five most recent builds of a module the mesh holds), the holder of a kept archive, " +
|
||||
"names it, or one of the five most recent builds of a module the mesh holds), named by a definition (no " +
|
||||
"record names it, and a definition in the catalogue pins it by digest, under the store's current name or " +
|
||||
"an older one: the module is said), the holder of a kept archive, " +
|
||||
"eligible (the mesh made it and keeps it for no reason), the holder of an eligible archive, let go yet present, " +
|
||||
"a named document the controller keeps under a tag, or unrecorded — no record names it. A platform manifest " +
|
||||
"takes the state of the recorded index naming it. Counted and sized by " +
|
||||
@@ -67,11 +69,19 @@ func Tools(s *Store, c Controller) []stdio.Tool {
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// The definitions are read while the store is: each bounded below the runtime's call limit.
|
||||
read := make(chan *Definitions, 1)
|
||||
go func() {
|
||||
older, why := OlderNames()
|
||||
defs := cat.Read(ctx, NamesOf(recs.Store, older))
|
||||
defs.NamesNotRead = why
|
||||
read <- defs
|
||||
}()
|
||||
v, err := s.View(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return References(v, recs, repo)
|
||||
return References(v, recs, <-read, repo)
|
||||
},
|
||||
},
|
||||
{
|
||||
@@ -116,7 +126,7 @@ func Tools(s *Store, c Controller) []stdio.Tool {
|
||||
if err != nil {
|
||||
return map[string]any{"controller": answer, "bytes_not_worked_out": "the records could not be read: " + err.Error()}, nil
|
||||
}
|
||||
return Collected(v, answer, dry, Classify(v, recs)), nil
|
||||
return Collected(v, answer, dry, Classify(v, recs, nil)), nil
|
||||
},
|
||||
},
|
||||
}
|
||||
@@ -225,11 +235,11 @@ func Usage(v *View, top int) map[string]any {
|
||||
}
|
||||
|
||||
// References answers artifact_store_references.
|
||||
func References(v *View, recs *Records, only string) (map[string]any, error) {
|
||||
func References(v *View, recs *Records, defs *Definitions, only string) (map[string]any, error) {
|
||||
if only != "" && v.L.Repos[only] == nil {
|
||||
return nil, fmt.Errorf("the store holds no repository %q", only)
|
||||
}
|
||||
entries := Classify(v, recs)
|
||||
entries := Classify(v, recs, defs)
|
||||
byRepo := map[string][]Entry{}
|
||||
for _, e := range entries {
|
||||
byRepo[e.Repository] = append(byRepo[e.Repository], e)
|
||||
@@ -243,7 +253,7 @@ func References(v *View, recs *Records, only string) (map[string]any, error) {
|
||||
}
|
||||
whole := Summarise(v, entries, entries)
|
||||
said := []string{}
|
||||
for _, state := range []string{StateKept, StateHolderKept, StateEligible, StateHolderEligible, StateCollectedPresent, StateNamedDocument, StateUnrecorded} {
|
||||
for _, state := range []string{StateKept, StateNamedByDefinition, StateHolderKept, StateEligible, StateHolderEligible, StateCollectedPresent, StateNamedDocument, StateUnrecorded} {
|
||||
if s := whole[state]; s != nil {
|
||||
said = append(said, fmt.Sprintf("%s: %d manifests marking %s, %s of it marked by nothing else", state, s.Manifests, human(s.Bytes), human(s.OnlyTheirBytes)))
|
||||
}
|
||||
@@ -276,9 +286,57 @@ func References(v *View, recs *Records, only string) (map[string]any, error) {
|
||||
if u := unreadNote(v); u != nil {
|
||||
out["unread"] = u
|
||||
}
|
||||
out["definitions"] = definitionsSaid(defs)
|
||||
if warn := definitionsWarning(defs); warn != "" {
|
||||
// Said first: what it leaves out is a manifest a machine may run, read as unrecorded.
|
||||
out["said"] = append([]string{warn}, said...)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// definitionsSaid is how the definitions were read: how many, under which names, and what was not read.
|
||||
func definitionsSaid(defs *Definitions) map[string]any {
|
||||
if defs == nil {
|
||||
return map[string]any{"read": 0, "not_asked": "the definitions were not read"}
|
||||
}
|
||||
var current, older []string
|
||||
for _, n := range defs.Names.Current {
|
||||
current = append(current, shownName(n))
|
||||
}
|
||||
for _, n := range defs.Names.Older {
|
||||
older = append(older, shownName(n))
|
||||
}
|
||||
out := map[string]any{"read": defs.Read, "store_names": nonNil(current), "older_store_names": nonNil(older)}
|
||||
if defs.NotAsked != "" {
|
||||
out["not_asked"] = defs.NotAsked
|
||||
}
|
||||
if defs.NamesNotRead != "" {
|
||||
out["older_store_names_not_read"] = defs.NamesNotRead
|
||||
}
|
||||
if len(defs.Unread) > 0 {
|
||||
out["unread"] = defs.Unread
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// definitionsWarning says, when any definition was not read, that a manifest it pins reads as unrecorded.
|
||||
func definitionsWarning(defs *Definitions) string {
|
||||
switch {
|
||||
case defs == nil:
|
||||
return ""
|
||||
case defs.NotAsked != "":
|
||||
return "the definitions could not be read (" + defs.NotAsked + "), so a manifest a definition pins " +
|
||||
"and no record names reads as unrecorded here: ask again before reading unrecorded as unused"
|
||||
case defs.NamesNotRead != "":
|
||||
return defs.NamesNotRead + ", so a manifest a definition pins under an older name of the store and no " +
|
||||
"record names reads as unrecorded here"
|
||||
case len(defs.Unread) > 0:
|
||||
return fmt.Sprintf("%d definitions could not be read (listed in definitions.unread), so a manifest one "+
|
||||
"of them pins and no record names reads as unrecorded here: ask again — what was read is kept", len(defs.Unread))
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// Collected answers artifact_store_collect: the controller's answer, and what the store's collector frees.
|
||||
func Collected(v *View, a *CollectAnswer, dry bool, entries []Entry) map[string]any {
|
||||
refs := a.LetGo
|
||||
|
||||
@@ -3,7 +3,9 @@
|
||||
"version": "1",
|
||||
"invokes": [
|
||||
"seat:mesh-controller.artifacts",
|
||||
"seat:mesh-controller.collect"
|
||||
"seat:mesh-controller.collect",
|
||||
"mesh-catalog.catalog_modules",
|
||||
"mesh-catalog.catalog_module"
|
||||
],
|
||||
"tools": [
|
||||
"artifact_store_repositories",
|
||||
@@ -11,6 +13,29 @@
|
||||
"artifact_store_references",
|
||||
"artifact_store_collect"
|
||||
],
|
||||
"settings": {
|
||||
"older-store-names": {
|
||||
"kind": "preference",
|
||||
"default": "none",
|
||||
"why": "the names the artifact store had before, as a definition may still pin an image under them: host names or host:port, separated by spaces; none when it had no other. artifact_store_references reads every definition under these as well as under the store's current name, so an image a definition pins under one is said to be named by a definition (novox/hq issue 323)"
|
||||
}
|
||||
},
|
||||
"resources": [
|
||||
{
|
||||
"id": "state",
|
||||
"type": "directory",
|
||||
"mode": "0700",
|
||||
"place": "."
|
||||
},
|
||||
{
|
||||
"id": "store-names",
|
||||
"type": "file",
|
||||
"path": "${dir:state}/store-names",
|
||||
"mode": "0600",
|
||||
"trusted": false,
|
||||
"content": "# The names the artifact store had before, from the setting older-store-names, written by the mesh.\n# artifact_store_references reads every definition under these too (novox/hq issue 323).\n# Replaced on every push; change the setting, never this file.\nolder=${setting:older-store-names}\n"
|
||||
}
|
||||
],
|
||||
"build": {
|
||||
"artifacts": [
|
||||
{
|
||||
@@ -24,7 +49,8 @@
|
||||
"artifact-store-tools"
|
||||
],
|
||||
"env": {
|
||||
"MESH_ARTIFACT_STORE_CONTAINER": "mesh-registry"
|
||||
"MESH_ARTIFACT_STORE_CONTAINER": "mesh-registry",
|
||||
"MESH_ARTIFACT_STORE_NAMES_FILE": "${dir:state}/store-names"
|
||||
}
|
||||
}
|
||||
]
|
||||
|
||||
@@ -103,7 +103,8 @@
|
||||
"path": "${dir:mesh-state}/config.json",
|
||||
"mode": "0600",
|
||||
"content": "{\n \"password\": \"${secret:admin}\",\n \"host\": \"${bound:route:name}\"\n}\n",
|
||||
"merge": "json"
|
||||
"merge": "json",
|
||||
"protected": ["password"]
|
||||
}
|
||||
],
|
||||
"build": {
|
||||
|
||||
+278
-51
@@ -1,8 +1,12 @@
|
||||
# betterbird
|
||||
|
||||
Betterbird, the mail and calendar client of g14 and shanks (a Thunderbird fork, with the operator's
|
||||
Exchange calendar through the Owl add-on), as a module (novox/hq issue 345, ADR 0208). It requires
|
||||
`x11-display`, so it is assigned only where a display server is held on the same machine.
|
||||
Exchange calendar through the Owl add-on), as a module (novox/hq issue 345, issue 586, ADR 0208, ADR 0316,
|
||||
ADR 0317, ADR 0329, to-be 56). It requires `x11-display`, so it is assigned only where a display server is held
|
||||
on the same machine. It installs Betterbird from a pinned upstream release, ships Owl with it through
|
||||
Betterbird's enterprise policy, and owns Betterbird's whole configuration as settings: the preferences, the
|
||||
mail accounts without their credentials, the identities with their signatures, the CalDAV calendars, the
|
||||
CardDAV address books, the filters and the tags.
|
||||
|
||||
It exists because a symptom had no tools: on a workstation, a calendar reminder appeared as a
|
||||
transparent window with nothing in it (2026-10-09). Nothing in the mesh could say which window that
|
||||
@@ -11,74 +15,285 @@ issue 345.
|
||||
|
||||
## Owns
|
||||
|
||||
| what | where |
|
||||
|---|---|
|
||||
| the module's directory | `~/.config/betterbird/` |
|
||||
| the module's `user.js`, filled from the setting `prefs` | `~/.config/betterbird/user.js` |
|
||||
| what | where | class |
|
||||
|---|---|---|
|
||||
| Betterbird, the upstream release (`archive`, pinned by URL and sha256) | `/opt/betterbird` | owned, whole |
|
||||
| the enterprise policy (`install/distribution/policies.json`) | `/opt/betterbird/distribution` | owned, whole |
|
||||
| Owl's xpi (`owl`, pinned by the vendor's sha256) | `/opt/betterbird/owl` | owned, whole |
|
||||
| the module's defaults (`install/defaults-pref/mesh-defaults.js`: the preferences kept at Betterbird's own default, no value of the operator's) and the spelling dictionaries' default (`vendor-prefs.js`) | `/opt/betterbird/defaults/pref` | owned: its two files |
|
||||
| the tools' program, for the start's preparation | `/usr/local/lib/mesh-betterbird` | owned, whole |
|
||||
| the start | `/usr/local/bin/betterbird` | owned, whole |
|
||||
| the launcher entry | `/usr/local/share/applications/eu.betterbird.Betterbird.desktop` | owned, whole |
|
||||
| the module's directory, with the settings as the node-engine writes them (`prefs.json`, `accounts.json`, `configuration.json`, `tags.json`), the records of what was adopted or created, and the filter files it replaced (`kept/`) | `~/.config/betterbird/` | owned, whole |
|
||||
| the profile's `user.js`, placed by the tools | the profile | the module's, or never touched |
|
||||
| the lists in the profile's `prefs.js` (accounts, outgoing servers, an adopted account's identities, the calendars' order), and the preferences of what the module created and the settings no longer hold | the profile | written into: the module's keys only, only while Betterbird is closed |
|
||||
| each declared account's `msgFilterRules.dat` (setting `filters`) | the account's directory in the profile | owned, whole, only while Betterbird is closed |
|
||||
| the profile's own values of the preferences the module manages, in `prefs.js` | the profile | cleared only by `betterbird_config_apply` with apply, Betterbird quit |
|
||||
| the profile: mail, address books, calendars, logins | `~/.thunderbird/` | found (ADR 0182) |
|
||||
|
||||
Nothing else. It holds no seat and adds no start.
|
||||
**The community package `betterbird-bin` is declared absent** (ADR 0180), before anything is placed in
|
||||
`/opt/betterbird`: the node-engine removes it with `pacman -R`, then unpacks the release where the package
|
||||
had it. Betterbird chooses its profile by its installation's path, so the same profile opens after the
|
||||
switch. A package left in place (its removal failed) holds the path, and the release is refused there by
|
||||
the node-engine rather than mixed in. The module creates no link: the start, a launcher entry naming the
|
||||
icon in the installation, and the dictionaries as a default preference replace the package's links.
|
||||
|
||||
- **No package.** g14 and shanks both run `betterbird-bin` 153.1.0esr-1, installed explicitly from the
|
||||
AUR: `pacman` counts it as foreign. The node-engine installs packages from the official
|
||||
repositories only, so the module does not declare it, and **installing and upgrading Betterbird stay
|
||||
the operator's** (an AUR helper). `betterbird_status` says the version and that it is from the AUR;
|
||||
`betterbird_check` says when it is missing.
|
||||
- **No start.** Betterbird is started by the operator from the launcher (on the workstation measured,
|
||||
it runs in the login session's scope, started by hand, with no autostart entry and no line in i3's
|
||||
configuration). `betterbird_restart` starts it as the transient user unit `mesh-betterbird`, so its
|
||||
output then goes to the journal.
|
||||
- **The profile is Betterbird's** (ADR 0182): `~/.thunderbird/` holds the mail, the address books,
|
||||
the calendars' caches, the logins and the keys. The module declares none of it.
|
||||
**Every new build waits for the operator** (`upgrade: record`): the first push to a node removes the
|
||||
package and replaces what a running Betterbird has open, so the operator quits Betterbird, then pushes.
|
||||
An upgrade is a change to the pins, reviewed and delivered like any other (ADR 0316 §5).
|
||||
|
||||
## The user.js: how a preference is declared
|
||||
**The pins.** The release is `betterbird-153.4.0esr-bb10.en-US.linux-x86_64.tar.xz` (81,620,684 bytes) and
|
||||
Owl is 1.5.6.1 (430,201 bytes), each pinned by sha256 in `module.json`. The build seat fetches each once,
|
||||
refuses it when the digest differs, keeps it under its sha256, and packs it into the artifact store; a node
|
||||
receives the mesh's archive by digest and never asks the vendor. **Owl is never in this repository**: its
|
||||
licence forbids copying it to third parties, and its vendor serves only its newest version, at one
|
||||
address. Keeping the pinned copy in the mesh's private artifact store for the operator's own machines was
|
||||
the operator's call (2026-10-12, ADR 0316 §6).
|
||||
|
||||
Betterbird reads `user.js` in its profile when it starts, and a preference set there wins over the
|
||||
one it saved in `prefs.js`. The profile's directory has a random name per machine
|
||||
(`<random>.default-default`), which a manifest cannot name. So:
|
||||
**Owl's range.** Owl declares which Betterbird versions it supports (`strict_min_version` to
|
||||
`strict_max_version`: 115 to 156 for 1.5.6.1). `owlSupports` in `manifest_test.go` holds the range read
|
||||
from each pinned xpi, and the module's tests refuse a Betterbird pin outside it: moving Owl's pin means
|
||||
reading the new range there (ADR 0316 §8).
|
||||
|
||||
1. the node-engine writes `~/.config/betterbird/user.js` from the module's setting `prefs`, one line of
|
||||
`user_pref("name", value);` statements (default: a comment, no preference);
|
||||
2. the module's tools copy that file whole into the profile: `betterbird_user_js` with `apply`, or
|
||||
`betterbird_restart`, which places it before it starts Betterbird again;
|
||||
3. Betterbird reads it at that start.
|
||||
**The policy** turns Betterbird's own updater off (`DisableAppUpdate`, which Betterbird's update check
|
||||
honours) and installs Owl into every profile from `file:///opt/betterbird/owl/owl.xpi` as
|
||||
`force_installed`, with `updates_disabled`. A profile that already runs another Owl keeps it: the policy
|
||||
installs the pinned one where Owl is missing. A newer Owl in the profile (g14 runs 1.5.7) is said as a
|
||||
fact by `betterbird_status`; Owl installed by policy cannot be removed in Betterbird, so the remedy is
|
||||
moving the pin. An older one is a finding of `betterbird_check`, with the same remedy.
|
||||
|
||||
**Betterbird never asks to be the default client.** The policy locks `mail.shell.checkDefaultClient` off
|
||||
(`Preferences`, `Status: locked`): Betterbird's own default (`all-thunderbird.js`) is to ask at every start,
|
||||
with a window to make it the default for mail, news, feeds and calendar files, while that preference is true
|
||||
(`messenger.js`). Betterbird 153 has no policy of its own for it (its `policies-schema.json`), and its
|
||||
`Preferences` policy accepts the `mail.` prefix. A lock, not a default: the window's own "always check" box
|
||||
writes the preference into the profile, which wins over a default; a lock wins over the profile and greys the
|
||||
box out in the settings. `betterbird_check` says when Betterbird would ask (novox/hq issue 345). **Owl's licence stays the person's**: the mesh holds no licence, key or payment (ADR 0316 §9).
|
||||
|
||||
**No start of its own.** Betterbird is started by the operator from the launcher. The start
|
||||
(`/usr/local/bin/betterbird`) first runs `betterbird-tools prepare`, which, with no Betterbird of the
|
||||
account running, places the user.js and writes what Betterbird rewrites itself (the lists, the removals, the
|
||||
filter files); then it
|
||||
starts Betterbird either way. `betterbird_restart` starts it as the transient user unit
|
||||
`mesh-betterbird`, so its output then goes to the journal.
|
||||
|
||||
|
||||
## The preferences: locks, the setting prefs, and Betterbird's own defaults
|
||||
|
||||
Betterbird's preferences come in three layers (novox/hq issue 345, issue 586). **The operator's own choices
|
||||
live in the module's configuration, never in the catalogue**: the module ships only its policy's locks and the
|
||||
names it keeps at Betterbird's own default.
|
||||
|
||||
| layer | where | who changes it | holds |
|
||||
|---|---|---|---|
|
||||
| **locked** | `/opt/betterbird/distribution/policies.json` (`install/distribution/`) | only a reviewed catalogue change | what must never differ on a node; Betterbird's settings show it greyed out |
|
||||
| **declared** | the setting `prefs`, mesh-wide with a node's layer over it, written to `~/.config/betterbird/prefs.json` and placed in the profile's `user.js` | the operator, through the settings layers (trusted: the terminal or the operator's warrant) | the operator's setup, and a node's exceptions, such as a font size for a larger screen |
|
||||
| **Betterbird's default** | Betterbird's own defaults; `mesh-defaults.js` names those the module manages there | a catalogue change | the preferences kept at Betterbird's default |
|
||||
|
||||
A locked value wins over everything; a declared one (in `user.js`) wins over the profile's `prefs.js` at every
|
||||
start; `prefs.js` wins over a default.
|
||||
|
||||
**The setting `prefs` is one entry per preference** (issue 586): `{"prefs": {"mail.uifontsize": 14, …}}`, the
|
||||
value true, false, a whole number or a string. The node-engine merges the layers key by key, so a node's layer
|
||||
overrides one preference, or takes one out with `null`. The operator's setup approved on 2026-10-12 (compact
|
||||
density, 14-point text, the vertical view with multi-line cards and threads, no new-mail pop-up, sound or tray
|
||||
icon, the calendar's day from 09:00 to 20:00, reminders missed while asleep shown, chat off, remote content
|
||||
blocked, Do Not Track; 21 preferences) moved from the catalogue's defaults file into the mesh-wide layer of
|
||||
`prefs`, set on the operator's Approve.
|
||||
|
||||
**The order that leaves no node on Betterbird's own defaults** (issue 586): merge this build (the `record`
|
||||
upgrade policy holds it back from the nodes); propose the mesh-wide layer through the controller's `settings`
|
||||
verb with `propose`, carrying the current `accounts` verbatim and `prefs` with the 21 values; the operator
|
||||
approves; only then push the desktop nodes. Caveat: between that approval and a node's push, any other send to a
|
||||
desktop node writes the map's text into the old build's user.js, and `betterbird_check` there says the user.js is
|
||||
refused until the push.
|
||||
|
||||
**Locked** (the policy): Betterbird's updater off (`DisableAppUpdate`); telemetry off (`DisableTelemetry`);
|
||||
add-ons change only through the mesh's pins (`ExtensionUpdate` false); and three preferences locked off:
|
||||
`mail.shell.checkDefaultClient` (the ask to be the default client), `mail.inappnotifications.enabled` (the
|
||||
vendor's in-app notices), and `mailnews.start_page.enabled` (the start page, which otherwise loads from the
|
||||
vendor at every start).
|
||||
|
||||
**Kept at Betterbird's default** (`// keeps Betterbird's default: <name>` in `mesh-defaults.js`): the theme,
|
||||
system colours, always-underlined links, manual spam marking, and the composer's attachment pane. A value the
|
||||
profile saved for one of them is drift.
|
||||
|
||||
**Drift.** `betterbird_config` shows each managed preference (locked, declared, kept) through the layers, and
|
||||
flags **drift**: a declared value the profile's `user.js` does not hold yet (remedy: `betterbird_user_js` with
|
||||
apply, then `betterbird_restart`), or a value the profile saved where the module keeps Betterbird's default.
|
||||
`betterbird_check` reports it.
|
||||
|
||||
- `betterbird_config_apply` with `apply` takes a kept preference's saved value out of `prefs.js`, so Betterbird's
|
||||
default holds at the next start. It acts only while Betterbird is quit (no main process, and no `lock` in the
|
||||
profile), and says what it cleared.
|
||||
- **The theme is the exception.** Betterbird keeps the active theme in its add-on database (`extensions.json`),
|
||||
and `extensions.activeThemeID` only mirrors it. Another theme is drift whose remedy is to enable the default
|
||||
theme in the add-on manager. `betterbird_config_apply` never clears it, and `keep` refuses it.
|
||||
- `betterbird_config` with `keep` proposes this node's layer of `prefs` holding the profile's values of the named
|
||||
preferences beside what the layer holds, through the controller's settings verb, for the operator's Approve.
|
||||
|
||||
**One owner per value.** `prefs` refuses a preference whose name can hold a credential, an account's
|
||||
(`mail.account.`, `mail.identity.`, `mail.server.`, `mail.smtpserver.` and the account lists, ADR 0316 §11), and
|
||||
those the settings below own: `calendar.registry.`, `calendar.list.`, `ldap_2.servers.` and `mailnews.tags.`
|
||||
(ADR 0329 §6).
|
||||
|
||||
## The user.js
|
||||
|
||||
Betterbird reads `user.js` in its profile when it starts, and a preference set there wins over the one it saved
|
||||
in `prefs.js`. The profile's directory has a random name per machine (`<random>.default-default`), which a
|
||||
manifest cannot name. So the tools compose the user.js from the settings — `prefs`, then the declared accounts,
|
||||
then the declared identities, calendars, address books and tags under the keys each resolved to — and place it
|
||||
in the profile: at the start from the launcher, `betterbird_user_js` with `apply`, or `betterbird_restart`.
|
||||
|
||||
**The profile is found** as the one the running Betterbird holds open; with none running, as the
|
||||
installation's default in `profiles.ini` (`[Install…] Default`), else the profile marked `Default=1`.
|
||||
On the workstation measured these differ: `Profile1` is marked `Default=1`, and the installation's
|
||||
default, the one in use, is the other.
|
||||
|
||||
**A user.js the module did not write is never overwritten.** The module's begins with the line
|
||||
`// Betterbird's user.js (module betterbird`; any other is the operator's, and `apply` refuses with
|
||||
what to do. The tools refuse a declared file that holds anything but comments and `user_pref`
|
||||
statements, or that sets a preference whose name can hold a credential.
|
||||
`// Betterbird's user.js (module betterbird`; any other is the operator's, and `apply` refuses with what to do.
|
||||
|
||||
The file is **trusted** (it carries no `"trusted": false`): a preference can weaken how the mail client
|
||||
checks a server, so the setting is changed only at the controller's terminal, never through a verb
|
||||
(novox/hq issue 339).
|
||||
## The accounts: declared without credentials (ADR 0316 §10–§16, ADR 0317)
|
||||
|
||||
The setting `accounts` reaches the machine as `~/.config/betterbird/accounts.json`, a trusted mergeable
|
||||
file: a map of accounts by name, each with `address`, `from-name` and `label` (optional), `incoming` (`kind`
|
||||
imap or pop3, `host`, `port`, `security` tls, starttls or none, `sign-in` password, encrypted-password,
|
||||
oauth, kerberos, ntlm or tls-certificate, `user`) and `outgoing` (the same, without a kind; its sign-in may
|
||||
be none), and, only in a node's own layer, `key`. **Never a password or a token**: the whole file is
|
||||
refused when any key in it is shaped like one, and an account signs in at its first use, in Betterbird's
|
||||
own window, where Betterbird keeps the result in its login store. The work account through Owl is never
|
||||
declared: the person creates it in Owl's setup (ADR 0316 §13).
|
||||
|
||||
At every apply (the start from the launcher, `betterbird_user_js` with apply, `betterbird_restart`), each
|
||||
declared account, in its place in the setting, is (ADR 0317 §1):
|
||||
|
||||
1. **kept by its recorded keys**, when the module recorded it on this node for this profile;
|
||||
2. **its creation**, when it is under the module's own keys (`account_mesh_<name>`, `id_mesh_<name>`,
|
||||
`server_mesh_<name>`, `smtp_mesh_<name>`);
|
||||
3. **adopted by the key a node's layer names** (`key`), when that account is in the list, its default
|
||||
identity has the address and it is not Owl's;
|
||||
4. **adopted by its address**, when exactly one account's default identity has it: its keys (account,
|
||||
default identity, incoming server, and the outgoing server that identity names) become the module's, and
|
||||
the account lists are left as they are;
|
||||
5. **created** under the module's own keys when no account has it: its keys are added at the end of
|
||||
`mail.accountmanager.accounts` and `mail.smtpservers` in `prefs.js`, every other key kept in its place,
|
||||
only while Betterbird is not running.
|
||||
|
||||
**Refused, for that account alone and with the remedy** (§7): two accounts with the address (remove the
|
||||
duplicate in Betterbird, or name the `key`), Owl's account at the address, a named key that does not
|
||||
qualify, an account of another kind than declared (never converted), an identity that names no outgoing
|
||||
server, and an outgoing server that another identity, neither adopted nor the module's, also uses
|
||||
(Betterbird lets identities share one, and the declared values would change that other account's sending;
|
||||
ADR 0317 §2 leaves the identity's server and the lists as they are, so the module refuses rather than give
|
||||
it a server of its own). A recorded adoption follows the identity's current outgoing server at every apply,
|
||||
under the same rule. The other accounts go ahead; the start and `betterbird_check` say it.
|
||||
`betterbird_user_js` answers an account's preferences by name with their values withheld.
|
||||
|
||||
The declared fields are written in the user.js under the account's keys, so they win at every start; only
|
||||
the fields the setting names, never the account's directory, folders, filters, caches, mail or kept
|
||||
sign-in, so an adopted account keeps its sign-in (§5). The apply names, by preference name only, each one
|
||||
whose value it changed (§4). What was adopted or created is recorded in
|
||||
`~/.config/betterbird/accounts-record.json`. **Leaving the setting** releases an adopted account (its
|
||||
lines and its record go, the account stays the person's) and takes a created account's keys out of the
|
||||
lists (its mail stays in the profile).
|
||||
|
||||
**The value is derived, not retyped** (§9): `betterbird_accounts` with `propose` builds `accounts` from the
|
||||
profile (every account but Owl's, no credential) and proposes it in the mesh-wide layer as it stands, through
|
||||
the controller's settings verb; the operator approves on a channel that proves who answers. Its answer shows
|
||||
the value it proposed (ADR 0329 §10).
|
||||
|
||||
**A node with no profile yet.** The preparation needs the profile Betterbird makes at its first start: on a
|
||||
node set up from nothing, the declared accounts are written at the second start from the launcher (or at
|
||||
`betterbird_restart`).
|
||||
|
||||
## Identities, calendars, address books, filters and tags (ADR 0329)
|
||||
|
||||
Five more settings, mesh-wide with a node's layer over each (an entry set to `null` in a node's layer is not on
|
||||
that node). `identities`, `calendars`, `address-books` and `filters` reach the machine as
|
||||
`~/.config/betterbird/configuration.json`, trusted as `accounts` is; `tags` as `tags.json`, not trusted (§5).
|
||||
**No value holds a password, a token or a cookie**: a key shaped like one is refused, and so is an address whose
|
||||
user part carries a password (§2). A setting holds no line break, so a signature is a list of its lines.
|
||||
|
||||
| setting | an entry | matched in the profile by |
|
||||
|---|---|---|
|
||||
| `identities` | by a name: `account` (its name in `accounts`), and for another identity than the account's default `address` and `name`; `reply-to`, `organization`, `signature` (lines), `signature-html`, `signature-above-quote`; `key` in a node's layer | the account's default (no `address`), or an identity of that account with the address |
|
||||
| `calendars` | by a name: `name`, `url` (CalDAV), `color`, `read-only`, `reminders`, `user`, `invitations` (an identities entry); `key` in a node's layer | its address |
|
||||
| `address-books` | by a name: `name`, `url` (CardDAV), `user`; `key` in a node's layer | its address |
|
||||
| `tags` | by its key: `name`, `color`, `order` | its key |
|
||||
| `filters` | by an account's name: the ordered list, each with `name`, `enabled`, `when` (new-mail, manual, after-junk, sending, archiving, periodic), `match` (all, any, every-message), `conditions` (`field`, `op`, `value`, in Betterbird's words) and `actions` (`action` in Betterbird's words; `folder` as a path within the account, or `value`) | owned whole per account |
|
||||
|
||||
**Adopted as accounts are** (§7): an entry recorded on this node is kept by its key; one under the module's own
|
||||
key (`ident_mesh_<name>`, `mesh_<name>` for a calendar or an address book) is its creation; a `key` in the
|
||||
node's layer is adopted when the profile holds it and it matches; else one match is adopted, none is created
|
||||
under the module's key, and several are refused with the remedy. The declared fields go into the user.js under
|
||||
that key. An entry leaving the setting releases what was adopted (its lines go) and removes the preferences of
|
||||
what was created, never a calendar's events, an address book's contacts or a cache. An entry no setting declares
|
||||
stays the person's. Owl's accounts and calendars, an ICS calendar (its address is often the secret), local
|
||||
calendars and address books are never declared (§13).
|
||||
|
||||
**Filters are owned whole per declared account** (§8): the module writes that account's `msgFilterRules.dat`
|
||||
from the list, each folder's path turned into its address on this node, so a filter added in Betterbird is gone
|
||||
at the next apply (drift shows it first). Before its first write it keeps the replaced file in
|
||||
`~/.config/betterbird/kept/<profile>/`; when the account leaves `filters`, the module stops writing and the file
|
||||
stays the person's; `betterbird_filters` says where the kept copy is. A filter a setting cannot hold as it is (a
|
||||
folder of another account, an add-on's action, a news filter) is named and not derived.
|
||||
|
||||
**Written only while Betterbird is closed** (§9): the lists in `prefs.js` and the filter files, because
|
||||
Betterbird rewrites them as it runs and quits; the user.js is placed as before. `betterbird_check` counts per
|
||||
kind the changes that wait.
|
||||
|
||||
**Derived, then proposed one layer at a time** (§10): `betterbird_derive` asks every desktop node holding
|
||||
Betterbird for what its profile holds. An entry on every such node goes into the mesh-wide layer with the value of
|
||||
the node the derive was called on, and the answer lists each field another node holds differently (which then
|
||||
shows as drift there); one on some nodes only goes into those nodes' own layers. An account whose filter file
|
||||
holds no filter is not put in `filters`. With `propose`, the next layer (the mesh-wide one, then each node's) is
|
||||
proposed with the layer as it stands beside the derived kinds; never while a proposal of the module's waits, and
|
||||
never while a desktop node does not answer. `prefs` on the call carries values of the setting `prefs` into the
|
||||
mesh-wide layer. A `key` is never proposed for the mesh-wide layer.
|
||||
|
||||
## Tools
|
||||
|
||||
They are served by the node's runtime as the operator account (ADR 0175). **No answer carries mail, a
|
||||
calendar entry, an account, an address or a credential.**
|
||||
They are served by the node's runtime as the operator account (ADR 0175). **They show the operator's own
|
||||
configuration with its values, and never a credential** (ADR 0329 §1, §2): every answer passes the module's
|
||||
credential check, which withholds a field whose name says password, token, secret or cookie and the password
|
||||
part of any address. No answer carries mail, a calendar entry or a contact.
|
||||
|
||||
| tool | does |
|
||||
|---|---|
|
||||
| `betterbird_status` (r) | whether it runs (pid, since, scope, children), the package and version and that it is from the AUR, where its output goes, the profile and how it was found, the user.js (declared, placed, in effect), the add-ons the operator installed (ids), and the preferences that decide how its windows and notifications look |
|
||||
| `betterbird_prefs` (r) | named preferences or prefixes (ending in `.` or `*`, at most 200; an empty name, `*` or `.` is refused): value and source (user.js, prefs.js or the default). Withheld without its value: a name that can hold a credential (`password`, `token`, `oauth`, `signon`, …); one that names an account, server, calendar, recent folder or person (`mail.server.`, `mail.identity.`, `mail.last_msg…`, `messenger.account.`, `mail.cloud_files.`, `calendar.registry.`, …); and every `extensions.*` but the theme's id (`extensions.activeThemeID`). In every value answered, the home is `~`, and a mail address, a `user%40domain@host` URI and a host name are masked |
|
||||
| `betterbird_user_js` (r; a with `apply`) | the declared user.js against the profile's; `apply` places it |
|
||||
| `betterbird_log` (r) | the last lines of the journal of `mesh-betterbird` (or of the reminder test), addresses and credentials masked (the value after `password=`, `token:` and the like, and the token after `Bearer`/`Basic`). A Betterbird the module did not start prints elsewhere, and the answer says where |
|
||||
| `betterbird_restart` (a) | ends Betterbird (asked to quit, forced after 8 s), places the user.js where allowed, starts it as `mesh-betterbird`. **Refused while a composer window is open** (X instance `Msgcompose`, whatever its class), because the unsent message would be lost, unless `discard_unsent` is true, which only the operator's yes justifies. It answers within the runtime's 30 s: a start not yet seen after 4 s is said in `note`, and `betterbird_status` shows it |
|
||||
| `betterbird_check` (r) | installed; one running; profile found; the user.js on the machine, valid, placed, in effect; its output read |
|
||||
| `betterbird_status` (r) | whether it runs (pid, since, scope, children); what is installed and from where, its version beside the newest published build; the policy; Owl's version against the pin and whether it is active; where its output goes; the profile and how it was found; the user.js (declared, placed, in effect, with every declared entry as it resolved); the declared accounts by count; the add-ons; the preferences that decide how its windows and notifications look |
|
||||
| `betterbird_prefs` (r) | named preferences or prefixes: value and source, and the default with the file that gives it. Withheld without its value: a name that can hold a credential, one that names an account, server, calendar or person, and every `extensions.*` but the theme's id. Values masked (the home as `~`, addresses and hosts) |
|
||||
| `betterbird_config` (r; a with `keep`) | each managed preference: its lock, the declared value, Betterbird's default where the module keeps that, the profile's saved value, the value in effect, and drift; with `keep`, this node's layer of `prefs` proposed with the profile's values |
|
||||
| `betterbird_config_apply` (r; a with `apply`) | a kept preference's drifting values listed; with `apply`, while Betterbird is quit, taken out of `prefs.js` |
|
||||
| `betterbird_user_js` (r; a with `apply`) | the user.js composed from the settings against the profile's; `apply` places it, writes what Betterbird rewrites while it is closed, and counts what waits |
|
||||
| `betterbird_log` (r) | the last lines of the journal of `mesh-betterbird` (or of the reminder test), addresses and credentials masked |
|
||||
| `betterbird_restart` (a) | ends Betterbird (asked to quit, forced after 8 s), places the user.js and writes what waited, starts it as `mesh-betterbird`. **Refused while a composer window is open**, unless `discard_unsent` is true, which only the operator's yes justifies |
|
||||
| `betterbird_check` (r) | installed from the mesh's release; the policy and the module's defaults placed; no managed preference drifting; no ask to be the default mail client; Owl at its pin and active; one running; profile found; the user.js valid, placed, in effect; the declared accounts in the list, none refused, each signed in once; per kind, the declared entries drifting or refused and the changes waiting for Betterbird to close; its output read |
|
||||
| `betterbird_accounts` (r; a with `propose`) | the profile's accounts with their values and the declared account each is, Owl's by name and address and marked; with `propose`, `accounts` built from them and proposed in the mesh-wide layer as it stands |
|
||||
| `betterbird_identities`, `betterbird_calendars`, `betterbird_address_books`, `betterbird_filters`, `betterbird_tags` (r) | the profile's entries with their keys and values beside the declared ones and how each resolved (adopted, created or refused with why); an ICS calendar by its name only, Owl's marked; `betterbird_filters` also where the kept filter files are |
|
||||
| `betterbird_derive` (r; a with `propose`) | the five kinds derived from the desktop nodes' profiles, per layer, with the fields another node holds differently and what was left out; with `propose`, the next layer proposed; `here` answers this node's profile, as another node asks it |
|
||||
| `betterbird_drift` (r) | per kind and declared entry: holds, differs (each field with both values), lacks, or refused; and the profile's entries nothing declares |
|
||||
| `betterbird_reminder_test` (a) | a reminder on demand without the operator's calendar (below) |
|
||||
|
||||
### A reminder on demand
|
||||
|
||||
**It opens windows on the operator's screen**, and its description says so first, so a caller tells the
|
||||
operator before calling it. On 2026-10-12 at 12:05 a test Betterbird with an empty profile carrying the
|
||||
operator's theme looked like the operator's configuration lost; so the test is **visibly a test**: its
|
||||
profile carries a `userChrome.css` that frames every window of the test Betterbird in red and runs a red
|
||||
band reading MESH TEST across its top, and its event is called "MESH TEST reminder (a throw-away
|
||||
Betterbird of the mesh, not your calendar)". The reminder window itself is left unstyled, because how
|
||||
Betterbird draws it is what the test looks at; its title is the event's. A window title cannot be set by a
|
||||
preference, so the marking is the band, the frame and the event's name.
|
||||
|
||||
`betterbird_reminder_test` starts a **second** Betterbird with `--no-remote` in a throw-away profile
|
||||
under `/run/user/<uid>/mesh-betterbird-reminder-test/`, as the user unit
|
||||
`mesh-betterbird-reminder-test` (ended after 10 minutes at most). The profile holds the module's
|
||||
user.js and one calendar: a local `.ics` file with one event, "Mesh reminder test", whose reminder is
|
||||
user.js (the setting `prefs`; never a declared account) and one calendar: a local `.ics` file with one
|
||||
event, "MESH TEST reminder", whose reminder is
|
||||
due `after` seconds from the call (default 40). `carry` copies named preferences of the operator's
|
||||
profile into it (a theme, a colour switch), under the same limits as `betterbird_prefs`. Then:
|
||||
profile into it (a theme, a colour switch), under the same limits as `betterbird_prefs`. The test's own
|
||||
preferences come last in its user.js, so nothing declared or carried overrides them; one of them turns
|
||||
Betterbird's update check off (`betterbird.update.checkIntervalDays` 0), because a fresh profile checks
|
||||
at its start and, when a newer build is published, opens the About window on the operator's screen
|
||||
(seen 2026-10-12). Then:
|
||||
|
||||
- `xorg_windows` with class `Calendar` answers the reminder window: role `AlarmWindow`, type `Dialog`,
|
||||
its depth (32: a visual with alpha, which the compositor blends), its map state;
|
||||
@@ -106,13 +321,20 @@ A window of depth 32 that its program has not drawn is transparent under a compo
|
||||
that is why the symptom looks the way it does. Why Betterbird did not draw it is hq issue 345's
|
||||
question.
|
||||
|
||||
## What changes when it is assigned
|
||||
## What changes when it is pushed
|
||||
|
||||
| | g14 | shanks |
|
||||
|---|---|---|
|
||||
| files | `~/.config/betterbird/` and its `user.js` (no preference) | the same |
|
||||
| the profile | untouched until `betterbird_user_js` with `apply` or `betterbird_restart` | the same; the profile has no user.js today, so `apply` may place one |
|
||||
| Betterbird | keeps running as started | keeps running as started (since 2026-10-04, its output on the session's pipe nobody reads: `betterbird_check` names it until `betterbird_restart`) |
|
||||
The module is assigned to shanks today (look-only, from issue 345's first part); g14 runs Betterbird
|
||||
outside the mesh. With this version, a push to a node (each waits for the operator: `upgrade: record`):
|
||||
|
||||
| | on a node with the community package (g14, shanks) |
|
||||
|---|---|
|
||||
| the package | `betterbird-bin` removed (quit Betterbird first) |
|
||||
| `/opt/betterbird` | the mesh's 153.4.0esr-bb10, in the same place, so the same profile opens |
|
||||
| the policy, Owl | placed; Owl installed into the profile by Betterbird at its next start where it is missing |
|
||||
| `/usr/local` | the start, the launcher entry, the tools' program |
|
||||
| `~/.config/betterbird/` | `prefs.json`, `accounts.json`, `configuration.json`, `tags.json` (each empty until its setting is approved) |
|
||||
| the profile | untouched until the next start from the launcher, `betterbird_user_js` with apply, or `betterbird_restart` |
|
||||
| the configuration | the policy's locks at Betterbird's next start; the settings in the user.js from the next start. Pushed in the order above, the operator's 21 preferences are in `prefs` before the build arrives; pushed before that layer is approved, Betterbird runs with its own defaults for them until it is |
|
||||
|
||||
## Relies on
|
||||
|
||||
@@ -120,3 +342,8 @@ question.
|
||||
- **`xorg`** for `xorg_windows` and `xorg_screenshot`, which look at Betterbird's windows. This module
|
||||
carries no window code of its own.
|
||||
- `systemd-run` and `journalctl` of the account's user manager.
|
||||
- The build seat's archive pinned by URL and sha256 (mesh-controller, ADR 0316 §2): a controller that does
|
||||
not know it refuses this manifest.
|
||||
- The controller's `settings` verb, for every proposal and to read a layer as it stands; its `nodes` and `node`
|
||||
verbs, for which machines hold the module; and the module's own `betterbird_derive` on those machines
|
||||
(`invokes`).
|
||||
|
||||
@@ -0,0 +1,842 @@
|
||||
package main
|
||||
|
||||
// The operator's mail accounts, declared without credentials (novox/hq ADR 0316 §10–§14, to-be 56 §4).
|
||||
//
|
||||
// The setting `accounts` reaches the machine as ~/.config/betterbird/accounts.json, a file the node-engine
|
||||
// writes from the module's settings layers. Per account it names the address, the incoming server and the
|
||||
// outgoing server; never a password or a token, and the whole file is refused when any key in it is shaped
|
||||
// like one. Betterbird's accounts are preferences, so a declared account becomes:
|
||||
//
|
||||
// - its declared fields as preferences in the user.js the module places, where they win at every start
|
||||
// (§11): under the keys of the account the profile already holds at that address, which the module adopts
|
||||
// (novox/hq ADR 0317), or, where the profile holds none, under keys of the module's own that end in
|
||||
// mesh_<name> (account_mesh_<name>, id_mesh_<name>, server_mesh_<name>, smtp_mesh_<name>);
|
||||
// - for an account the module created, its key in the account list (mail.accountmanager.accounts) and in the
|
||||
// list of outgoing servers (mail.smtpservers) of the profile's prefs.js: written into, never owned, and
|
||||
// only while Betterbird is not running, because it rewrites prefs.js when it quits (§12). Every other key
|
||||
// keeps its place, and an adopted account's keys are in the lists already.
|
||||
//
|
||||
// Which account is adopted, created or refused is adoption.go's.
|
||||
//
|
||||
// Nobody's password passes through here: an account signs in at its first use, in Betterbird's own window,
|
||||
// and Betterbird keeps the result in its login store (§14). Of that store the module reads only the server
|
||||
// names each entry is kept for, which Betterbird writes in the clear, to count the declared accounts that
|
||||
// have not signed in yet (§16); never a user name or a password, which are encrypted there anyway.
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
)
|
||||
|
||||
// accountsFile is the setting accounts on the machine, written by the node-engine.
|
||||
const accountsFile = ".config/betterbird/accounts.json"
|
||||
|
||||
// Account is one declared mail account, in the shape of the setting accounts: keyed by its name.
|
||||
type Account struct {
|
||||
Address string `json:"address"`
|
||||
// FromName is the name the account's mail is sent under. Absent: Betterbird's, none.
|
||||
FromName string `json:"from-name,omitempty"`
|
||||
// Label is what Betterbird shows the account as. Absent: its address.
|
||||
Label string `json:"label,omitempty"`
|
||||
Incoming Server `json:"incoming"`
|
||||
Outgoing Server `json:"outgoing"`
|
||||
// Key is the profile's account key to adopt, given only in a node's own layer to settle an address the
|
||||
// profile holds more than once (novox/hq ADR 0317 §8).
|
||||
Key string `json:"key,omitempty"`
|
||||
}
|
||||
|
||||
// Server is an account's incoming or outgoing server: never a password or a token.
|
||||
type Server struct {
|
||||
// Kind is imap or pop3, for the incoming server; the outgoing one is smtp and says nothing.
|
||||
Kind string `json:"kind,omitempty"`
|
||||
Host string `json:"host"`
|
||||
Port int `json:"port,omitempty"`
|
||||
Security string `json:"security"`
|
||||
SignIn string `json:"sign-in"`
|
||||
User string `json:"user,omitempty"`
|
||||
}
|
||||
|
||||
// The words a server's connection security and sign-in method are said in, and Betterbird's numbers for
|
||||
// them (socketType / try_ssl, authMethod).
|
||||
var (
|
||||
securities = map[string]int{"none": 0, "starttls": 2, "tls": 3}
|
||||
signIns = map[string]int{"none": 1, "password": 3, "encrypted-password": 4, "kerberos": 5, "ntlm": 6,
|
||||
"tls-certificate": 7, "oauth": 10}
|
||||
)
|
||||
|
||||
func wordFor(words map[string]int, n int) string {
|
||||
for w, v := range words {
|
||||
if v == n {
|
||||
return w
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
var (
|
||||
accountName = regexp.MustCompile(`^[a-z][a-z0-9]{0,30}$`)
|
||||
addressShape = regexp.MustCompile(`^[^\s@"\\]+@[A-Za-z0-9.\-]+\.[A-Za-z]{2,}$`)
|
||||
hostShape = regexp.MustCompile(`^[A-Za-z0-9](?:[A-Za-z0-9.\-]*[A-Za-z0-9])?$`)
|
||||
profileKey = regexp.MustCompile(`^[A-Za-z0-9_]{1,64}$`)
|
||||
// plainText is a value written into a preference: no quote, backslash or line break.
|
||||
plainText = regexp.MustCompile(`^[^"\\\n\r]*$`)
|
||||
// secretKey is a key the accounts file may never carry, at any depth: the setting holds no credential.
|
||||
secretKey = regexp.MustCompile(`(?i)(pass|secret|token|oauth.?refresh|refresh|credential|api.?key|private.?key|cookie|bearer)`)
|
||||
)
|
||||
|
||||
// keysOf are the module's own keys for a declared account: each ends in mesh_<name>.
|
||||
func keysOf(name string) (account, identity, server, smtp string) {
|
||||
return "account_mesh_" + name, "id_mesh_" + name, "server_mesh_" + name, "smtp_mesh_" + name
|
||||
}
|
||||
|
||||
// ownKey is a key of an account list the module wrote, and the name it was declared under.
|
||||
var ownKey = regexp.MustCompile(`^(?:account|smtp)_mesh_([a-z][a-z0-9]{0,30})$`)
|
||||
|
||||
// readAccounts reads the declared accounts. A missing file is no account (the module is older here); a file
|
||||
// that carries a key shaped like a credential, an unknown field or a value that cannot be written is refused
|
||||
// whole, naming the key and never a value.
|
||||
func (m *Machine) readAccounts() (map[string]Account, bool, error) {
|
||||
raw, err := readBounded(m.home(accountsFile))
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
return map[string]Account{}, false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
accounts, err := parseAccounts(raw)
|
||||
return accounts, true, err
|
||||
}
|
||||
|
||||
func parseAccounts(raw []byte) (map[string]Account, error) {
|
||||
var tree any
|
||||
if err := json.Unmarshal(raw, &tree); err != nil {
|
||||
return nil, fmt.Errorf("~/%s is not JSON: %v", accountsFile, err)
|
||||
}
|
||||
if where := secretKeyIn(tree, ""); where != "" {
|
||||
// An account's name is said by nothing: its segment of the path is withheld.
|
||||
if parts := strings.SplitN(where, ".", 3); len(parts) == 3 && parts[0] == "accounts" {
|
||||
where = "accounts.<an account>." + parts[2]
|
||||
}
|
||||
return nil, fmt.Errorf("the setting accounts carries %s, a key shaped like a credential: an account is declared "+
|
||||
"without its password or token, which it signs in with at its first use (novox/hq ADR 0316 §10, §14). "+
|
||||
"Take it out at the controller's terminal", where)
|
||||
}
|
||||
var top map[string]json.RawMessage
|
||||
if err := json.Unmarshal(raw, &top); err != nil {
|
||||
return nil, fmt.Errorf("~/%s is not an object: %v", accountsFile, err)
|
||||
}
|
||||
accounts := map[string]Account{}
|
||||
if body, ok := top["accounts"]; ok && string(body) != "null" {
|
||||
dec := json.NewDecoder(bytes.NewReader(body))
|
||||
dec.DisallowUnknownFields()
|
||||
if err := dec.Decode(&accounts); err != nil {
|
||||
return nil, fmt.Errorf("the setting accounts is not a map of accounts by name: %v", err)
|
||||
}
|
||||
}
|
||||
var problems []string
|
||||
names := sortedNames(accounts)
|
||||
for i, name := range names {
|
||||
problems = append(problems, accountProblems(fmt.Sprintf("declared account %d of %d", i+1, len(names)),
|
||||
name, accounts[name])...)
|
||||
}
|
||||
if len(problems) > 0 {
|
||||
return nil, fmt.Errorf("the setting accounts is refused: %s", strings.Join(problems, "; "))
|
||||
}
|
||||
return accounts, nil
|
||||
}
|
||||
|
||||
// secretKeyIn is the path of the first key shaped like a credential in a JSON tree, or "".
|
||||
func secretKeyIn(node any, at string) string {
|
||||
switch v := node.(type) {
|
||||
case map[string]any:
|
||||
keys := make([]string, 0, len(v))
|
||||
for k := range v {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
for _, k := range keys {
|
||||
here := k
|
||||
if at != "" {
|
||||
here = at + "." + k
|
||||
}
|
||||
if secretKey.MatchString(k) {
|
||||
return here
|
||||
}
|
||||
if w := secretKeyIn(v[k], here); w != "" {
|
||||
return w
|
||||
}
|
||||
}
|
||||
case []any:
|
||||
for i, x := range v {
|
||||
if w := secretKeyIn(x, fmt.Sprintf("%s[%d]", at, i)); w != "" {
|
||||
return w
|
||||
}
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func sortedNames(accounts map[string]Account) []string {
|
||||
names := make([]string, 0, len(accounts))
|
||||
for n := range accounts {
|
||||
names = append(names, n)
|
||||
}
|
||||
sort.Strings(names)
|
||||
return names
|
||||
}
|
||||
|
||||
// accountProblems is what is wrong with one declared account, said of it by its place (label) and never by a
|
||||
// value of it, its name included (novox/hq ADR 0316 §16).
|
||||
func accountProblems(label, name string, a Account) []string {
|
||||
var out []string
|
||||
say := func(format string, args ...any) { out = append(out, label+": "+fmt.Sprintf(format, args...)) }
|
||||
if !accountName.MatchString(name) {
|
||||
say("an account's name is lower-case letters and digits, starting with a letter (it names the module's keys, mesh_<name>)")
|
||||
}
|
||||
if !addressShape.MatchString(a.Address) {
|
||||
say("the address is not a mail address")
|
||||
}
|
||||
if a.Key != "" && !profileKey.MatchString(a.Key) {
|
||||
say("key is an account key of the profile, such as account2")
|
||||
}
|
||||
for field, v := range map[string]string{"from-name": a.FromName, "label": a.Label} {
|
||||
if !plainText.MatchString(v) {
|
||||
say("%s holds a quote, a backslash or a line break", field)
|
||||
}
|
||||
}
|
||||
for side, s := range map[string]Server{"incoming": a.Incoming, "outgoing": a.Outgoing} {
|
||||
switch {
|
||||
case side == "incoming" && s.Kind != "imap" && s.Kind != "pop3":
|
||||
say("the incoming server's kind is imap or pop3")
|
||||
case side == "outgoing" && s.Kind != "" && s.Kind != "smtp":
|
||||
say("the outgoing server is smtp")
|
||||
}
|
||||
if !hostShape.MatchString(s.Host) {
|
||||
say("the %s server's host is not a host name", side)
|
||||
}
|
||||
if s.Port < 0 || s.Port > 65535 {
|
||||
say("the %s server's port is not a port", side)
|
||||
}
|
||||
if _, ok := securities[s.Security]; !ok {
|
||||
say("the %s server's security is tls, starttls or none", side)
|
||||
}
|
||||
if _, ok := signIns[s.SignIn]; !ok || (side == "incoming" && s.SignIn == "none") {
|
||||
say("the %s server's sign-in is password, encrypted-password, oauth, kerberos, ntlm or tls-certificate%s",
|
||||
side, map[bool]string{true: ", or none", false: ""}[side == "outgoing"])
|
||||
}
|
||||
if !plainText.MatchString(s.User) {
|
||||
say("the %s server's user holds a quote, a backslash or a line break", side)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// quoted is a preference's string value as user.js writes it.
|
||||
func quoted(s string) string { return strconv.Quote(s) }
|
||||
|
||||
// declaredPrefs are a declared account's fields as preferences under the keys it resolved to, name to value
|
||||
// as user.js writes it, in a fixed order. Only the fields the setting names (novox/hq ADR 0317 §5): never
|
||||
// the account's directory, folders, filters or caches. An adopted account keeps its kind, the outgoing server
|
||||
// its identity names, and its own name and sender's name where the declaration says none; a created one
|
||||
// gets every field.
|
||||
func declaredPrefs(k keySet, a Account, adopted bool) [][2]string {
|
||||
label := a.Label
|
||||
if label == "" && !adopted {
|
||||
label = a.Address
|
||||
}
|
||||
inPort := a.Incoming.Port
|
||||
if inPort == 0 {
|
||||
inPort = defaultPort(a.Incoming.Kind, a.Incoming.Security)
|
||||
}
|
||||
outPort := a.Outgoing.Port
|
||||
if outPort == 0 {
|
||||
outPort = defaultPort("smtp", a.Outgoing.Security)
|
||||
}
|
||||
var out [][2]string
|
||||
add := func(key, value string) { out = append(out, [2]string{key, value}) }
|
||||
if !adopted {
|
||||
add("mail.account."+k.Account+".identities", quoted(k.Identity))
|
||||
add("mail.account."+k.Account+".server", quoted(k.Server))
|
||||
}
|
||||
if a.FromName != "" || !adopted {
|
||||
add("mail.identity."+k.Identity+".fullName", quoted(a.FromName))
|
||||
}
|
||||
add("mail.identity."+k.Identity+".useremail", quoted(a.Address))
|
||||
if !adopted {
|
||||
add("mail.identity."+k.Identity+".smtpServer", quoted(k.SMTP))
|
||||
add("mail.identity."+k.Identity+".valid", "true")
|
||||
add("mail.server."+k.Server+".type", quoted(a.Incoming.Kind))
|
||||
}
|
||||
add("mail.server."+k.Server+".hostname", quoted(a.Incoming.Host))
|
||||
add("mail.server."+k.Server+".port", strconv.Itoa(inPort))
|
||||
add("mail.server."+k.Server+".socketType", strconv.Itoa(securities[a.Incoming.Security]))
|
||||
add("mail.server."+k.Server+".authMethod", strconv.Itoa(signIns[a.Incoming.SignIn]))
|
||||
add("mail.server."+k.Server+".userName", quoted(a.Incoming.User))
|
||||
if label != "" {
|
||||
add("mail.server."+k.Server+".name", quoted(label))
|
||||
}
|
||||
add("mail.smtpserver."+k.SMTP+".hostname", quoted(a.Outgoing.Host))
|
||||
add("mail.smtpserver."+k.SMTP+".port", strconv.Itoa(outPort))
|
||||
add("mail.smtpserver."+k.SMTP+".try_ssl", strconv.Itoa(securities[a.Outgoing.Security]))
|
||||
add("mail.smtpserver."+k.SMTP+".authMethod", strconv.Itoa(signIns[a.Outgoing.SignIn]))
|
||||
add("mail.smtpserver."+k.SMTP+".username", quoted(a.Outgoing.User))
|
||||
if label != "" {
|
||||
add("mail.smtpserver."+k.SMTP+".description", quoted(label))
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// defaultPort is the port a server listens on when the declaration says none.
|
||||
func defaultPort(kind, security string) int {
|
||||
switch {
|
||||
case kind == "imap" && security == "tls":
|
||||
return 993
|
||||
case kind == "imap":
|
||||
return 143
|
||||
case kind == "pop3" && security == "tls":
|
||||
return 995
|
||||
case kind == "pop3":
|
||||
return 110
|
||||
case security == "tls":
|
||||
return 465
|
||||
case security == "starttls":
|
||||
return 587
|
||||
}
|
||||
return 25
|
||||
}
|
||||
|
||||
// composeUserJS is the user.js the module places in the profile: the declared file (the setting prefs), then
|
||||
// each adopted or created account's declared fields under its keys. A refused account has none.
|
||||
func composeUserJS(declaredText string, accounts map[string]Account, resolved []resolvedAccount) string {
|
||||
var lines []string
|
||||
for _, r := range resolved {
|
||||
if r.How == refused {
|
||||
continue
|
||||
}
|
||||
lines = append(lines, fmt.Sprintf("// declared account %d: %s under %s", r.Place, r.How, r.Keys.Account))
|
||||
for _, p := range declaredPrefs(r.Keys, accounts[r.name], r.How == adopted) {
|
||||
lines = append(lines, fmt.Sprintf("user_pref(%s, %s);", quoted(p[0]), p[1]))
|
||||
}
|
||||
}
|
||||
if len(lines) == 0 {
|
||||
return declaredText
|
||||
}
|
||||
return strings.TrimRight(declaredText, "\n") + "\n\n" +
|
||||
"// The accounts the setting accounts declares (novox/hq ADR 0316 §11, ADR 0317): the fields it names, under\n" +
|
||||
"// the keys of the account adopted or created. No password or token: an account signs in at its first use.\n" +
|
||||
strings.Join(lines, "\n") + "\n"
|
||||
}
|
||||
|
||||
// profileAccount is one account the profile holds, as its preferences say it.
|
||||
type profileAccount struct {
|
||||
Key, Server, Kind string
|
||||
// Identities are its identities, the default first, and Addresses their addresses in that order.
|
||||
Identities []string
|
||||
Addresses []string
|
||||
Owl, Ours bool
|
||||
}
|
||||
|
||||
// profileAccounts reads the accounts the profile's list names, from prefs.js and user.js (user.js wins).
|
||||
func profileAccounts(prefs map[string]string) []profileAccount {
|
||||
var out []profileAccount
|
||||
for _, key := range listOf(prefs["mail.accountmanager.accounts"]) {
|
||||
a := profileAccount{Key: key, Server: unquote(prefs["mail.account."+key+".server"])}
|
||||
a.Kind = unquote(prefs["mail.server."+a.Server+".type"])
|
||||
a.Owl = strings.HasPrefix(a.Kind, "owl")
|
||||
a.Ours = ownKey.MatchString(key)
|
||||
for _, id := range listOf(prefs["mail.account."+key+".identities"]) {
|
||||
a.Identities = append(a.Identities, id)
|
||||
a.Addresses = append(a.Addresses, unquote(prefs["mail.identity."+id+".useremail"]))
|
||||
}
|
||||
out = append(out, a)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// mergedPrefs are a profile's preferences as Betterbird starts with them: prefs.js, then user.js over it.
|
||||
func mergedPrefs(profile string) map[string]string {
|
||||
prefsJS, _ := readBounded(filepath.Join(profile, "prefs.js"))
|
||||
userJS, _ := readBounded(filepath.Join(profile, "user.js"))
|
||||
out := readPrefs(prefsJS)
|
||||
for k, v := range readPrefs(userJS) {
|
||||
out[k] = v
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// listOf reads a comma-separated list preference's value.
|
||||
func listOf(v string) []string {
|
||||
var out []string
|
||||
for _, k := range strings.Split(unquote(v), ",") {
|
||||
if k = strings.TrimSpace(k); k != "" {
|
||||
out = append(out, k)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// unquote is a preference's string value without its quotes; any other value as written.
|
||||
func unquote(v string) string {
|
||||
if s, err := strconv.Unquote(v); err == nil {
|
||||
return s
|
||||
}
|
||||
return v
|
||||
}
|
||||
|
||||
// writeAccountLists writes the module's keys into the profile's account list and list of outgoing servers
|
||||
// (§12): the declared accounts' keys added at the end, the module's keys of accounts no longer declared taken
|
||||
// out, every other key and line kept as it was. Only while Betterbird is not running. Says what changed.
|
||||
func (m *Machine) writeAccountLists(profile string, resolved []resolvedAccount) (string, error) {
|
||||
path := filepath.Join(profile, "prefs.js")
|
||||
raw, err := readBounded(path)
|
||||
if err != nil && !errors.Is(err, os.ErrNotExist) {
|
||||
return "", err
|
||||
}
|
||||
var wantAccounts, wantSMTP []string
|
||||
for _, r := range resolved {
|
||||
if r.How == created {
|
||||
wantAccounts, wantSMTP = append(wantAccounts, r.Keys.Account), append(wantSMTP, r.Keys.SMTP)
|
||||
}
|
||||
}
|
||||
out, changed := withListed(string(raw), map[string][]string{
|
||||
"mail.accountmanager.accounts": wantAccounts, "mail.smtpservers": wantSMTP})
|
||||
if !changed {
|
||||
return "unchanged", nil
|
||||
}
|
||||
// Only while Betterbird is closed: no main process and no lock on the profile (novox/hq ADR 0316 §12, ADR 0329 §9).
|
||||
if why := m.closedFor(profile); why != "" {
|
||||
return "", errors.New(why + ": the account list is written at its next start from the launcher, or by betterbird_restart")
|
||||
}
|
||||
mode := os.FileMode(0o600)
|
||||
if st, err := os.Stat(path); err == nil {
|
||||
mode = st.Mode().Perm()
|
||||
}
|
||||
if err := writeAtomic(path, []byte(out), mode); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return fmt.Sprintf("written: %d account(s) the module created in the account list", len(wantAccounts)), nil
|
||||
}
|
||||
|
||||
// withListed is prefs.js with each list preference holding the module's wanted keys: its own keys that are
|
||||
// not wanted taken out, the wanted ones not there added at its end, everything else as it was. A list the
|
||||
// file does not hold yet is added as a line, when anything is wanted in it.
|
||||
func withListed(text string, want map[string][]string) (string, bool) {
|
||||
changed := false
|
||||
done := map[string]bool{}
|
||||
var lines []string
|
||||
s := bufio.NewScanner(strings.NewReader(text))
|
||||
s.Buffer(make([]byte, 64<<10), 4<<20)
|
||||
for s.Scan() {
|
||||
line := s.Text()
|
||||
if mm := prefLine.FindStringSubmatch(line); mm != nil {
|
||||
if keys, ok := want[mm[1]]; ok && !done[mm[1]] {
|
||||
done[mm[1]] = true
|
||||
next := listedWith(listOf(mm[2]), keys)
|
||||
if next != strings.Join(listOf(mm[2]), ",") {
|
||||
line = fmt.Sprintf("user_pref(%s, %s);", quoted(mm[1]), quoted(next))
|
||||
changed = true
|
||||
}
|
||||
}
|
||||
}
|
||||
lines = append(lines, line)
|
||||
}
|
||||
names := make([]string, 0, len(want))
|
||||
for n := range want {
|
||||
names = append(names, n)
|
||||
}
|
||||
sort.Strings(names)
|
||||
for _, n := range names {
|
||||
if !done[n] && len(want[n]) > 0 {
|
||||
lines = append(lines, fmt.Sprintf("user_pref(%s, %s);", quoted(n), quoted(strings.Join(want[n], ","))))
|
||||
changed = true
|
||||
}
|
||||
}
|
||||
out := strings.Join(lines, "\n")
|
||||
if text == "" || strings.HasSuffix(text, "\n") {
|
||||
out += "\n"
|
||||
}
|
||||
return out, changed
|
||||
}
|
||||
|
||||
// listedWith is a list with the module's own keys made the wanted ones, every other key in its place.
|
||||
func listedWith(have, wanted []string) string {
|
||||
want := map[string]bool{}
|
||||
for _, k := range wanted {
|
||||
want[k] = true
|
||||
}
|
||||
var out []string
|
||||
there := map[string]bool{}
|
||||
for _, k := range have {
|
||||
if ownKey.MatchString(k) && !want[k] {
|
||||
continue
|
||||
}
|
||||
out = append(out, k)
|
||||
there[k] = true
|
||||
}
|
||||
for _, k := range wanted {
|
||||
if !there[k] {
|
||||
out = append(out, k)
|
||||
}
|
||||
}
|
||||
return strings.Join(out, ",")
|
||||
}
|
||||
|
||||
// AccountCount is what status and check say of the accounts: numbers, places and keys, never an address, a
|
||||
// host or a name (novox/hq ADR 0316 §16, ADR 0317 §3).
|
||||
type AccountCount struct {
|
||||
Declared int `json:"declared"`
|
||||
// InList are the declared accounts the profile's account list names, adopted or created.
|
||||
InList int `json:"declared_in_the_list"`
|
||||
// Others are the accounts the profile holds that no declaration names (the work account in Owl among them).
|
||||
Others int `json:"other_accounts"`
|
||||
// WithoutSignIn are the declared accounts with no sign-in kept in Betterbird's login store yet.
|
||||
WithoutSignIn int `json:"declared_without_a_sign_in,omitempty"`
|
||||
// Each is every declared account by its place in the setting: adopted, created or refused, and its keys.
|
||||
Each []resolvedAccount `json:"each,omitempty"`
|
||||
Problem string `json:"problem,omitempty"`
|
||||
}
|
||||
|
||||
// countAccounts counts the declared accounts against the profile's.
|
||||
func (m *Machine) countAccounts(profile, profileDir string) AccountCount {
|
||||
var c AccountCount
|
||||
accounts, _, err := m.readAccounts()
|
||||
if err != nil {
|
||||
c.Problem = err.Error()
|
||||
return c
|
||||
}
|
||||
c.Declared = len(accounts)
|
||||
prefs := mergedPrefs(profile)
|
||||
have := profileAccounts(prefs)
|
||||
c.Each = resolveAccounts(accounts, prefs, m.readRecord(profileDir))
|
||||
mine := map[string]bool{}
|
||||
refusedN := 0
|
||||
for _, r := range c.Each {
|
||||
if r.How == refused {
|
||||
refusedN++
|
||||
continue
|
||||
}
|
||||
mine[r.Keys.Account] = true
|
||||
}
|
||||
for _, a := range have {
|
||||
switch {
|
||||
case mine[a.Key]:
|
||||
c.InList++
|
||||
case a.Kind != "none":
|
||||
c.Others++
|
||||
}
|
||||
}
|
||||
if refusedN > 0 {
|
||||
c.Problem = fmt.Sprintf("%d declared account(s) are refused: each says why and what to do", refusedN)
|
||||
}
|
||||
c.WithoutSignIn = len(withoutSignIn(accounts, keptSignIns(profile)))
|
||||
return c
|
||||
}
|
||||
|
||||
// keptSignIns are the server names Betterbird's login store keeps a sign-in for: the `hostname` of each
|
||||
// entry of logins.json, which Betterbird writes in the clear. Nothing else of the file is read.
|
||||
func keptSignIns(profile string) map[string]bool {
|
||||
raw, err := readBounded(filepath.Join(profile, "logins.json"))
|
||||
out := map[string]bool{}
|
||||
if err != nil {
|
||||
return out
|
||||
}
|
||||
var store struct {
|
||||
Logins []struct {
|
||||
Hostname string `json:"hostname"`
|
||||
} `json:"logins"`
|
||||
}
|
||||
if json.Unmarshal(raw, &store) != nil {
|
||||
return out
|
||||
}
|
||||
for _, l := range store.Logins {
|
||||
out[strings.ToLower(l.Hostname)] = true
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// oauthIssuers are the sign-in services of the providers whose servers sign in through OAuth, by a suffix of
|
||||
// the server's host: their kept sign-in is under the service's name, not the server's.
|
||||
var oauthIssuers = map[string]string{"gmail.com": "oauth://accounts.google.com",
|
||||
"googlemail.com": "oauth://accounts.google.com", "office365.com": "oauth://login.microsoftonline.com",
|
||||
"outlook.com": "oauth://login.microsoftonline.com"}
|
||||
|
||||
// withoutSignIn are the declared accounts whose incoming server has no sign-in kept yet. A server signed in
|
||||
// to without a password (none, kerberos, ntlm, a certificate) keeps nothing and is never counted.
|
||||
func withoutSignIn(accounts map[string]Account, kept map[string]bool) []string {
|
||||
var out []string
|
||||
for _, name := range sortedNames(accounts) {
|
||||
in := accounts[name].Incoming
|
||||
switch in.SignIn {
|
||||
case "password", "encrypted-password":
|
||||
if !kept[strings.ToLower(in.Kind+"://"+in.Host)] {
|
||||
out = append(out, name)
|
||||
}
|
||||
case "oauth":
|
||||
issuer := ""
|
||||
for suffix, i := range oauthIssuers {
|
||||
if strings.HasSuffix(strings.ToLower(in.Host), suffix) {
|
||||
issuer = i
|
||||
}
|
||||
}
|
||||
found := false
|
||||
for k := range kept {
|
||||
if (issuer != "" && k == issuer) || (issuer == "" && strings.HasPrefix(k, "oauth://")) {
|
||||
found = true
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
out = append(out, name)
|
||||
}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// AccountsAnswer is what betterbird_accounts answers its caller: the profile's accounts with the operator's own
|
||||
// values (novox/hq ADR 0329 §1, §12), Owl's marked and by name and address only (§13), and counts and kinds.
|
||||
// Never a password, a token or anything of the login store (§2).
|
||||
type AccountsAnswer struct {
|
||||
Profile string `json:"profile"`
|
||||
// Accounts are the profile's accounts, in its list's order.
|
||||
Accounts []listedAccount `json:"accounts"`
|
||||
// Derived is the value of the setting accounts the propose built, as it was proposed (ADR 0329 §10).
|
||||
Derived map[string]Account `json:"derived,omitempty"`
|
||||
// Declarable are the profile's mail accounts a declaration can hold (IMAP and POP3), by kind.
|
||||
Declarable map[string]int `json:"declarable"`
|
||||
// KeptByOwl are the accounts Owl keeps (the work account through Exchange): never declared (ADR 0316 §13).
|
||||
KeptByOwl int `json:"kept_by_owl"`
|
||||
// NotDeclarable are accounts of another kind (local folders, feeds, news, chat), by kind.
|
||||
NotDeclarable map[string]int `json:"not_declarable,omitempty"`
|
||||
// AlreadyDeclared are the declarable accounts under the module's own keys.
|
||||
AlreadyDeclared int `json:"under_the_modules_keys"`
|
||||
// Proposed is the controller's answer to the proposal, with every address and host masked.
|
||||
Proposed string `json:"proposed,omitempty"`
|
||||
Note string `json:"note"`
|
||||
}
|
||||
|
||||
// listedAccount is one account of the profile as betterbird_accounts lists it.
|
||||
type listedAccount struct {
|
||||
Key string `json:"key"`
|
||||
Kind string `json:"kind"`
|
||||
// DeclaredAs is the name the setting accounts declares it under, or empty: the person's own.
|
||||
DeclaredAs string `json:"declared_as,omitempty"`
|
||||
Owl bool `json:"owl,omitempty"`
|
||||
// Name is what Betterbird shows it as, and Address its default identity's.
|
||||
Name string `json:"name,omitempty"`
|
||||
Address string `json:"address,omitempty"`
|
||||
Value *Account `json:"value,omitempty"`
|
||||
Note string `json:"note,omitempty"`
|
||||
}
|
||||
|
||||
// profileDeclarable reads the profile's accounts: the value a declaration of them would be, by name, and
|
||||
// what is answered of them.
|
||||
func (m *Machine) profileDeclarable() (map[string]Account, AccountsAnswer, error) {
|
||||
p, err := m.profile()
|
||||
if err != nil {
|
||||
return nil, AccountsAnswer{}, err
|
||||
}
|
||||
prefs := mergedPrefs(p.path)
|
||||
a := AccountsAnswer{Profile: p.Dir, Accounts: []listedAccount{}, Declarable: map[string]int{}, NotDeclarable: map[string]int{}}
|
||||
value := map[string]Account{}
|
||||
used := map[string]bool{}
|
||||
declaredAs := map[string]string{}
|
||||
if accounts, _, err := m.readAccounts(); err == nil {
|
||||
for _, r := range resolveAccounts(accounts, prefs, m.readRecord(p.Dir)) {
|
||||
if r.How != refused {
|
||||
declaredAs[r.Keys.Account] = r.name
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, pa := range profileAccounts(prefs) {
|
||||
listed := listedAccount{Key: pa.Key, Kind: pa.Kind, DeclaredAs: declaredAs[pa.Key], Owl: pa.Owl,
|
||||
Name: unquote(prefs["mail.server."+pa.Server+".name"])}
|
||||
if len(pa.Addresses) > 0 {
|
||||
listed.Address = pa.Addresses[0]
|
||||
}
|
||||
switch {
|
||||
case pa.Owl:
|
||||
a.KeptByOwl++
|
||||
listed.Note = "Owl's: never declared (novox/hq ADR 0316 §13)"
|
||||
a.Accounts = append(a.Accounts, listed)
|
||||
continue
|
||||
case pa.Kind != "imap" && pa.Kind != "pop3":
|
||||
kind := pa.Kind
|
||||
if kind == "none" {
|
||||
kind = "local folders"
|
||||
}
|
||||
a.NotDeclarable[kind]++
|
||||
listed.Note = "not an account a declaration holds"
|
||||
a.Accounts = append(a.Accounts, listed)
|
||||
continue
|
||||
}
|
||||
a.Declarable[pa.Kind]++
|
||||
if pa.Ours {
|
||||
a.AlreadyDeclared++
|
||||
}
|
||||
address := ""
|
||||
if len(pa.Addresses) > 0 {
|
||||
address = pa.Addresses[0]
|
||||
}
|
||||
name := nameFor(pa, address, used)
|
||||
if declaredAs[pa.Key] != "" {
|
||||
name = declaredAs[pa.Key]
|
||||
}
|
||||
used[name] = true
|
||||
acc := readAccount(prefs, pa)
|
||||
value[name] = acc
|
||||
listed.Value = &acc
|
||||
a.Accounts = append(a.Accounts, listed)
|
||||
}
|
||||
if len(a.NotDeclarable) == 0 {
|
||||
a.NotDeclarable = nil
|
||||
}
|
||||
return value, a, nil
|
||||
}
|
||||
|
||||
// asker calls a verb of the mesh as this module, through the runtime: a variable so a test answers.
|
||||
var asker = stdio.Ask
|
||||
|
||||
// ProfileAccounts says, by count and kind, which accounts the profile in use holds; with propose it builds
|
||||
// the setting accounts from them (every account but Owl's, without any credential) and proposes it through
|
||||
// the controller's settings verb for the operator's Approve (novox/hq ADR 0317 §9, ADR 0277). The value
|
||||
// never comes back to the caller: the operator reads it whole at their desk (`settings proposals <id>`) and
|
||||
// checks the fingerprint the ask shows.
|
||||
func (m *Machine) ProfileAccounts(propose bool) (AccountsAnswer, error) {
|
||||
value, a, err := m.profileDeclarable()
|
||||
if err != nil {
|
||||
return a, err
|
||||
}
|
||||
a.Note = "the profile's accounts with their values, never a password or token (novox/hq ADR 0329 §1, §2). With " +
|
||||
"propose, the setting accounts is built from them (never Owl's) and proposed to the operator in the mesh-wide " +
|
||||
"layer as it stands, who approves it on a channel that proves who answers (novox/hq ADR 0317 §9)"
|
||||
if !propose {
|
||||
return a, nil
|
||||
}
|
||||
if len(value) == 0 {
|
||||
return a, errors.New("the profile holds no IMAP or POP3 account to declare: nothing was proposed")
|
||||
}
|
||||
var problems []string
|
||||
for i, name := range sortedNames(value) {
|
||||
if p := accountProblems("", name, value[name]); len(p) > 0 {
|
||||
problems = append(problems, fmt.Sprintf("account %d of %d: %d field(s) cannot be declared as the profile "+
|
||||
"holds them", i+1, len(value), len(p)))
|
||||
}
|
||||
}
|
||||
if len(problems) > 0 {
|
||||
return a, fmt.Errorf("nothing was proposed: %s; betterbird_prefs shows the account's preferences by name",
|
||||
strings.Join(problems, "; "))
|
||||
}
|
||||
// A layer is replaced whole: the proposal carries the mesh-wide layer as it stands, with this value of accounts.
|
||||
layer, err := readLayer("")
|
||||
if err != nil {
|
||||
return a, fmt.Errorf("nothing was proposed: %v", err)
|
||||
}
|
||||
layer["accounts"] = value
|
||||
a.Derived = value
|
||||
said, err := proposeLayer("", layer, false)
|
||||
a.Proposed = m.maskValue(said)
|
||||
return a, err
|
||||
}
|
||||
|
||||
// verbSaid is a verb's words and whether it went ahead, whatever wraps them: the controller's {ok, output},
|
||||
// or the protocol's content list.
|
||||
func verbSaid(raw json.RawMessage) (string, bool) {
|
||||
var s string
|
||||
if json.Unmarshal(raw, &s) == nil {
|
||||
return verbSaid(json.RawMessage(s))
|
||||
}
|
||||
var m map[string]json.RawMessage
|
||||
if json.Unmarshal(raw, &m) != nil {
|
||||
return string(raw), true
|
||||
}
|
||||
if content, has := m["content"]; has {
|
||||
var items []struct {
|
||||
Text string `json:"text"`
|
||||
}
|
||||
var e struct {
|
||||
IsError bool `json:"isError"`
|
||||
}
|
||||
_ = json.Unmarshal(raw, &e)
|
||||
if json.Unmarshal(content, &items) == nil && len(items) > 0 {
|
||||
out, ok := verbSaid(json.RawMessage(items[0].Text))
|
||||
return out, ok && !e.IsError
|
||||
}
|
||||
}
|
||||
if okRaw, has := m["ok"]; has {
|
||||
var ok bool
|
||||
_ = json.Unmarshal(okRaw, &ok)
|
||||
var output string
|
||||
_ = json.Unmarshal(m["output"], &output)
|
||||
return output, ok
|
||||
}
|
||||
return string(raw), true
|
||||
}
|
||||
|
||||
// readAccount reads one profile account's preferences into the shape of the setting.
|
||||
func readAccount(prefs map[string]string, pa profileAccount) Account {
|
||||
num := func(key string, def int) int {
|
||||
if n, err := strconv.Atoi(prefs[key]); err == nil {
|
||||
return n
|
||||
}
|
||||
return def
|
||||
}
|
||||
text := func(key string) string { return unquote(prefs[key]) }
|
||||
srv := "mail.server." + pa.Server + "."
|
||||
acc := Account{Incoming: Server{Kind: pa.Kind, Host: text(srv + "hostname"), User: text(srv + "userName"),
|
||||
Security: wordFor(securities, num(srv+"socketType", 0)), SignIn: wordFor(signIns, num(srv+"authMethod", 3))}}
|
||||
acc.Incoming.Port = num(srv+"port", defaultPort(pa.Kind, acc.Incoming.Security))
|
||||
ids := listOf(prefs["mail.account."+pa.Key+".identities"])
|
||||
if len(ids) > 0 {
|
||||
id := "mail.identity." + ids[0] + "."
|
||||
acc.Address, acc.FromName = text(id+"useremail"), text(id+"fullName")
|
||||
smtp := text(id + "smtpServer")
|
||||
if smtp == "" {
|
||||
smtp = text("mail.smtp.defaultserver")
|
||||
}
|
||||
if smtp != "" {
|
||||
s := "mail.smtpserver." + smtp + "."
|
||||
acc.Outgoing = Server{Host: text(s + "hostname"), User: text(s + "username"),
|
||||
Security: wordFor(securities, num(s+"try_ssl", 0)), SignIn: wordFor(signIns, num(s+"authMethod", 3))}
|
||||
acc.Outgoing.Port = num(s+"port", defaultPort("smtp", acc.Outgoing.Security))
|
||||
}
|
||||
}
|
||||
if label := text(srv + "name"); label != "" && label != acc.Address {
|
||||
acc.Label = label
|
||||
}
|
||||
return acc
|
||||
}
|
||||
|
||||
// nameFor is the name a profile account is answered under: its own name when the module declared it, else
|
||||
// the first label of its address's domain, made unique.
|
||||
func nameFor(pa profileAccount, address string, used map[string]bool) string {
|
||||
if mm := ownKey.FindStringSubmatch(pa.Key); mm != nil {
|
||||
return mm[1]
|
||||
}
|
||||
base := "account"
|
||||
if _, domain, ok := strings.Cut(address, "@"); ok {
|
||||
label, _, _ := strings.Cut(strings.ToLower(domain), ".")
|
||||
label = regexp.MustCompile(`[^a-z0-9]`).ReplaceAllString(label, "")
|
||||
if label != "" && label[0] >= 'a' && label[0] <= 'z' {
|
||||
base = label
|
||||
}
|
||||
}
|
||||
if len(base) > 28 {
|
||||
base = base[:28]
|
||||
}
|
||||
name := base
|
||||
for i := 2; used[name]; i++ {
|
||||
name = base + strconv.Itoa(i)
|
||||
}
|
||||
return name
|
||||
}
|
||||
@@ -0,0 +1,567 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// The setting accounts (novox/hq ADR 0316 §10–§16, ADR 0317 §9 for betterbird_accounts): declared without
|
||||
// credentials, written under the module's own keys, the account list written into and never owned, refused
|
||||
// rather than duplicated, and said by count.
|
||||
|
||||
const personal = `{"accounts": {
|
||||
"personal": {"address": "me@mail.example.org", "from-name": "Me",
|
||||
"incoming": {"kind": "imap", "host": "imap.example.org", "security": "tls", "sign-in": "password", "user": "me"},
|
||||
"outgoing": {"host": "smtp.example.org", "port": 587, "security": "starttls", "sign-in": "password", "user": "me"}}
|
||||
}}`
|
||||
|
||||
// profileWithAccounts is a profile holding an account of the person's at another address, Owl's work account,
|
||||
// local folders, and an account list in that order.
|
||||
func (f *fake) profileWithAccounts() {
|
||||
f.betterbird()
|
||||
f.write(profileDir+"/prefs.js", "// Mozilla User Preferences\n"+
|
||||
`user_pref("mail.account.account1.server", "server1");`+"\n"+
|
||||
`user_pref("mail.account.account1.identities", "id1");`+"\n"+
|
||||
`user_pref("mail.account.account2.server", "server2");`+"\n"+
|
||||
`user_pref("mail.account.account3.server", "server3");`+"\n"+
|
||||
`user_pref("mail.account.account3.identities", "id3");`+"\n"+
|
||||
`user_pref("mail.accountmanager.accounts", "account1,account3,account2");`+"\n"+
|
||||
`user_pref("mail.identity.id1.useremail", "other@mail.example.org");`+"\n"+
|
||||
`user_pref("mail.identity.id1.fullName", "Other");`+"\n"+
|
||||
`user_pref("mail.identity.id1.smtpServer", "smtp1");`+"\n"+
|
||||
`user_pref("mail.identity.id3.useremail", "work@corp.example.com");`+"\n"+
|
||||
`user_pref("mail.server.server1.type", "imap");`+"\n"+
|
||||
`user_pref("mail.server.server1.hostname", "imap.example.org");`+"\n"+
|
||||
`user_pref("mail.server.server1.socketType", 3);`+"\n"+
|
||||
`user_pref("mail.server.server1.authMethod", 10);`+"\n"+
|
||||
`user_pref("mail.server.server1.userName", "other");`+"\n"+
|
||||
`user_pref("mail.server.server1.directory", "/home/operator/.thunderbird/x/ImapMail/imap.example.org");`+"\n"+
|
||||
`user_pref("mail.server.server2.type", "none");`+"\n"+
|
||||
`user_pref("mail.server.server3.type", "owl-ews");`+"\n"+
|
||||
`user_pref("mail.smtpserver.smtp1.hostname", "smtp.example.org");`+"\n"+
|
||||
`user_pref("mail.smtpserver.smtp1.try_ssl", 3);`+"\n"+
|
||||
`user_pref("mail.smtpserver.smtp1.authMethod", 10);`+"\n"+
|
||||
`user_pref("mail.smtpserver.smtp1.username", "other");`+"\n"+
|
||||
`user_pref("mail.smtpservers", "smtp1");`+"\n"+
|
||||
`user_pref("mailnews.tags.version", 2);`+"\n")
|
||||
}
|
||||
|
||||
func (f *fake) declareAccounts(body string) {
|
||||
f.write(testHome+"/"+accountsFile, body)
|
||||
}
|
||||
|
||||
func (f *fake) read(path string) string {
|
||||
f.t.Helper()
|
||||
raw, err := os.ReadFile(filepath.Join(f.Root, path))
|
||||
if err != nil {
|
||||
f.t.Fatal(err)
|
||||
}
|
||||
return string(raw)
|
||||
}
|
||||
|
||||
func TestAnAccountsValueCarryingAPasswordLikeFieldIsRefusedWithoutItsValue(t *testing.T) {
|
||||
for _, body := range []string{
|
||||
`{"accounts": {"personal": {"address": "me@mail.example.org", "password": "hunter2"}}}`,
|
||||
`{"accounts": {"personal": {"incoming": {"oauth_refresh": "1//hunter2"}}}}`,
|
||||
`{"token": "hunter2", "accounts": {}}`,
|
||||
} {
|
||||
_, err := parseAccounts([]byte(body))
|
||||
if err == nil || !strings.Contains(err.Error(), "shaped like a credential") || strings.Contains(err.Error(), "hunter2") {
|
||||
t.Errorf("%s: %v", body, err)
|
||||
}
|
||||
}
|
||||
// An unknown field is refused too, and a value that cannot be declared, said by the account's place.
|
||||
if _, err := parseAccounts([]byte(`{"accounts": {"personal": {"address": "me@mail.example.org", "keychain": 1}}}`)); err == nil {
|
||||
t.Error("an unknown field was taken")
|
||||
}
|
||||
_, err := parseAccounts([]byte(strings.Replace(personal, `"security": "tls"`, `"security": "maybe"`, 1)))
|
||||
if err == nil || !strings.Contains(err.Error(), "declared account 1 of 1") || strings.Contains(err.Error(), "personal") ||
|
||||
strings.Contains(err.Error(), "example.org") {
|
||||
t.Errorf("a bad value is said by the account's place, never its name or a value: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheSettingPrefsRefusesAnAccountsPreferences(t *testing.T) {
|
||||
for _, name := range []string{"mail.account.account1.server", "mail.identity.id1.useremail", "mail.server.server1.hostname",
|
||||
"mail.smtpserver.smtp1.hostname", "mail.accountmanager.accounts", "mail.smtpservers"} {
|
||||
err := validUserJS(mark + ")\nuser_pref(\"" + name + "\", \"x\");\n")
|
||||
if err == nil || !strings.Contains(err.Error(), "setting accounts") {
|
||||
t.Errorf("%s was taken in prefs: %v", name, err)
|
||||
}
|
||||
}
|
||||
if err := validUserJS(mark + ")\nuser_pref(\"calendar.alarms.show\", true);\n"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// noPackage answers pacman as a machine without the community package.
|
||||
func (f *fake) noPackage() {
|
||||
f.answer = func(name string, args []string) Output {
|
||||
if name == "pacman" {
|
||||
return Output{Code: 1}
|
||||
}
|
||||
return Output{}
|
||||
}
|
||||
}
|
||||
|
||||
// userPrefLines are the user_pref lines of a user.js, by preference name.
|
||||
func userPrefLines(t *testing.T, text string) map[string]string {
|
||||
t.Helper()
|
||||
out := map[string]string{}
|
||||
for _, line := range strings.Split(text, "\n") {
|
||||
if !strings.HasPrefix(line, "user_pref(") {
|
||||
continue
|
||||
}
|
||||
if !statement.MatchString(line) {
|
||||
t.Errorf("not a statement Betterbird reads as one: %s", line)
|
||||
}
|
||||
out[line[len(`user_pref("`):strings.Index(line, `",`)]] = line
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func TestADeclaredAccountTheProfileDoesNotHoldIsCreatedUnderTheModulesKeysAndListed(t *testing.T) {
|
||||
f := newFake(t)
|
||||
f.profileWithAccounts()
|
||||
f.noPackage()
|
||||
links(t, nil)
|
||||
f.declare("// none")
|
||||
f.declareAccounts(personal)
|
||||
// Betterbird is running: the user.js is placed, the list waits for its next start.
|
||||
u, err := f.userJS(true)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !strings.HasPrefix(u.Lists, "not written") || !strings.Contains(u.Lists, "running") {
|
||||
t.Fatalf("the list is never written while Betterbird runs: %+v", u)
|
||||
}
|
||||
if len(u.Each) != 1 || u.Each[0].How != created || u.Each[0].Keys != (keySet{"account_mesh_personal", "id_mesh_personal",
|
||||
"server_mesh_personal", "smtp_mesh_personal"}) {
|
||||
t.Fatalf("%+v", u.Each)
|
||||
}
|
||||
placed := f.read(profileDir + "/user.js")
|
||||
for name := range userPrefLines(t, placed) {
|
||||
if key := strings.Split(name, ".")[2]; !strings.HasSuffix(key, "mesh_personal") {
|
||||
t.Errorf("a created account's preference outside the module's keys: %s", name)
|
||||
}
|
||||
if strings.Contains(strings.ToLower(name), "password") || strings.Contains(name, "directory") {
|
||||
t.Errorf("the module writes no credential and none of Betterbird's own preferences: %s", name)
|
||||
}
|
||||
}
|
||||
for _, want := range []string{`user_pref("mail.server.server_mesh_personal.port", 993);`,
|
||||
`user_pref("mail.server.server_mesh_personal.socketType", 3);`,
|
||||
`user_pref("mail.smtpserver.smtp_mesh_personal.try_ssl", 2);`,
|
||||
`user_pref("mail.identity.id_mesh_personal.useremail", "me@mail.example.org");`} {
|
||||
if !strings.Contains(placed, want) {
|
||||
t.Errorf("the user.js lacks %s:\n%s", want, placed)
|
||||
}
|
||||
}
|
||||
|
||||
// Quit: the list gains the module's keys at its end, every other line as it was.
|
||||
before := f.read(profileDir + "/prefs.js")
|
||||
os.RemoveAll(filepath.Join(f.Root, "proc", "4000"))
|
||||
if err := f.Prepare(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
after := f.read(profileDir + "/prefs.js")
|
||||
if !strings.Contains(after, `user_pref("mail.accountmanager.accounts", "account1,account3,account2,account_mesh_personal");`) ||
|
||||
!strings.Contains(after, `user_pref("mail.smtpservers", "smtp1,smtp_mesh_personal");`) {
|
||||
t.Fatalf("the lists:\n%s", after)
|
||||
}
|
||||
bl, al := strings.Split(before, "\n"), strings.Split(after, "\n")
|
||||
if len(bl) != len(al) {
|
||||
t.Fatalf("lines were added or lost: %d then %d", len(bl), len(al))
|
||||
}
|
||||
for i := range bl {
|
||||
if bl[i] != al[i] && !strings.Contains(bl[i], "mail.accountmanager.accounts") && !strings.Contains(bl[i], "mail.smtpservers") {
|
||||
t.Errorf("line %d changed: %q to %q", i+1, bl[i], al[i])
|
||||
}
|
||||
}
|
||||
|
||||
// The record lost: an account under the module's own keys is still its creation, never adopted.
|
||||
os.Remove(filepath.Join(f.Root, testHome, recordFile))
|
||||
u, err = f.userJS(true)
|
||||
if err != nil || u.Each[0].How != created {
|
||||
t.Fatalf("without the record, the module's keys still say created: %+v %v", u.Each, err)
|
||||
}
|
||||
|
||||
// The account leaves the setting: only the module's keys go, the mail stays in the profile.
|
||||
f.declareAccounts(`{"accounts": {}}`)
|
||||
if err := f.Prepare(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := f.read(profileDir + "/prefs.js"); got != before {
|
||||
t.Fatalf("removing the module's keys left the file otherwise as it was:\n%s", got)
|
||||
}
|
||||
}
|
||||
|
||||
// adoptable is a declaration of the profile's account1 (other@mail.example.org): equal but for its outgoing
|
||||
// port, said explicitly, and its incoming user.
|
||||
const adoptable = `{"accounts": {
|
||||
"mail": {"address": "Other@mail.example.org",
|
||||
"incoming": {"kind": "imap", "host": "imap.example.org", "security": "tls", "sign-in": "oauth", "user": "other2"},
|
||||
"outgoing": {"host": "smtp.example.org", "port": 465, "security": "tls", "sign-in": "oauth", "user": "other"}}}}`
|
||||
|
||||
func TestADeclaredAccountTheProfileHoldsIsAdoptedUnderItsOwnKeysAndTheListIsLeftAlone(t *testing.T) {
|
||||
f := newFake(t)
|
||||
f.profileWithAccounts()
|
||||
f.noPackage()
|
||||
links(t, nil)
|
||||
f.declare("// none")
|
||||
f.declareAccounts(adoptable)
|
||||
os.RemoveAll(filepath.Join(f.Root, "proc", "4000"))
|
||||
before := f.read(profileDir + "/prefs.js")
|
||||
u, err := f.userJS(true)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(u.Each) != 1 || u.Each[0].How != adopted || u.Each[0].Keys != (keySet{"account1", "id1", "server1", "smtp1"}) {
|
||||
t.Fatalf("adopted under the profile's keys: %+v", u.Each)
|
||||
}
|
||||
// The address's case and the user name differ; the default port declared explicitly is no change. Names
|
||||
// only, no value.
|
||||
if strings.Join(u.Each[0].Changed, ",") != "mail.identity.id1.useremail,mail.server.server1.userName" {
|
||||
t.Fatalf("the changed preferences, by name: %v", u.Each[0].Changed)
|
||||
}
|
||||
if f.read(profileDir+"/prefs.js") != before || u.Lists != "unchanged" {
|
||||
t.Fatalf("an adopted account's keys are in the lists already: nothing written to prefs.js (%s)", u.Lists)
|
||||
}
|
||||
lines := userPrefLines(t, f.read(profileDir+"/user.js"))
|
||||
for name := range lines {
|
||||
key := strings.Split(name, ".")[2]
|
||||
if key != "account1" && key != "id1" && key != "server1" && key != "smtp1" {
|
||||
t.Errorf("an adopted account's preference outside its keys: %s", name)
|
||||
}
|
||||
for _, never := range []string{"directory", ".type", "smtpServer", "identities", ".server"} {
|
||||
if strings.HasSuffix(name, never) {
|
||||
t.Errorf("the module wrote %s, which is Betterbird's or the account's own", name)
|
||||
}
|
||||
}
|
||||
}
|
||||
if !strings.Contains(lines["mail.server.server1.userName"], `"other2"`) {
|
||||
t.Fatalf("the declared value wins at every start: %v", lines)
|
||||
}
|
||||
// The status says it by place and keys, never by address, host or name.
|
||||
s, err := f.Status()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
raw, _ := json.Marshal(s.Accounts)
|
||||
if s.Accounts.InList != 1 || s.Accounts.Others != 1 || strings.Contains(string(raw), "example") ||
|
||||
strings.Contains(string(raw), `"mail"`) || !strings.Contains(string(raw), `"account1"`) {
|
||||
t.Fatalf("%s", raw)
|
||||
}
|
||||
|
||||
// Leaving the setting releases it: its lines and its record go, the account and the lists stay.
|
||||
f.declareAccounts(`{"accounts": {}}`)
|
||||
if err := f.Prepare(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if f.read(profileDir+"/prefs.js") != before || strings.Contains(f.read(profileDir+"/user.js"), "server1") ||
|
||||
strings.Contains(f.read(testHome+"/"+recordFile), "account1") {
|
||||
t.Fatal("a released account stays the person's, with no line or record of the module's")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnAmbiguousOwlsOrOtherKindOfMatchIsRefusedForThatAccountAlone(t *testing.T) {
|
||||
f := newFake(t)
|
||||
f.profileWithAccounts()
|
||||
f.noPackage()
|
||||
links(t, nil)
|
||||
f.declare("// none")
|
||||
os.RemoveAll(filepath.Join(f.Root, "proc", "4000"))
|
||||
// A second account with account1's address.
|
||||
prefs := f.read(profileDir + "/prefs.js")
|
||||
prefs = strings.Replace(prefs, `"account1,account3,account2"`, `"account1,account3,account2,account4"`, 1) +
|
||||
`user_pref("mail.account.account4.server", "server4");` + "\n" +
|
||||
`user_pref("mail.account.account4.identities", "id4");` + "\n" +
|
||||
`user_pref("mail.identity.id4.useremail", "other@mail.example.org");` + "\n" +
|
||||
`user_pref("mail.identity.id4.smtpServer", "smtp4");` + "\n" +
|
||||
`user_pref("mail.smtpserver.smtp4.hostname", "smtp.example.org");` + "\n" +
|
||||
`user_pref("mail.server.server4.type", "imap");` + "\n"
|
||||
f.write(profileDir+"/prefs.js", prefs)
|
||||
both := strings.Replace(adoptable, `{"accounts": {`, `{"accounts": {
|
||||
"zz": {"address": "new@mail.example.org",
|
||||
"incoming": {"kind": "imap", "host": "imap.example.org", "security": "tls", "sign-in": "password"},
|
||||
"outgoing": {"host": "smtp.example.org", "security": "tls", "sign-in": "password"}},`, 1)
|
||||
f.declareAccounts(both)
|
||||
err := f.Prepare()
|
||||
if err == nil || !strings.Contains(err.Error(), "declared account 1 of 2: 2 accounts in the profile have its address (account1, account4)") ||
|
||||
!strings.Contains(err.Error(), "key") || strings.Contains(err.Error(), "example.org") {
|
||||
t.Fatalf("refused by place, with the matches' keys and the remedy, never the address: %v", err)
|
||||
}
|
||||
u, _ := f.userJS(false)
|
||||
if u.Each[0].How != refused || u.Each[1].How != created {
|
||||
t.Fatalf("the other declared account goes ahead: %+v", u.Each)
|
||||
}
|
||||
if strings.Contains(f.read(profileDir+"/user.js"), "server1") || strings.Contains(f.read(profileDir+"/user.js"), "server4") {
|
||||
t.Fatal("a refused account gets no line")
|
||||
}
|
||||
|
||||
// The node's layer names the key: that one is adopted.
|
||||
f.declareAccounts(strings.Replace(both, `"address": "Other@mail.example.org",`, `"address": "Other@mail.example.org", "key": "account4",`, 1))
|
||||
if err := f.Prepare(); err != nil {
|
||||
t.Fatalf("%v", err)
|
||||
}
|
||||
if u, _ := f.userJS(false); u.Each[0].How != adopted || u.Each[0].Keys.Account != "account4" {
|
||||
t.Fatalf("%+v", u.Each)
|
||||
}
|
||||
os.Remove(filepath.Join(f.Root, testHome, recordFile))
|
||||
for key, want := range map[string]string{"account9": "not in the profile's account list", "account3": "Owl's",
|
||||
"account2": "another address"} {
|
||||
f.declareAccounts(strings.Replace(adoptable, `"address": "Other@mail.example.org",`, `"address": "Other@mail.example.org", "key": "`+key+`",`, 1))
|
||||
if err := f.Prepare(); err == nil || !strings.Contains(err.Error(), want) {
|
||||
t.Errorf("key %s: %v", key, err)
|
||||
}
|
||||
}
|
||||
|
||||
// Owl's account at the address, and an account of another kind than declared.
|
||||
f.declareAccounts(`{"accounts": {"work": {"address": "work@corp.example.com",
|
||||
"incoming": {"kind": "imap", "host": "outlook.example.com", "security": "tls", "sign-in": "oauth"},
|
||||
"outgoing": {"host": "smtp.example.com", "security": "tls", "sign-in": "oauth"}}}}`)
|
||||
if err := f.Prepare(); err == nil || !strings.Contains(err.Error(), "Owl's") {
|
||||
t.Errorf("Owl's account is never adopted: %v", err)
|
||||
}
|
||||
f.write(profileDir+"/prefs.js", strings.Replace(prefs, `"mail.server.server4.type", "imap"`, `"mail.server.server4.type", "pop3"`, 1))
|
||||
f.declareAccounts(strings.Replace(adoptable, `"address": "Other@mail.example.org",`, `"address": "Other@mail.example.org", "key": "account4",`, 1))
|
||||
if err := f.Prepare(); err == nil || !strings.Contains(err.Error(), "never converts") {
|
||||
t.Errorf("an account of another kind is refused, not converted: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeclaredAccountsWithoutASignInAreCountedFromTheServerNamesAlone(t *testing.T) {
|
||||
f := newFake(t)
|
||||
f.profileWithAccounts()
|
||||
links(t, nil)
|
||||
f.declareAccounts(`{"accounts": {
|
||||
"personal": {"address": "me@mail.example.org", "incoming": {"kind": "imap", "host": "imap.example.org", "security": "tls", "sign-in": "password"},
|
||||
"outgoing": {"host": "smtp.example.org", "security": "tls", "sign-in": "password"}},
|
||||
"google": {"address": "me@gmail.com", "incoming": {"kind": "imap", "host": "imap.gmail.com", "security": "tls", "sign-in": "oauth"},
|
||||
"outgoing": {"host": "smtp.gmail.com", "security": "tls", "sign-in": "oauth"}},
|
||||
"lan": {"address": "me@lan.example.org", "incoming": {"kind": "pop3", "host": "pop.example.org", "security": "tls", "sign-in": "kerberos"},
|
||||
"outgoing": {"host": "smtp.example.org", "security": "tls", "sign-in": "none"}}}}`)
|
||||
// The username and password fields are encrypted and never read; a fixture holding junk there proves it.
|
||||
f.write(profileDir+"/logins.json", `{"logins": [
|
||||
{"hostname": "imap://imap.example.org", "httpRealm": "imap://imap.example.org", "encryptedUsername": "MEIE...", "encryptedPassword": "MDoE..."},
|
||||
{"hostname": "smtp://smtp.example.org", "encryptedUsername": "x", "encryptedPassword": "y"}]}`)
|
||||
c := f.countAccounts(filepath.Join(f.Root, profileDir), "~/.thunderbird/60016jsp.default-default")
|
||||
if c.Declared != 3 || c.WithoutSignIn != 1 || c.Others != 2 || c.InList != 0 {
|
||||
t.Fatalf("three declared, the OAuth one not signed in, the person's and Owl's other (local folders not counted): %+v", c)
|
||||
}
|
||||
raw, _ := json.Marshal(c)
|
||||
if strings.Contains(string(raw), "example") || strings.Contains(string(raw), "gmail") {
|
||||
t.Fatalf("a count names nothing: %s", raw)
|
||||
}
|
||||
}
|
||||
|
||||
// ADR 0329 §1, §10, §12, §13: betterbird_accounts lists the accounts with their values, Owl's by name and address
|
||||
// and marked; its propose carries the mesh-wide layer as it stands with accounts in it, and answers the value.
|
||||
func TestBetterbirdAccountsListsTheValuesAndProposesThemInTheLayerAsItStands(t *testing.T) {
|
||||
f := newFake(t)
|
||||
f.profileWithAccounts()
|
||||
f.write(profileDir+"/logins.json", `{"logins":[{"hostname":"imap://imap.example.org","encryptedUsername":"MDoEEPgAAAAA","encryptedPassword":"hunter2secret"}]}`)
|
||||
links(t, nil)
|
||||
a, err := f.ProfileAccounts(false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if a.Declarable["imap"] != 1 || a.KeptByOwl != 1 || a.NotDeclarable["local folders"] != 1 || a.Proposed != "" {
|
||||
t.Fatalf("%+v", a)
|
||||
}
|
||||
raw, _ := json.Marshal(credentialChecked(a))
|
||||
for _, shown := range []string{"other@mail.example.org", "imap.example.org", "smtp.example.org", `"port":993`, `"work@corp.example.com"`} {
|
||||
if !strings.Contains(string(raw), shown) {
|
||||
t.Errorf("the answer shows the operator's own %s (ADR 0329 §1): %s", shown, raw)
|
||||
}
|
||||
}
|
||||
for _, never := range []string{"hunter2", "MDoEEP"} {
|
||||
if strings.Contains(string(raw), never) {
|
||||
t.Errorf("the answer carries the login store's %q", never)
|
||||
}
|
||||
}
|
||||
for _, l := range a.Accounts {
|
||||
if l.Key == "account3" && (!l.Owl || l.Value != nil || l.Address != "work@corp.example.com") {
|
||||
t.Errorf("Owl's account is listed by name and address, marked, never as a value to declare: %+v", l)
|
||||
}
|
||||
}
|
||||
|
||||
var asked []map[string]any
|
||||
asker = func(key string, body any) (json.RawMessage, error) {
|
||||
if key != "seat:mesh-controller.settings" {
|
||||
t.Fatalf("asked %s", key)
|
||||
}
|
||||
args := body.(map[string]any)
|
||||
asked = append(asked, args)
|
||||
if args["propose"] == nil {
|
||||
return json.RawMessage(`{"ok": true, "output": "{\n \"tags\": {\"$label1\": {\"name\": \"Important\"}}\n}\n approved by the operator\nbetterbird on the whole mesh, every value"}`), nil
|
||||
}
|
||||
return json.RawMessage(`{"ok": true, "output": "proposed p-17 for betterbird on the whole mesh: accounts (other@mail.example.org); fingerprint ab12"}`), nil
|
||||
}
|
||||
t.Cleanup(func() { asker = nil })
|
||||
a, err = f.ProfileAccounts(true)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(asked) != 2 || asked[1]["module"] != "betterbird" || asked[1]["propose"] != "true" || asked[1]["node"] != nil ||
|
||||
asked[1]["replace"] != nil {
|
||||
t.Fatalf("the layer read, then proposed for the whole mesh, never set, never removing: %v", asked)
|
||||
}
|
||||
var layer map[string]json.RawMessage
|
||||
if err := json.Unmarshal([]byte(asked[1]["values"].(string)), &layer); err != nil || string(layer["tags"]) == "" {
|
||||
t.Fatalf("the proposal carries the layer as it stands: %v %s", err, asked[1]["values"])
|
||||
}
|
||||
value, err := parseAccounts([]byte(asked[1]["values"].(string)))
|
||||
if err != nil {
|
||||
t.Fatalf("the proposed value is one the module takes: %v", err)
|
||||
}
|
||||
other, ok := value["mail"]
|
||||
if len(value) != 1 || !ok || other.Address != "other@mail.example.org" || other.Incoming.SignIn != "oauth" ||
|
||||
other.Incoming.Port != 993 || other.Outgoing.Security != "tls" || other.Outgoing.Port != 465 {
|
||||
t.Fatalf("the value holds the person's account and not Owl's: %+v", value)
|
||||
}
|
||||
if a.Derived["mail"].Address != "other@mail.example.org" {
|
||||
t.Fatalf("the answer shows the value it proposed (ADR 0329 §10): %+v", a.Derived)
|
||||
}
|
||||
if strings.Contains(a.Proposed, "other@") || !strings.Contains(a.Proposed, "p-17") {
|
||||
t.Fatalf("the controller's words come back with every address masked: %s", a.Proposed)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnAdoptedAccountNeverWritesIntoAnOutgoingServerAnotherIdentityUses(t *testing.T) {
|
||||
f := newFake(t)
|
||||
f.profileWithAccounts()
|
||||
f.noPackage()
|
||||
links(t, nil)
|
||||
f.declare("// none")
|
||||
os.RemoveAll(filepath.Join(f.Root, "proc", "4000"))
|
||||
// A second account of the person's, not declared, sends through smtp1 too.
|
||||
prefs := strings.Replace(f.read(profileDir+"/prefs.js"), `"account1,account3,account2"`, `"account1,account3,account2,account5"`, 1) +
|
||||
`user_pref("mail.account.account5.server", "server5");` + "\n" +
|
||||
`user_pref("mail.account.account5.identities", "id5");` + "\n" +
|
||||
`user_pref("mail.identity.id5.useremail", "fifth@mail.example.org");` + "\n" +
|
||||
`user_pref("mail.identity.id5.smtpServer", "smtp1");` + "\n" +
|
||||
`user_pref("mail.server.server5.type", "imap");` + "\n"
|
||||
f.write(profileDir+"/prefs.js", prefs)
|
||||
f.declareAccounts(adoptable)
|
||||
err := f.Prepare()
|
||||
if err == nil || !strings.Contains(err.Error(), "also used by identity id5") || strings.Contains(err.Error(), "example.org") {
|
||||
t.Fatalf("refused with the remedy, by keys: %v", err)
|
||||
}
|
||||
if user, _ := os.ReadFile(filepath.Join(f.Root, profileDir, "user.js")); strings.Contains(string(user), "smtpserver.smtp1") {
|
||||
t.Fatal("the shared outgoing server was written")
|
||||
}
|
||||
|
||||
// Recorded adoption, then the person moves the identity to a server of its own: the module follows it.
|
||||
f.write(profileDir+"/prefs.js", strings.Replace(prefs, `"mail.identity.id5.smtpServer", "smtp1"`, `"mail.identity.id5.smtpServer", "smtp5"`, 1))
|
||||
if err := f.Prepare(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
moved := strings.Replace(f.read(profileDir+"/prefs.js"), `"mail.identity.id1.smtpServer", "smtp1"`, `"mail.identity.id1.smtpServer", "smtp9"`, 1) +
|
||||
`user_pref("mail.smtpserver.smtp9.hostname", "smtp.example.org");` + "\n"
|
||||
f.write(profileDir+"/prefs.js", moved)
|
||||
u, err := f.userJS(true)
|
||||
if err != nil || u.Each[0].How != adopted || u.Each[0].Keys.SMTP != "smtp9" {
|
||||
t.Fatalf("the identity's current outgoing server is followed: %+v %v", u.Each, err)
|
||||
}
|
||||
user := f.read(profileDir + "/user.js")
|
||||
if strings.Contains(user, "smtpserver.smtp1.") || !strings.Contains(user, "smtpserver.smtp9.hostname") {
|
||||
t.Fatalf("never the old one:\n%s", user)
|
||||
}
|
||||
// And the move back onto a shared server is refused again.
|
||||
f.write(profileDir+"/prefs.js", strings.Replace(moved, `"mail.identity.id5.smtpServer", "smtp5"`, `"mail.identity.id5.smtpServer", "smtp9"`, 1))
|
||||
if err := f.Prepare(); err == nil || !strings.Contains(err.Error(), "also used by identity id5") {
|
||||
t.Fatalf("%v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheUserJSAnswerWithholdsTheAccountsValues(t *testing.T) {
|
||||
f := newFake(t)
|
||||
f.profileWithAccounts()
|
||||
f.noPackage()
|
||||
links(t, nil)
|
||||
f.declare(`user_pref("mail.compose.default_to", "x");`)
|
||||
f.declareAccounts(personal)
|
||||
u, err := f.userJS(false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, never := range []string{"example.org", `"me"`, `"Me"`} {
|
||||
if strings.Contains(u.Content, never) {
|
||||
t.Errorf("the answer carries %s:\n%s", never, u.Content)
|
||||
}
|
||||
}
|
||||
if !strings.Contains(u.Content, `user_pref("mail.server.server_mesh_personal.hostname", "<withheld>");`) {
|
||||
t.Fatalf("an account's preference is named and its value withheld:\n%s", u.Content)
|
||||
}
|
||||
// What is placed in the profile keeps the values: only the answer withholds them.
|
||||
if f.read(testHome+"/"+accountsFile) == "" {
|
||||
t.Fatal("no declaration")
|
||||
}
|
||||
}
|
||||
|
||||
func TestACredentialRefusalNamesNoAccount(t *testing.T) {
|
||||
_, err := parseAccounts([]byte(`{"accounts": {"personal": {"incoming": {"password": "hunter2"}}}}`))
|
||||
if err == nil || strings.Contains(err.Error(), "personal") || !strings.Contains(err.Error(), "accounts.<an account>.incoming.password") {
|
||||
t.Fatalf("%v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnAccountsOwnAliasesAndNonMailIdentitiesDoNotShareItsOutgoingServer(t *testing.T) {
|
||||
f := newFake(t)
|
||||
f.profileWithAccounts()
|
||||
f.noPackage()
|
||||
links(t, nil)
|
||||
f.declare("// none")
|
||||
os.RemoveAll(filepath.Join(f.Root, "proc", "4000"))
|
||||
// account1 has an alias identity on smtp1; smtp1 is the default server; Owl's account and local folders
|
||||
// have identities that name no outgoing server.
|
||||
prefs := strings.Replace(f.read(profileDir+"/prefs.js"), `"mail.account.account1.identities", "id1"`, `"mail.account.account1.identities", "id1,id1b"`, 1)
|
||||
prefs = strings.Replace(prefs, `"mail.account.account2.server", "server2");`, `"mail.account.account2.server", "server2");`+"\n"+
|
||||
`user_pref("mail.account.account2.identities", "id2");`, 1) +
|
||||
`user_pref("mail.identity.id1b.useremail", "alias@mail.example.org");` + "\n" +
|
||||
`user_pref("mail.identity.id1b.smtpServer", "smtp1");` + "\n" +
|
||||
`user_pref("mail.identity.id2.useremail", "local@mail.example.org");` + "\n" +
|
||||
`user_pref("mail.smtp.defaultserver", "smtp1");` + "\n"
|
||||
f.write(profileDir+"/prefs.js", prefs)
|
||||
f.declareAccounts(adoptable)
|
||||
if err := f.Prepare(); err != nil {
|
||||
t.Fatalf("an account's own alias, and identities that do not send by SMTP, are no sharing: %v", err)
|
||||
}
|
||||
// An identity of another mail account naming no server does use the default one.
|
||||
f.write(profileDir+"/prefs.js", strings.Replace(prefs, `"account1,account3,account2"`, `"account1,account3,account2,account6"`, 1)+
|
||||
`user_pref("mail.account.account6.server", "server6");`+"\n"+
|
||||
`user_pref("mail.account.account6.identities", "id6");`+"\n"+
|
||||
`user_pref("mail.identity.id6.useremail", "six@mail.example.org");`+"\n"+
|
||||
`user_pref("mail.server.server6.type", "pop3");`+"\n")
|
||||
os.Remove(filepath.Join(f.Root, testHome, recordFile))
|
||||
if err := f.Prepare(); err == nil || !strings.Contains(err.Error(), "also used by identity id6") {
|
||||
t.Fatalf("%v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Review 121: the account lists wait for Betterbird to close by its profile's lock too, not only by its process.
|
||||
func TestTheAccountListsWaitWhileTheProfileIsLocked(t *testing.T) {
|
||||
f := newFake(t)
|
||||
f.profileWithAccounts()
|
||||
f.noPackage()
|
||||
links(t, nil)
|
||||
f.declare("// none")
|
||||
f.declareAccounts(personal)
|
||||
stopBetterbird(t, f)
|
||||
f.write(profileDir+"/lock", "")
|
||||
before := f.read(profileDir + "/prefs.js")
|
||||
u, err := f.userJS(true)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !strings.HasPrefix(u.Lists, "not written") || !strings.Contains(u.Lists, "locked") || f.read(profileDir+"/prefs.js") != before {
|
||||
t.Fatalf("with the profile locked and no Betterbird running, prefs.js is unchanged: %q", u.Lists)
|
||||
}
|
||||
os.Remove(filepath.Join(f.Root, profileDir, "lock"))
|
||||
if u, err = f.userJS(true); err != nil || !strings.HasPrefix(u.Lists, "written") {
|
||||
t.Fatalf("once unlocked the list is written: %q %v", u.Lists, err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,331 @@
|
||||
package main
|
||||
|
||||
// Which account of the profile a declared account is (novox/hq ADR 0317): adopted, created or refused.
|
||||
//
|
||||
// At every apply, per declared account in its place in the setting (§1):
|
||||
//
|
||||
// 1. an account the module recorded on this node for this profile, adopted or created, is kept by its
|
||||
// recorded keys without matching again, while its account key is still in the profile's account list;
|
||||
// 2. otherwise an account under the module's own mesh_<name> keys is the module's creation;
|
||||
// 3. otherwise a node's layer may name the key to adopt (§8): adopted only when it is in the account list,
|
||||
// its default identity has the declared address and it is not Owl's;
|
||||
// 4. otherwise the profile's accounts whose default identity has the declared address: one is adopted, none
|
||||
// means the account is created under mesh_<name> keys (ADR 0316 §11).
|
||||
//
|
||||
// Refused, with the remedy, for that account alone (§7): two or more accounts with the address, the one that
|
||||
// has it being Owl's, a named key that does not qualify, an account of another kind than the declaration says
|
||||
// (the module never converts one: the review of ADR 0317), and an identity that names no outgoing server. The
|
||||
// other declared accounts go ahead.
|
||||
//
|
||||
// Adopting takes the account's keys as the module's (§2): its account key, its default identity, its incoming
|
||||
// server and the outgoing server that identity names. The account lists are left as they are, because the keys
|
||||
// are in them already; the declared fields are written under those keys (§4, §5). What was adopted or created
|
||||
// is recorded in the module's directory in the home, by the tools and never by a setting (§3); leaving the
|
||||
// setting releases an adopted account (its lines and its record go, the account stays the person's, §6).
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// recordFile is where the module records, per profile, what it adopted or created (ADR 0317 §3).
|
||||
const recordFile = ".config/betterbird/accounts-record.json"
|
||||
|
||||
// How a declared account resolved.
|
||||
const (
|
||||
adopted = "adopted"
|
||||
created = "created"
|
||||
refused = "refused"
|
||||
)
|
||||
|
||||
// keySet are an account's four keys.
|
||||
type keySet struct {
|
||||
Account string `json:"account"`
|
||||
Identity string `json:"identity"`
|
||||
Server string `json:"server"`
|
||||
SMTP string `json:"smtp"`
|
||||
}
|
||||
|
||||
// resolvedAccount is one declared account as it resolved, said by its place and keys, never its name, address
|
||||
// or host (ADR 0316 §16, ADR 0317 §3).
|
||||
type resolvedAccount struct {
|
||||
Place int `json:"place"`
|
||||
How string `json:"how"`
|
||||
Keys keySet `json:"keys,omitempty"`
|
||||
// Why is a refusal's reason and its remedy.
|
||||
Why string `json:"why,omitempty"`
|
||||
// Changed are the preferences an apply changed for it, by name only (ADR 0317 §4).
|
||||
Changed []string `json:"changed,omitempty"`
|
||||
name string
|
||||
}
|
||||
|
||||
// record is what the module adopted or created, per profile (its directory as the tools say it), per
|
||||
// declared account's name.
|
||||
type record map[string]map[string]recorded
|
||||
|
||||
type recorded struct {
|
||||
How string `json:"how"`
|
||||
Keys keySet `json:"keys"`
|
||||
}
|
||||
|
||||
func (m *Machine) readRecord(profileDir string) map[string]recorded {
|
||||
raw, err := readBounded(m.home(recordFile))
|
||||
if err != nil {
|
||||
return map[string]recorded{}
|
||||
}
|
||||
var r record
|
||||
if json.Unmarshal(raw, &r) != nil || r[profileDir] == nil {
|
||||
return map[string]recorded{}
|
||||
}
|
||||
return r[profileDir]
|
||||
}
|
||||
|
||||
// writeRecord keeps what this apply resolved for the profile: a refused account is not recorded, and an
|
||||
// account no longer declared is dropped (§6).
|
||||
func (m *Machine) writeRecord(profileDir string, resolved []resolvedAccount) error {
|
||||
r := record{}
|
||||
if raw, err := readBounded(m.home(recordFile)); err == nil {
|
||||
_ = json.Unmarshal(raw, &r)
|
||||
}
|
||||
mine := map[string]recorded{}
|
||||
for _, a := range resolved {
|
||||
if a.How != refused {
|
||||
mine[a.name] = recorded{How: a.How, Keys: a.Keys}
|
||||
}
|
||||
}
|
||||
if len(mine) == 0 {
|
||||
delete(r, profileDir)
|
||||
} else {
|
||||
r[profileDir] = mine
|
||||
}
|
||||
body, err := json.MarshalIndent(r, "", " ")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
path := m.home(recordFile)
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
|
||||
return err
|
||||
}
|
||||
tmp := path + ".mesh-new"
|
||||
if err := os.WriteFile(tmp, append(body, '\n'), 0o600); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := os.Rename(tmp, path); err != nil {
|
||||
os.Remove(tmp)
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// resolveAccounts resolves every declared account, in its place in the setting (the file's order, by name).
|
||||
func resolveAccounts(accounts map[string]Account, prefs map[string]string, rec map[string]recorded) []resolvedAccount {
|
||||
have := profileAccounts(prefs)
|
||||
inList := map[string]profileAccount{}
|
||||
for _, a := range have {
|
||||
inList[a.Key] = a
|
||||
}
|
||||
names := sortedNames(accounts)
|
||||
taken := map[string]bool{}
|
||||
var out []resolvedAccount
|
||||
for i, name := range names {
|
||||
r := resolveOne(i+1, len(names), name, accounts[name], prefs, have, inList, rec, taken)
|
||||
r.name = name
|
||||
if r.How != refused {
|
||||
taken[r.Keys.Account] = true
|
||||
}
|
||||
out = append(out, r)
|
||||
}
|
||||
return refuseSharedOutgoing(out, prefs, have)
|
||||
}
|
||||
|
||||
// refuseSharedOutgoing refuses an adopted account whose outgoing server another identity uses that is neither
|
||||
// adopted nor the module's (review 34 of mesh-catalog #232): Betterbird lets identities share an outgoing
|
||||
// server, and the declared values written into it would change that other account's sending at every start.
|
||||
// ADR 0317 §2 adopts the outgoing server the identity names and leaves the lists as they are, so the module
|
||||
// does not give the identity a server of its own; it refuses, with the remedy.
|
||||
func refuseSharedOutgoing(resolved []resolvedAccount, prefs map[string]string, have []profileAccount) []resolvedAccount {
|
||||
// Every identity of an adopted or created account is its own, its aliases too (review 36).
|
||||
accountOf := map[string]profileAccount{}
|
||||
kindOf := map[string]string{}
|
||||
for _, a := range have {
|
||||
accountOf[a.Key] = a
|
||||
for _, id := range a.Identities {
|
||||
kindOf[id] = a.Kind
|
||||
}
|
||||
}
|
||||
ours := map[string]bool{}
|
||||
for _, r := range resolved {
|
||||
if r.How == refused {
|
||||
continue
|
||||
}
|
||||
ours[r.Keys.Identity] = true
|
||||
for _, id := range accountOf[r.Keys.Account].Identities {
|
||||
ours[id] = true
|
||||
}
|
||||
}
|
||||
defaultServer := unquote(prefs["mail.smtp.defaultserver"])
|
||||
users := map[string][]string{}
|
||||
seen := map[string]bool{}
|
||||
use := func(id string) {
|
||||
// An identity the module created for a declared account sends as that account does (ADR 0329 §7).
|
||||
if seen[id] || ours[id] || ownIdentity.MatchString(id) {
|
||||
return
|
||||
}
|
||||
seen[id] = true
|
||||
server := unquote(prefs["mail.identity."+id+".smtpServer"])
|
||||
// Only a mail account's identity sends through the default server when it names none: Owl's, local
|
||||
// folders and feeds do not send by SMTP (review 36).
|
||||
if server == "" && (kindOf[id] == "imap" || kindOf[id] == "pop3") {
|
||||
server = defaultServer
|
||||
}
|
||||
if server != "" {
|
||||
users[server] = append(users[server], id)
|
||||
}
|
||||
}
|
||||
for _, a := range have {
|
||||
for _, id := range a.Identities {
|
||||
use(id)
|
||||
}
|
||||
}
|
||||
for k := range prefs {
|
||||
if strings.HasPrefix(k, "mail.identity.") && strings.HasSuffix(k, ".smtpServer") {
|
||||
use(strings.TrimSuffix(strings.TrimPrefix(k, "mail.identity."), ".smtpServer"))
|
||||
}
|
||||
}
|
||||
for i, r := range resolved {
|
||||
if r.How != adopted {
|
||||
continue
|
||||
}
|
||||
if others := users[r.Keys.SMTP]; len(others) > 0 {
|
||||
sort.Strings(others)
|
||||
resolved[i] = resolvedAccount{Place: r.Place, How: refused, name: r.name, Why: fmt.Sprintf(
|
||||
"declared account %d of %d: the outgoing server %s of the profile's account %s is also used by "+
|
||||
"identity %s, whose sending the declared values would change: give the account an outgoing "+
|
||||
"server of its own in Betterbird's account settings, then start Betterbird again",
|
||||
r.Place, len(resolved), r.Keys.SMTP, r.Keys.Account, strings.Join(others, ", "))}
|
||||
}
|
||||
}
|
||||
return resolved
|
||||
}
|
||||
|
||||
func resolveOne(place, of int, name string, a Account, prefs map[string]string, have []profileAccount,
|
||||
inList map[string]profileAccount, rec map[string]recorded, taken map[string]bool) resolvedAccount {
|
||||
r := resolvedAccount{Place: place}
|
||||
refuse := func(format string, args ...any) resolvedAccount {
|
||||
r.How, r.Keys = refused, keySet{}
|
||||
r.Why = fmt.Sprintf("declared account %d of %d: ", place, of) + fmt.Sprintf(format, args...)
|
||||
return r
|
||||
}
|
||||
// 1. Recorded, while its account is still in the list.
|
||||
if was, ok := rec[name]; ok {
|
||||
if _, there := inList[was.Keys.Account]; there || was.How == created {
|
||||
r.How, r.Keys = was.How, was.Keys
|
||||
if was.How == adopted {
|
||||
// The identity's outgoing server as it is now: a person may have moved it since.
|
||||
smtpKey := unquote(prefs["mail.identity."+was.Keys.Identity+".smtpServer"])
|
||||
if smtpKey == "" {
|
||||
return refuse("the profile's account %s has an identity that names no outgoing server: choose one "+
|
||||
"for it in Betterbird's account settings, then start Betterbird again", was.Keys.Account)
|
||||
}
|
||||
r.Keys.SMTP = smtpKey
|
||||
}
|
||||
return r
|
||||
}
|
||||
}
|
||||
// 2. Under the module's own keys: its creation.
|
||||
account, identity, server, smtp := keysOf(name)
|
||||
own := keySet{account, identity, server, smtp}
|
||||
if _, there := inList[account]; there {
|
||||
r.How, r.Keys = created, own
|
||||
return r
|
||||
}
|
||||
adopt := func(pa profileAccount) resolvedAccount {
|
||||
if pa.Kind != a.Incoming.Kind {
|
||||
return refuse("the profile's account %s is of kind %s and the declaration says %s: the module never "+
|
||||
"converts an account. Declare the kind the account has, or remove the account in Betterbird",
|
||||
pa.Key, pa.Kind, a.Incoming.Kind)
|
||||
}
|
||||
smtpKey := unquote(prefs["mail.identity."+pa.Identities[0]+".smtpServer"])
|
||||
if smtpKey == "" {
|
||||
return refuse("the profile's account %s has an identity that names no outgoing server: choose one for it "+
|
||||
"in Betterbird's account settings, then start Betterbird again", pa.Key)
|
||||
}
|
||||
r.How, r.Keys = adopted, keySet{pa.Key, pa.Identities[0], pa.Server, smtpKey}
|
||||
return r
|
||||
}
|
||||
matchesDefault := func(pa profileAccount) bool {
|
||||
return len(pa.Addresses) > 0 && strings.EqualFold(pa.Addresses[0], a.Address)
|
||||
}
|
||||
// 3. The key a node's layer names.
|
||||
if a.Key != "" {
|
||||
pa, there := inList[a.Key]
|
||||
switch {
|
||||
case !there:
|
||||
return refuse("the key %s this node's layer names is not in the profile's account list", a.Key)
|
||||
case pa.Owl:
|
||||
return refuse("the key %s this node's layer names is Owl's account, which is never declared "+
|
||||
"(novox/hq ADR 0316 §13)", a.Key)
|
||||
case !matchesDefault(pa):
|
||||
return refuse("the key %s this node's layer names is an account whose default identity has another "+
|
||||
"address", a.Key)
|
||||
case taken[pa.Key]:
|
||||
return refuse("the key %s is already another declared account's", a.Key)
|
||||
}
|
||||
return adopt(pa)
|
||||
}
|
||||
// 4. By the default identity's address.
|
||||
var matches []profileAccount
|
||||
for _, pa := range have {
|
||||
if !pa.Ours && !taken[pa.Key] && matchesDefault(pa) {
|
||||
matches = append(matches, pa)
|
||||
}
|
||||
}
|
||||
switch {
|
||||
case len(matches) == 0:
|
||||
r.How, r.Keys = created, own
|
||||
return r
|
||||
case len(matches) > 1:
|
||||
var keys []string
|
||||
for _, pa := range matches {
|
||||
keys = append(keys, pa.Key)
|
||||
}
|
||||
return refuse("%d accounts in the profile have its address (%s): remove the duplicate in Betterbird, or "+
|
||||
"give the one to adopt as key in this node's layer of the setting accounts (novox/hq ADR 0317 §7, §8)",
|
||||
len(matches), strings.Join(keys, ", "))
|
||||
case matches[0].Owl:
|
||||
return refuse("the profile's account with its address, %s, is Owl's, which is never declared (novox/hq "+
|
||||
"ADR 0316 §13): take it out of the setting accounts", matches[0].Key)
|
||||
}
|
||||
return adopt(matches[0])
|
||||
}
|
||||
|
||||
// changedPrefs are the preferences of a resolved account whose declared value differs from the profile's
|
||||
// saved one (prefs.js), by name only (ADR 0317 §4). A preference prefs.js does not hold is changed unless the
|
||||
// declared value is what Betterbird reads in its place.
|
||||
func changedPrefs(r resolvedAccount, a Account, saved map[string]string) []string {
|
||||
defaults := map[string]string{}
|
||||
if r.How == adopted {
|
||||
// What Betterbird reads when prefs.js holds no value: the same defaults betterbird_accounts reads with.
|
||||
pa := profileAccount{Key: r.Keys.Account, Server: r.Keys.Server, Kind: a.Incoming.Kind,
|
||||
Identities: []string{r.Keys.Identity}}
|
||||
view := readAccount(saved, pa)
|
||||
for _, p := range declaredPrefs(r.Keys, view, true) {
|
||||
defaults[p[0]] = p[1]
|
||||
}
|
||||
}
|
||||
var out []string
|
||||
for _, p := range declaredPrefs(r.Keys, a, r.How == adopted) {
|
||||
have, ok := saved[p[0]]
|
||||
if !ok {
|
||||
have, ok = defaults[p[0]]
|
||||
}
|
||||
if !ok || have != p[1] {
|
||||
out = append(out, p[0])
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -9,20 +9,24 @@ package main
|
||||
// preferences asked for are answered and never one that can hold a credential or names an account
|
||||
// (passwordSecret, accountPrefix);
|
||||
// - the profile's extensions.json, for the add-ons' ids and whether each is on;
|
||||
// - the module's declared user.js (~/.config/betterbird/user.js), which the node-engine writes from
|
||||
// the module's setting `prefs`.
|
||||
// - the module's settings as the node-engine writes them: prefs (~/.config/betterbird/prefs.json, prefs.go),
|
||||
// accounts (~/.config/betterbird/accounts.json, accounts.go), and identities, calendars, address-books,
|
||||
// filters and tags (configuration.json and tags.json, configuration.go);
|
||||
// - of logins.json, only the server name each kept sign-in is for, which Betterbird keeps in the clear,
|
||||
// to count the declared accounts that have not signed in yet (novox/hq ADR 0316 §16).
|
||||
//
|
||||
// Never read: the mail store, the address books, logins.json, key4.db, cert9.db, cookies, the
|
||||
// calendar's caches. Nothing of the operator's mail, calendar or accounts is answered.
|
||||
// Never read: the mail store, the address books' contacts, a user name or password in logins.json, key4.db,
|
||||
// cert9.db, cookies, the calendar's events and caches. The operator's own configuration is answered with its
|
||||
// values (novox/hq ADR 0329 §1), never a credential (§2).
|
||||
//
|
||||
// The one file a tool writes in the profile is user.js, and only the module's own: the declared file,
|
||||
// copied whole (betterbird_user_js with apply, betterbird_restart). A user.js that is not the module's
|
||||
// is never overwritten.
|
||||
// The files a tool writes in the profile are user.js, and only the module's own (betterbird_user_js with apply,
|
||||
// betterbird_restart, and the start from the launcher); and, only while Betterbird is not running, the module's
|
||||
// own keys in the lists of prefs.js, the removals of what it created, and each declared account's
|
||||
// msgFilterRules.dat (accounts.go, owned.go). A user.js that is not the module's is never overwritten.
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
@@ -35,16 +39,18 @@ import (
|
||||
)
|
||||
|
||||
const (
|
||||
mainComm = "betterbird-bin"
|
||||
startBin = "/usr/bin/betterbird"
|
||||
mainComm = "betterbird-bin"
|
||||
// startBin is Betterbird itself, where the mesh's release and the community package both put it. The
|
||||
// module's start (/usr/local/bin/betterbird) runs the preparation first; a tool that starts Betterbird
|
||||
// prepares itself and starts this.
|
||||
startBin = installDir + "/betterbird"
|
||||
// packageFor is the community package the mesh's release replaces (novox/hq ADR 0316 §4).
|
||||
packageFor = "betterbird-bin"
|
||||
appIni = "/opt/betterbird/application.ini"
|
||||
restartAs = "mesh-betterbird"
|
||||
testUnit = "mesh-betterbird-reminder-test"
|
||||
// profilesDir is where Betterbird keeps its profiles: Thunderbird's place, which Betterbird shares.
|
||||
profilesDir = ".thunderbird"
|
||||
// declared is the module's user.js, written by the node-engine from the setting prefs.
|
||||
declared = ".config/betterbird/user.js"
|
||||
// mark is the first line of the module's user.js: how a user.js is known to be the module's.
|
||||
mark = "// Betterbird's user.js (module betterbird"
|
||||
)
|
||||
@@ -255,13 +261,22 @@ func readPrefs(raw []byte) map[string]string {
|
||||
var (
|
||||
passwordSecret = regexp.MustCompile(`(?i)(password|passwd|secret|token|oauth|auth|cookie|credential|signon|apikey|api_key|privatekey|private_key|\.key$)`)
|
||||
accountPrefix = []string{"mail.server.", "mail.identity.", "mail.smtpserver.", "mail.smtp.", "mail.account.",
|
||||
"mail.accountmanager.", "ldap_2.", "extensions.owl.", "calendar.registry.", "mail.root.", "mailnews.",
|
||||
"mail.accountmanager.", "ldap_2.", "extensions.owl.", "calendar.registry.", "mail.root.",
|
||||
"services.sync.", "mail.displayname", "mail.default_", "mail.last_msg", "messenger.account.", "mail.cloud_files."}
|
||||
// allowedExtensions are the only extensions.* preferences answered: the add-ons' own preferences can
|
||||
// hold anything, and a theme's id is what a window's look depends on.
|
||||
allowedExtensions = map[string]bool{"extensions.activeThemeID": true}
|
||||
// mailnewsAccount is a mailnews.* name that holds a server, an account, an identity or the operator's own
|
||||
// tags: only those are withheld of mailnews.*, so a setting such as mailnews.start_page.enabled or
|
||||
// mailnews.default_view_flags is answered (novox/hq issue 345).
|
||||
mailnewsAccount = regexp.MustCompile(`(?i)^mailnews\..*(server|account|ident|smtp|ldap|imap|pop3|nntp|tags\.|email|addr)`)
|
||||
)
|
||||
|
||||
// accountOwned are the preferences of an account and of the account lists: the setting accounts' alone, so
|
||||
// the setting prefs refuses them and an account has one owner (novox/hq ADR 0316 §11).
|
||||
var accountOwned = []string{"mail.account.", "mail.identity.", "mail.server.", "mail.smtpserver.",
|
||||
"mail.accountmanager.", "mail.smtpservers", "mail.smtp.defaultserver"}
|
||||
|
||||
// withheld says why a preference's value is never answered, or "" when it may be.
|
||||
func withheld(name string) string {
|
||||
if strings.HasPrefix(name, "extensions.") && !allowedExtensions[name] {
|
||||
@@ -275,16 +290,24 @@ func withheld(name string) string {
|
||||
return "it names an account, a server, a calendar or a person"
|
||||
}
|
||||
}
|
||||
if mailnewsAccount.MatchString(name) {
|
||||
return "it names an account, a server, a calendar or a person"
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// Pref is one preference as the profile holds it.
|
||||
type Pref struct {
|
||||
Name string `json:"name"`
|
||||
// Value is what Betterbird uses: user.js's at the start, else prefs.js's. Absent: the default.
|
||||
Value string `json:"value,omitempty"`
|
||||
From string `json:"from,omitempty"`
|
||||
Withheld string `json:"withheld,omitempty"`
|
||||
// Value is what Betterbird uses: the policy's lock, else user.js's at the start, else prefs.js's, else the
|
||||
// default, when a defaults file gives it.
|
||||
Value string `json:"value,omitempty"`
|
||||
From string `json:"from,omitempty"`
|
||||
// Default is the default Betterbird starts with, and the file that gives it: Betterbird's own (omni.ja), or
|
||||
// the module's (novox/hq issue 345). Absent for a preference compiled into Betterbird.
|
||||
Default string `json:"default,omitempty"`
|
||||
DefaultFrom string `json:"default_from,omitempty"`
|
||||
Withheld string `json:"withheld,omitempty"`
|
||||
}
|
||||
|
||||
// mostPrefs bounds how many preferences one answer carries.
|
||||
@@ -306,7 +329,9 @@ func (m *Machine) Prefs(names []string) (map[string]any, error) {
|
||||
}
|
||||
prefsJS, _ := readBounded(filepath.Join(p.path, "prefs.js"))
|
||||
userJS, _ := readBounded(filepath.Join(p.path, "user.js"))
|
||||
fromPrefs, fromUser := readPrefs(prefsJS), readPrefs(userJS)
|
||||
fromPrefs, fromUser := readPrefs(prefsJS), statementsOf(userJS)
|
||||
own, ownErr := m.defaults(true)
|
||||
locked := m.locks()
|
||||
all := map[string]bool{}
|
||||
for n := range fromPrefs {
|
||||
all[n] = true
|
||||
@@ -341,25 +366,40 @@ func (m *Machine) Prefs(names []string) (map[string]any, error) {
|
||||
break
|
||||
}
|
||||
pr := Pref{Name: n}
|
||||
d, hasDefault := own[n]
|
||||
v, from := fromPrefs[n], "prefs.js"
|
||||
if u, ok := fromUser[n]; ok {
|
||||
if l, ok := locked[n]; ok {
|
||||
v, from = l, "locked by the module's policy"
|
||||
} else if u, ok := fromUser[n]; ok {
|
||||
v, from = u, "user.js (wins at the start)"
|
||||
} else if _, ok := fromPrefs[n]; !ok {
|
||||
from = "not set: Betterbird's default"
|
||||
v = ""
|
||||
switch {
|
||||
case hasDefault:
|
||||
v, from = d.Value, "not set: "+d.From
|
||||
default:
|
||||
v, from = "", "not set: Betterbird's built-in default (in no defaults file)"
|
||||
}
|
||||
}
|
||||
pr.From = from
|
||||
if hasDefault {
|
||||
pr.DefaultFrom = d.From
|
||||
}
|
||||
if why := withheld(n); why != "" {
|
||||
pr.Withheld = why
|
||||
} else if allowedExtensions[n] {
|
||||
pr.Value = v
|
||||
pr.Default = d.Value
|
||||
} else {
|
||||
pr.Value = m.maskValue(v)
|
||||
pr.Default = m.maskValue(d.Value)
|
||||
}
|
||||
out = append(out, pr)
|
||||
}
|
||||
}
|
||||
answer := map[string]any{"profile": p, "prefs": out}
|
||||
if ownErr != nil {
|
||||
answer["defaults_problem"] = ownErr.Error()
|
||||
}
|
||||
if cut {
|
||||
answer["cut"] = fmt.Sprintf("the first %d; ask for a narrower prefix", mostPrefs)
|
||||
}
|
||||
@@ -392,6 +432,18 @@ func validUserJS(text string) error {
|
||||
if passwordSecret.MatchString(name) {
|
||||
return fmt.Errorf("line %d sets %s, a preference that can hold a credential: not through the mesh", i+1, name)
|
||||
}
|
||||
for _, p := range accountOwned {
|
||||
if strings.HasPrefix(name, p) {
|
||||
return fmt.Errorf("line %d sets %s, an account's preference: an account is declared in the setting "+
|
||||
"accounts, never in prefs (novox/hq ADR 0316 §11)", i+1, name)
|
||||
}
|
||||
}
|
||||
for _, p := range configurationOwned {
|
||||
if strings.HasPrefix(name, p) {
|
||||
return fmt.Errorf("line %d sets %s, which the setting calendars, address-books or tags owns, never "+
|
||||
"prefs (novox/hq ADR 0329 §6)", i+1, name)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
@@ -433,25 +485,78 @@ type UserJS struct {
|
||||
Effective string `json:"effective,omitempty"`
|
||||
Content string `json:"content,omitempty"`
|
||||
Applied bool `json:"applied,omitempty"`
|
||||
// Accounts are how many accounts the setting accounts declares, and what is wrong with them.
|
||||
Accounts int `json:"declared_accounts"`
|
||||
AccountsProblem string `json:"accounts_problem,omitempty"`
|
||||
// Each is every declared account by its place: adopted, created or refused (with why), its keys, and the
|
||||
// preferences an apply changed for it, by name (novox/hq ADR 0317 §3, §4).
|
||||
Each []resolvedAccount `json:"accounts,omitempty"`
|
||||
// Lists says whether the keys of the accounts the module created were written into the account lists.
|
||||
Lists string `json:"account_lists,omitempty"`
|
||||
// Configuration is every declared identity, calendar, address book and tag by its name: adopted, created or
|
||||
// refused (with why), and its key (novox/hq ADR 0329 §7).
|
||||
Configuration []resolvedEntry `json:"configuration,omitempty"`
|
||||
ConfigurationProblem string `json:"configuration_problem,omitempty"`
|
||||
// Filters are the declared accounts' filter files.
|
||||
Filters []filterPlan `json:"filters,omitempty"`
|
||||
// Closed says what was written of what Betterbird rewrites itself, or why it waits (§9).
|
||||
Closed string `json:"written_while_closed,omitempty"`
|
||||
// Waiting counts, per kind, the changes that wait for Betterbird to close.
|
||||
Waiting map[string]int `json:"waiting_for_betterbird_to_close,omitempty"`
|
||||
}
|
||||
|
||||
// userJS compares the declared user.js with the profile's; apply places the declared one when the
|
||||
// profile has none or the module's.
|
||||
// refusedAccounts are the reasons of the declared accounts that were refused.
|
||||
func (u UserJS) refusedAccounts() []string {
|
||||
var out []string
|
||||
for _, r := range u.Each {
|
||||
if r.How == refused {
|
||||
out = append(out, r.Why)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// userJS compares the user.js the module places — the declared file, then the declared accounts' fields under
|
||||
// the keys each was adopted or created under (adoption.go) — with the profile's; apply places it when the
|
||||
// profile has none or the module's, records what was adopted or created, and writes the keys of the accounts
|
||||
// it created into the profile's account lists when Betterbird is not running. A refused account gets nothing;
|
||||
// the others go ahead (novox/hq ADR 0317 §7).
|
||||
func (m *Machine) userJS(apply bool) (UserJS, error) {
|
||||
a := UserJS{Declared: "~/" + declared}
|
||||
want, err := readBounded(m.home(declared))
|
||||
a.Present = err == nil
|
||||
a := UserJS{Declared: "~/" + prefsFile}
|
||||
declaredText, present, derr := m.declaredText()
|
||||
a.Present = present
|
||||
if derr != nil {
|
||||
a.Present, a.Valid = true, derr.Error()
|
||||
}
|
||||
p, perr := m.profile()
|
||||
if perr != nil {
|
||||
return a, perr
|
||||
}
|
||||
target := filepath.Join(p.path, "user.js")
|
||||
a.Profile = m.tilde(strings.TrimPrefix(target, m.Root))
|
||||
if a.Present {
|
||||
if verr := validUserJS(string(want)); verr != nil {
|
||||
accounts, _, aerr := m.readAccounts()
|
||||
merged := mergedPrefs(p.path)
|
||||
if aerr != nil {
|
||||
a.AccountsProblem = aerr.Error()
|
||||
accounts = nil
|
||||
} else {
|
||||
a.Accounts = len(accounts)
|
||||
a.Each = resolveAccounts(accounts, merged, m.readRecord(p.Dir))
|
||||
}
|
||||
conf, cerr := m.readConfiguration()
|
||||
if cerr != nil {
|
||||
a.ConfigurationProblem = cerr.Error()
|
||||
}
|
||||
plan := m.planConfiguration(p, conf, accounts, a.Each, merged)
|
||||
a.Configuration, a.Filters = plan.Entries, plan.filters
|
||||
a.Waiting = m.waiting(p.path, plan, a.Each)
|
||||
var want []byte
|
||||
if a.Present && a.Valid == "" {
|
||||
if verr := validUserJS(declaredText); verr != nil {
|
||||
a.Valid = verr.Error()
|
||||
}
|
||||
a.Content = string(want)
|
||||
want = []byte(composeUserJS(declaredText, accounts, a.Each) + plan.configBlock())
|
||||
a.Content = m.answeredUserJS(string(want))
|
||||
for _, line := range strings.Split(string(want), "\n") {
|
||||
for _, st := range splitStatements(strings.TrimSpace(line)) {
|
||||
if statement.MatchString(st) {
|
||||
@@ -474,9 +579,22 @@ func (m *Machine) userJS(apply bool) (UserJS, error) {
|
||||
if apply {
|
||||
switch {
|
||||
case !a.Present:
|
||||
return a, fmt.Errorf("the module's user.js is not on this machine (~/%s): the module is not pushed here yet", declared)
|
||||
return a, fmt.Errorf("the setting prefs is not on this machine (~/%s): the module is not pushed here yet", prefsFile)
|
||||
case a.Valid != "":
|
||||
return a, fmt.Errorf("the module's user.js is refused: %s; change the setting prefs", a.Valid)
|
||||
case a.AccountsProblem != "":
|
||||
return a, fmt.Errorf("nothing was placed: %s", a.AccountsProblem)
|
||||
case a.ConfigurationProblem != "":
|
||||
return a, fmt.Errorf("nothing was placed: %s", a.ConfigurationProblem)
|
||||
}
|
||||
saved, _ := readBounded(filepath.Join(p.path, "prefs.js"))
|
||||
savedPrefs := readPrefs(saved)
|
||||
for i, r := range a.Each {
|
||||
if r.How != refused {
|
||||
a.Each[i].Changed = changedPrefs(r, accounts[r.name], savedPrefs)
|
||||
}
|
||||
}
|
||||
switch {
|
||||
case herr == nil && !bytes.HasPrefix(have, []byte(mark)):
|
||||
return a, fmt.Errorf("%s is not the module's: move it aside (its preferences belong in the setting prefs), then apply again", a.Profile)
|
||||
}
|
||||
@@ -491,6 +609,24 @@ func (m *Machine) userJS(apply bool) (UserJS, error) {
|
||||
}
|
||||
}
|
||||
a.Applied, a.Placed = true, "the module's, as declared"
|
||||
if err := m.writeRecord(p.Dir, a.Each); err != nil {
|
||||
return a, fmt.Errorf("the user.js is placed, and what was adopted or created could not be recorded: %v", err)
|
||||
}
|
||||
if lists, err := m.writeAccountLists(p.path, a.Each); err != nil {
|
||||
a.Lists = "not written: " + err.Error()
|
||||
} else {
|
||||
a.Lists = lists
|
||||
}
|
||||
if closed, err := m.writeClosed(p, &plan); err != nil {
|
||||
a.Closed = "not written: " + err.Error()
|
||||
} else {
|
||||
a.Closed = closed
|
||||
}
|
||||
a.Filters = plan.filters
|
||||
if err := m.writeConfigRecord(p.Dir, plan.record); err != nil {
|
||||
return a, fmt.Errorf("the user.js is placed, and what was adopted or created could not be recorded: %v", err)
|
||||
}
|
||||
a.Waiting = m.waiting(p.path, plan, a.Each)
|
||||
}
|
||||
if ps := m.mainProcs(); len(ps) > 0 {
|
||||
if st, err := os.Stat(target); err == nil {
|
||||
@@ -504,27 +640,88 @@ func (m *Machine) userJS(apply bool) (UserJS, error) {
|
||||
return a, nil
|
||||
}
|
||||
|
||||
// answeredUserJS is the user.js as a tool answers it (novox/hq ADR 0316 §16, ADR 0317 §9): an account's
|
||||
// preferences with their values withheld, every other value masked as betterbird_prefs masks it.
|
||||
func (m *Machine) answeredUserJS(text string) string {
|
||||
lines := strings.Split(text, "\n")
|
||||
for i, line := range lines {
|
||||
for _, st := range splitStatements(strings.TrimSpace(line)) {
|
||||
mm := prefLine.FindStringSubmatch(st)
|
||||
if mm == nil {
|
||||
continue
|
||||
}
|
||||
value := m.maskValue(mm[2])
|
||||
if why := withheld(mm[1]); why != "" {
|
||||
value = `"<withheld>"`
|
||||
}
|
||||
lines[i] = fmt.Sprintf("user_pref(%s, %s);", quoted(mm[1]), value)
|
||||
}
|
||||
}
|
||||
return strings.Join(lines, "\n")
|
||||
}
|
||||
|
||||
// Prepare is what the module's start runs before Betterbird (`betterbird-tools prepare`): with no Betterbird
|
||||
// of this account running, the user.js placed and the declared accounts written into the account lists. With
|
||||
// one running it does nothing, since the start then only hands over to it. Its error is said by the start,
|
||||
// which starts Betterbird either way.
|
||||
func (m *Machine) Prepare() error {
|
||||
if len(m.mainProcs()) > 0 {
|
||||
return nil
|
||||
}
|
||||
u, err := m.userJS(true)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var problems []string
|
||||
if strings.HasPrefix(u.Lists, "not written") {
|
||||
problems = append(problems, "the account lists were "+u.Lists)
|
||||
}
|
||||
problems = append(problems, u.refusedAccounts()...)
|
||||
if strings.HasPrefix(u.Closed, "not written") {
|
||||
problems = append(problems, "what Betterbird rewrites itself was "+u.Closed)
|
||||
}
|
||||
// Said by kind, name and key only: the start's words reach the journal, which holds no address (ADR 0329 §1).
|
||||
for _, e := range u.Configuration {
|
||||
if e.How == refused {
|
||||
problems = append(problems, fmt.Sprintf("%s %s is refused: %s", e.Kind, e.Name, e.Why))
|
||||
}
|
||||
}
|
||||
for _, fp := range u.Filters {
|
||||
if fp.Refused != "" {
|
||||
problems = append(problems, fmt.Sprintf("the filters of %s are refused: %s", fp.Account, fp.Refused))
|
||||
}
|
||||
}
|
||||
if len(problems) > 0 {
|
||||
return fmt.Errorf("%s", strings.Join(problems, "; "))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Status is what betterbird_status answers.
|
||||
type Status struct {
|
||||
Running []Proc `json:"running"`
|
||||
Children int `json:"children"`
|
||||
Installed string `json:"installed,omitempty"`
|
||||
Running []Proc `json:"running"`
|
||||
Children int `json:"children"`
|
||||
// Installation is what is installed and from where, the policy and Owl (novox/hq ADR 0316 §16).
|
||||
Installation Installation `json:"installation"`
|
||||
// Installed is the community package's version, while it is still there.
|
||||
Installed string `json:"community_package,omitempty"`
|
||||
Foreign bool `json:"from_the_aur,omitempty"`
|
||||
Version string `json:"version,omitempty"`
|
||||
Build string `json:"build,omitempty"`
|
||||
Profile *Profile `json:"profile,omitempty"`
|
||||
Problem string `json:"profile_problem,omitempty"`
|
||||
Output []Stream `json:"output,omitempty"`
|
||||
UserJS *UserJS `json:"user_js,omitempty"`
|
||||
AddOns []AddOn `json:"add_ons,omitempty"`
|
||||
// Accounts are counts: the declared, those in the profile's list, the others, those without a sign-in.
|
||||
Accounts *AccountCount `json:"accounts,omitempty"`
|
||||
AddOns []AddOn `json:"add_ons,omitempty"`
|
||||
// Display are the preferences that decide how its windows and notifications look.
|
||||
Display []Pref `json:"display_prefs,omitempty"`
|
||||
}
|
||||
|
||||
// AddOn is one add-on the operator installed: its id, and whether it is on.
|
||||
type AddOn struct {
|
||||
ID string `json:"id"`
|
||||
Active bool `json:"active"`
|
||||
ID string `json:"id"`
|
||||
Version string `json:"version,omitempty"`
|
||||
Active bool `json:"active"`
|
||||
}
|
||||
|
||||
// displayPrefs are the preferences betterbird_status answers: the reminder dialog, the theme, the
|
||||
@@ -550,39 +747,24 @@ func (m *Machine) Status() (Status, error) {
|
||||
if v, err := m.installed(packageFor); err == nil && v != "" {
|
||||
s.Installed, s.Foreign = v, m.foreign(packageFor)
|
||||
}
|
||||
if raw, err := readBounded(m.path(appIni)); err == nil {
|
||||
_, sec := iniSections(raw)
|
||||
s.Version, s.Build = sec["App"]["Version"], sec["App"]["BuildID"]
|
||||
}
|
||||
if len(s.Running) > 0 {
|
||||
s.Output = m.streams(s.Running[0].PID)
|
||||
}
|
||||
p, err := m.profile()
|
||||
if err != nil {
|
||||
s.Installation = m.installation("")
|
||||
s.Problem = err.Error()
|
||||
return s, nil
|
||||
}
|
||||
s.Profile = &p
|
||||
s.Installation = m.installation(p.path)
|
||||
if u, err := m.userJS(false); err == nil {
|
||||
u.Content = ""
|
||||
s.UserJS = &u
|
||||
}
|
||||
if raw, err := readBounded(filepath.Join(p.path, "extensions.json")); err == nil {
|
||||
var ext struct {
|
||||
Addons []struct {
|
||||
ID string `json:"id"`
|
||||
Active bool `json:"active"`
|
||||
Location string `json:"location"`
|
||||
} `json:"addons"`
|
||||
}
|
||||
if json.Unmarshal(raw, &ext) == nil {
|
||||
for _, a := range ext.Addons {
|
||||
if a.Location == "app-profile" {
|
||||
s.AddOns = append(s.AddOns, AddOn{a.ID, a.Active})
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
c := m.countAccounts(p.path, p.Dir)
|
||||
s.Accounts = &c
|
||||
s.AddOns = addOns(p.path)
|
||||
if got, err := m.Prefs(displayPrefs); err == nil {
|
||||
s.Display = got["prefs"].([]Pref)
|
||||
}
|
||||
@@ -711,7 +893,8 @@ func (m *Machine) Restart(discard bool) (RestartAnswer, error) {
|
||||
if u, err := m.userJS(true); err != nil {
|
||||
a.UserJS = "not placed: " + err.Error()
|
||||
} else {
|
||||
a.UserJS = fmt.Sprintf("placed: %d preference(s) from ~/%s", u.Statements, declared)
|
||||
a.UserJS = fmt.Sprintf("placed: %d preference(s) from ~/%s, %d declared account(s); account lists %s",
|
||||
u.Statements, prefsFile, u.Accounts, u.Lists)
|
||||
}
|
||||
// As detach does, with shorter bounds: the worst case of the whole call (3 s for the window tree,
|
||||
// 8 s to quit, 3 + 5 s to start, 4 s to see it) stays under the runtime's 30 s.
|
||||
@@ -746,16 +929,107 @@ type CheckAnswer struct {
|
||||
}
|
||||
|
||||
// Check verifies what the module promises: Betterbird installed, one running in the session, its
|
||||
// profile found, the module's user.js declared, valid, placed and in effect, and its output read.
|
||||
// profile found, the module's user.js declared, valid, placed and in effect, its output read, and no ask at its
|
||||
// start to be the default mail client (novox/hq issue 345).
|
||||
func (m *Machine) Check() (CheckAnswer, error) {
|
||||
a := CheckAnswer{Findings: []Finding{}}
|
||||
add := func(what, do string) { a.Findings = append(a.Findings, Finding{what, do}) }
|
||||
v, err := m.installed(packageFor)
|
||||
if err != nil {
|
||||
if _, err := m.installed(packageFor); err != nil {
|
||||
return a, err
|
||||
}
|
||||
if v == "" {
|
||||
add("Betterbird ("+packageFor+") is not installed", "install it from the AUR: the node-engine installs packages from the official repositories only, so the module does not")
|
||||
p, perr := m.profile()
|
||||
profilePath := ""
|
||||
if perr == nil {
|
||||
profilePath = p.path
|
||||
}
|
||||
in := m.installation(profilePath)
|
||||
switch {
|
||||
case in.From == "not installed":
|
||||
add("Betterbird is not installed in "+installDir, "the module's push places the mesh's release; it waits for the operator's push (upgrade policy record)")
|
||||
case in.From != "the mesh":
|
||||
add("Betterbird is "+in.From, "quit Betterbird, then push the module to this node: the push removes the community package and places the mesh's release")
|
||||
}
|
||||
if in.Behind {
|
||||
add("Betterbird "+in.Version+" (build "+in.Build+") is behind the newest published build "+in.Newest,
|
||||
"move the module's pin in a reviewed change (novox/hq ADR 0316 §5)")
|
||||
}
|
||||
if in.From == "the mesh" && !strings.HasPrefix(in.Policy, "placed:") {
|
||||
add("the enterprise policy is "+in.Policy, "push the module to this node")
|
||||
}
|
||||
if in.From == "the mesh" && !exists(m.path(meshDefaults)) {
|
||||
add("the module's defaults are not placed ("+meshDefaults+")", "push the module to this node")
|
||||
}
|
||||
if c, _, err := m.config(); err == nil {
|
||||
var cleared []string
|
||||
for _, cp := range c.Prefs {
|
||||
switch {
|
||||
case cp.Drift == "":
|
||||
case cp.Remedy != "":
|
||||
add(cp.Name+" drifts from the module's: "+cp.Drift, cp.Remedy)
|
||||
default:
|
||||
cleared = append(cleared, cp.Name)
|
||||
}
|
||||
}
|
||||
if len(cleared) > 0 {
|
||||
add(fmt.Sprintf("%d managed preference(s) drift from the module's defaults: the profile saved its own value and no "+
|
||||
"node layer says so (%s)", len(cleared), strings.Join(cleared, ", ")),
|
||||
"betterbird_config shows each; to keep a value on this node, betterbird_config with keep proposes a node "+
|
||||
"layer; otherwise quit Betterbird and call betterbird_config_apply with apply")
|
||||
}
|
||||
if len(c.Refused) > 0 {
|
||||
add(fmt.Sprintf("the module's defaults list %d preference(s) the tools never manage (%s)", len(c.Refused), strings.Join(c.Refused, "; ")),
|
||||
"take them out of install/defaults-pref/mesh-defaults.js in a reviewed catalogue change: an account is declared in the setting accounts")
|
||||
}
|
||||
}
|
||||
if why := m.defaultClientAsk(profilePath); why != "" && in.From != "not installed" {
|
||||
add("Betterbird asks at its start to be the default mail client: "+why,
|
||||
"push the module to this node: its policy locks "+defaultClientPref+" off (novox/hq issue 345)")
|
||||
}
|
||||
switch {
|
||||
case in.From != "the mesh":
|
||||
case in.Owl == nil || in.Owl.Pinned == "":
|
||||
add("Owl's xpi is not placed ("+owlFile+")", "push the module to this node")
|
||||
case in.Owl.InProfile == "":
|
||||
add("Owl is not installed in the profile yet", "Betterbird installs it from the policy at its next start")
|
||||
case !in.Owl.AtThePin && !newerVersion(in.Owl.InProfile, in.Owl.Pinned):
|
||||
add("Owl "+in.Owl.InProfile+" runs in the profile, older than the mesh's pin "+in.Owl.Pinned,
|
||||
"Owl is installed by policy and cannot be removed in Betterbird: move the module's Owl pin in a reviewed change (novox/hq ADR 0316 §6)")
|
||||
case !in.Owl.Active:
|
||||
add("Owl is installed and not active", "turn it on in Betterbird's add-ons")
|
||||
}
|
||||
if perr == nil {
|
||||
c := m.countAccounts(p.path, p.Dir)
|
||||
for _, r := range c.Each {
|
||||
if r.How == refused {
|
||||
add("refused: "+r.Why, "as it says; the other declared accounts go ahead")
|
||||
}
|
||||
}
|
||||
switch {
|
||||
case c.InList < c.Declared:
|
||||
add(fmt.Sprintf("%d of %d declared account(s) are not in the profile's account list yet", c.Declared-c.InList, c.Declared),
|
||||
"quit Betterbird and start it from the launcher, or betterbird_restart")
|
||||
}
|
||||
if c.WithoutSignIn > 0 {
|
||||
add(fmt.Sprintf("%d declared account(s) have not signed in yet", c.WithoutSignIn),
|
||||
"a person signs in once, in Betterbird's own window, when it asks (novox/hq ADR 0316 §14)")
|
||||
}
|
||||
// The five kinds of ADR 0329: drift and what waits for Betterbird to close, per kind (§9, §11, §12).
|
||||
if _, err := m.readConfiguration(); err != nil {
|
||||
add(err.Error(), "change the setting at the controller's terminal, or derive and propose it (betterbird_derive)")
|
||||
} else if v, err := m.view(); err == nil {
|
||||
drift := m.driftCounts(v)
|
||||
waiting := m.waiting(v.p.path, v.plan, v.ra)
|
||||
for _, k := range allKinds {
|
||||
if n := drift[k]; n > 0 {
|
||||
add(fmt.Sprintf("%d declared %s differ from the profile, are not in it yet or are refused", n, k),
|
||||
"betterbird_drift with kind "+k+" says which and how; the module's user.js holds them from Betterbird's next start (betterbird_restart)")
|
||||
}
|
||||
if n := waiting[k]; n > 0 {
|
||||
add(fmt.Sprintf("%d change(s) to %s are not written yet: they are written only while Betterbird is closed", n, k),
|
||||
"quit Betterbird and start it from the launcher, or betterbird_restart (novox/hq ADR 0329 §9)")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
ps := m.mainProcs()
|
||||
switch {
|
||||
@@ -769,9 +1043,11 @@ func (m *Machine) Check() (CheckAnswer, error) {
|
||||
case err != nil:
|
||||
add("no profile: "+err.Error(), "start Betterbird once")
|
||||
case !u.Present:
|
||||
add("the module's user.js is not on this machine (~/"+declared+")", "the module is not pushed here: the controller's push")
|
||||
add("the setting prefs is not on this machine (~/"+prefsFile+")", "the module is not pushed here: the controller's push")
|
||||
case u.Valid != "":
|
||||
add("the module's user.js is refused: "+u.Valid, "change the module's setting prefs")
|
||||
case u.AccountsProblem != "":
|
||||
add("the module's user.js is not placed: "+u.AccountsProblem, "change the module's setting accounts at the controller's terminal, or propose it (betterbird_accounts)")
|
||||
case !strings.HasPrefix(u.Placed, "the module's, as declared"):
|
||||
add("the profile's user.js is "+u.Placed, "betterbird_user_js with apply, then betterbird_restart")
|
||||
case strings.HasPrefix(u.Effective, "not yet"):
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
@@ -44,8 +45,20 @@ func (f *fake) betterbird() {
|
||||
f.write(profileDir+"/extensions.json", `{"addons":[{"id":"owl@beonex.com","active":true,"location":"app-profile"},{"id":"x@mozilla.org","active":true,"location":"app-builtin"}]}`)
|
||||
}
|
||||
|
||||
// declare writes the setting prefs as the node-engine does (prefs.json, novox/hq issue 586), from user_pref
|
||||
// statements: each one entry of the map.
|
||||
func (f *fake) declare(prefs string) {
|
||||
f.write(testHome+"/.config/betterbird/user.js", mark+", novox/hq issue 345). Owned by the mesh.\n//\n"+prefs+"\n")
|
||||
f.t.Helper()
|
||||
values := map[string]any{}
|
||||
for name, lit := range statementsOf([]byte(prefs)) {
|
||||
v, ok := prefValue(lit)
|
||||
if !ok {
|
||||
f.t.Fatalf("%s: %s is not a value the setting holds", name, lit)
|
||||
}
|
||||
values[name] = v
|
||||
}
|
||||
body, _ := json.Marshal(map[string]any{"prefs": values})
|
||||
f.write(testHome+"/"+prefsFile, string(body))
|
||||
}
|
||||
|
||||
func TestTheProfileIsTheOneTheRunningBetterbirdHoldsElseTheInstallationsDefault(t *testing.T) {
|
||||
@@ -106,9 +119,9 @@ func TestTheModulesUserJSIsPlacedOnlyWhereNoOperatorsFileIs(t *testing.T) {
|
||||
t.Fatalf("%+v %v", u, err)
|
||||
}
|
||||
placed, _ := os.ReadFile(filepath.Join(f.Root, profileDir, "user.js"))
|
||||
declared, _ := os.ReadFile(filepath.Join(f.Root, testHome, ".config/betterbird/user.js"))
|
||||
if string(placed) != string(declared) {
|
||||
t.Fatal("placed whole, as declared")
|
||||
declared, _, _ := f.declaredText()
|
||||
if string(placed) != declared || !strings.Contains(declared, `user_pref("ui.key.x", "a;b");`) {
|
||||
t.Fatalf("placed whole, as declared:\n%s", placed)
|
||||
}
|
||||
if !strings.HasPrefix(u.Effective, "yes") {
|
||||
t.Fatalf("the fake Betterbird started after the file's time: %q", u.Effective)
|
||||
@@ -161,7 +174,7 @@ func TestStatusSaysWhatRunsTheProfileTheAddOnsAndTheDisplayPrefs(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(s.Running) != 1 || s.Running[0].PID != 4000 || s.Children != 1 || s.Installed != "153.1.0esr-1" || !s.Foreign ||
|
||||
s.Version != "153.1.0" || s.Profile == nil || len(s.AddOns) != 1 || s.AddOns[0].ID != "owl@beonex.com" {
|
||||
s.Installation.Version != "153.1.0" || s.Profile == nil || len(s.AddOns) != 1 || s.AddOns[0].ID != "owl@beonex.com" {
|
||||
t.Fatalf("%+v", s)
|
||||
}
|
||||
if !s.Output[0].Dead {
|
||||
@@ -238,6 +251,13 @@ func TestTheReminderTestTouchesNothingOfTheOperatorsProfile(t *testing.T) {
|
||||
!strings.Contains(string(user), "file:///run/user/1000/"+testUnit+"/test.ics") || strings.Contains(string(user), "someone@") {
|
||||
t.Fatalf("%s", user)
|
||||
}
|
||||
// Betterbird's update check stays off, so no About window opens on the operator's screen, and the
|
||||
// test's own preferences come last, so nothing carried or declared overrides them.
|
||||
if !strings.Contains(string(user), `"betterbird.update.checkIntervalDays", 0`) ||
|
||||
strings.Index(string(user), "extensions.activeThemeID") > strings.Index(string(user), "calendar.registry.meshtest.uri") ||
|
||||
strings.Index(string(user), `"calendar.alarms.show"`) > strings.Index(string(user), "calendar.registry.meshtest.uri") {
|
||||
t.Fatalf("%s", user)
|
||||
}
|
||||
// The event starts 10 minutes after the reminder, which is due 40 s from now.
|
||||
if !strings.Contains(string(cal), "DTSTART:20270115T081040Z") || !strings.Contains(string(cal), "TRIGGER:-PT10M") ||
|
||||
a.Due != "2027-01-15T08:00:40Z" {
|
||||
|
||||
@@ -0,0 +1,666 @@
|
||||
package main
|
||||
|
||||
// The configuration of Betterbird's preferences (novox/hq issue 345, issue 586), in three layers:
|
||||
//
|
||||
// - LOCKED: the enterprise policy (/opt/betterbird/distribution/policies.json). Its Preferences entries with
|
||||
// Status locked win over everything, and Betterbird's settings show them greyed out.
|
||||
// - DECLARED: the setting prefs, the operator's setup: mesh-wide, with a node's layer over it key by key. The
|
||||
// node-engine writes it to ~/.config/betterbird/prefs.json and the tools place it in the profile's user.js,
|
||||
// where it wins over prefs.js at every start.
|
||||
// - BETTERBIRD'S DEFAULT: what Betterbird starts with. The module's defaults file
|
||||
// (/opt/betterbird/defaults/pref/mesh-defaults.js) holds no value of the operator's any more (issue 586): it
|
||||
// names the preferences the module keeps at Betterbird's own default, where a value the profile saved is drift.
|
||||
//
|
||||
// betterbird_config shows each managed preference through the layers and flags the drift; betterbird_config_apply
|
||||
// clears a drifting value from prefs.js while Betterbird is not running; keep proposes a node layer of the setting
|
||||
// prefs that holds the profile's value for this node instead, through the controller's settings verb, never by
|
||||
// writing user.js.
|
||||
|
||||
import (
|
||||
"archive/zip"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
const (
|
||||
// prefDir is the installation's own defaults directory; the module places its files there.
|
||||
prefDir = installDir + "/defaults/pref"
|
||||
// meshDefaults is the module's DEFAULT layer.
|
||||
meshDefaults = prefDir + "/mesh-defaults.js"
|
||||
// omniJa is Betterbird's packed application, holding its own default preferences.
|
||||
omniJa = installDir + "/omni.ja"
|
||||
)
|
||||
|
||||
// moduleFiles are the module's files in the installation's defaults directory: their values are the module's
|
||||
// defaults, not Betterbird's.
|
||||
var moduleFiles = map[string]bool{"mesh-defaults.js": true, "vendor-prefs.js": true}
|
||||
|
||||
var (
|
||||
// defaultLine is one pref(…) or sticky_pref(…) line of a defaults file; a comment may follow it.
|
||||
defaultLine = regexp.MustCompile(`^\s*(?:sticky_)?pref\(\s*"((?:[^"\\]|\\.)*)"\s*,\s*(.*?)\s*\)\s*;`)
|
||||
// keepsDefault is a line of the module's defaults naming a preference it manages without a value.
|
||||
keepsDefault = regexp.MustCompile(`^//\s*keeps Betterbird's default:\s*([A-Za-z0-9_.@{}\-]+)\s*$`)
|
||||
)
|
||||
|
||||
// builtIn are the defaults compiled into Betterbird of the preferences the module keeps at Betterbird's default,
|
||||
// which no defaults file gives: so a value saved equal to them is not drift.
|
||||
var builtIn = map[string]string{
|
||||
themePref: `"` + defaultTheme + `"`,
|
||||
"browser.display.use_system_colors": "false",
|
||||
"layout.css.always_underline_links": "false",
|
||||
}
|
||||
|
||||
const (
|
||||
themePref = "extensions.activeThemeID"
|
||||
defaultTheme = "default-theme@mozilla.org"
|
||||
// themeRemedy is how the theme comes back: Betterbird keeps the active theme in its add-on database
|
||||
// (extensions.json), and the preference only mirrors it (XPIDatabase addonChanged), so clearing it changes
|
||||
// nothing and betterbird_config_apply never touches it.
|
||||
themeRemedy = "enable the default theme in Betterbird's add-on manager (Add-ons and Themes, Themes): the active " +
|
||||
"theme is kept in the add-on database, and its preference only mirrors it, so clearing prefs.js would not change it"
|
||||
)
|
||||
|
||||
// activeTheme is the theme the profile's add-on database marks active, or "" when it cannot be read.
|
||||
func activeTheme(profile string) string {
|
||||
raw, err := readBounded(filepath.Join(profile, "extensions.json"))
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
var ext struct {
|
||||
Addons []struct {
|
||||
ID string `json:"id"`
|
||||
Type string `json:"type"`
|
||||
Active bool `json:"active"`
|
||||
} `json:"addons"`
|
||||
}
|
||||
if json.Unmarshal(raw, &ext) != nil {
|
||||
return ""
|
||||
}
|
||||
for _, a := range ext.Addons {
|
||||
if a.Type == "theme" && a.Active {
|
||||
return a.ID
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// unmanageable says why a name in the module's defaults is never managed by these tools, or "": a credential's,
|
||||
// an account's or a server's preference, which betterbird_config never answers and betterbird_config_apply never
|
||||
// clears, whatever the defaults file lists.
|
||||
func unmanageable(name string) string {
|
||||
if why := withheld(name); why != "" {
|
||||
return why
|
||||
}
|
||||
for _, p := range accountOwned {
|
||||
if strings.HasPrefix(name, p) {
|
||||
return "an account's preference, the setting accounts' alone"
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// answer is a value as these tools answer it: masked, but for the theme's id.
|
||||
func (m *Machine) answer(name, v string) string {
|
||||
if allowedExtensions[name] {
|
||||
return v
|
||||
}
|
||||
return m.maskValue(v)
|
||||
}
|
||||
|
||||
// nodeName is the machine this runtime serves, as the mesh names it: a variable so a test answers.
|
||||
var nodeName = func() string { return os.Getenv("MESH_NODE") }
|
||||
|
||||
// shipped is the module's DEFAULT layer as placed on the machine.
|
||||
type shipped struct {
|
||||
values map[string]string
|
||||
// keeps are the managed names whose module value is Betterbird's own default.
|
||||
keeps map[string]bool
|
||||
// order is every managed name, in the file's order.
|
||||
order []string
|
||||
}
|
||||
|
||||
// parseShipped reads the module's defaults file.
|
||||
func parseShipped(raw []byte) shipped {
|
||||
s := shipped{values: map[string]string{}, keeps: map[string]bool{}}
|
||||
for _, line := range strings.Split(string(raw), "\n") {
|
||||
if mm := keepsDefault.FindStringSubmatch(strings.TrimSpace(line)); mm != nil {
|
||||
if !s.keeps[mm[1]] {
|
||||
s.keeps[mm[1]] = true
|
||||
s.order = append(s.order, mm[1])
|
||||
}
|
||||
continue
|
||||
}
|
||||
if mm := defaultLine.FindStringSubmatch(line); mm != nil {
|
||||
if _, seen := s.values[mm[1]]; !seen {
|
||||
s.order = append(s.order, mm[1])
|
||||
}
|
||||
s.values[mm[1]] = mm[2]
|
||||
}
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// shippedDefaults reads the module's defaults file from the installation.
|
||||
func (m *Machine) shippedDefaults() (shipped, error) {
|
||||
raw, err := readBounded(m.path(meshDefaults))
|
||||
if err != nil {
|
||||
return shipped{values: map[string]string{}, keeps: map[string]bool{}}, err
|
||||
}
|
||||
return parseShipped(raw), nil
|
||||
}
|
||||
|
||||
// DefaultValue is a default preference and the file that gives it.
|
||||
type DefaultValue struct {
|
||||
Value string
|
||||
From string
|
||||
// Module says the module's files give it, not Betterbird's.
|
||||
Module bool
|
||||
}
|
||||
|
||||
// readDefaultsInto reads the pref(…) lines of the files of one place, in Gecko's order: it reads a place's
|
||||
// files last-first by name (Preferences.cpp, pref_ReadDefaultPrefs), so of a name in several files the value
|
||||
// of the file first by name holds. A later place overrides an earlier one.
|
||||
func readDefaultsInto(out map[string]DefaultValue, files map[string][]byte, label func(string) (string, bool)) {
|
||||
names := make([]string, 0, len(files))
|
||||
for n := range files {
|
||||
names = append(names, n)
|
||||
}
|
||||
sort.Strings(names)
|
||||
place := map[string]DefaultValue{}
|
||||
for i := len(names) - 1; i >= 0; i-- {
|
||||
from, module := label(names[i])
|
||||
for _, line := range strings.Split(string(files[names[i]]), "\n") {
|
||||
if mm := defaultLine.FindStringSubmatch(line); mm != nil {
|
||||
place[mm[1]] = DefaultValue{Value: mm[2], From: from, Module: module}
|
||||
}
|
||||
}
|
||||
}
|
||||
for k, v := range place {
|
||||
out[k] = v
|
||||
}
|
||||
}
|
||||
|
||||
// mostDefaultsFile bounds one defaults file read from omni.ja.
|
||||
const mostDefaultsFile = 4 << 20
|
||||
|
||||
// defaults reads every default preference Betterbird starts with: greprefs.js and defaults/pref/*.js in
|
||||
// omni.ja, then the installation's defaults/pref/*.js, where the module's files are. A preference compiled into
|
||||
// Betterbird (a static preference) is in no file, and is absent here. withModule false leaves the module's own
|
||||
// files out, so what remains is Betterbird's. The error says what could not be read.
|
||||
func (m *Machine) defaults(withModule bool) (map[string]DefaultValue, error) {
|
||||
out := map[string]DefaultValue{}
|
||||
var problems []string
|
||||
if r, err := zip.OpenReader(m.path(omniJa)); err != nil {
|
||||
problems = append(problems, omniJa+" is unreadable: "+err.Error())
|
||||
} else {
|
||||
gre, app := map[string][]byte{}, map[string][]byte{}
|
||||
for _, f := range r.File {
|
||||
switch {
|
||||
case f.Name == "greprefs.js":
|
||||
case strings.HasPrefix(f.Name, "defaults/pref/") && strings.HasSuffix(f.Name, ".js") && !strings.Contains(f.Name[len("defaults/pref/"):], "/"):
|
||||
default:
|
||||
continue
|
||||
}
|
||||
rc, err := f.Open()
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
b, _ := io.ReadAll(io.LimitReader(rc, mostDefaultsFile))
|
||||
rc.Close()
|
||||
if f.Name == "greprefs.js" {
|
||||
gre[f.Name] = b
|
||||
} else {
|
||||
app[filepath.Base(f.Name)] = b
|
||||
}
|
||||
}
|
||||
r.Close()
|
||||
readDefaultsInto(out, gre, func(n string) (string, bool) { return "Betterbird's default (omni.ja " + n + ")", false })
|
||||
readDefaultsInto(out, app, func(n string) (string, bool) {
|
||||
return "Betterbird's default (omni.ja defaults/pref/" + n + ")", false
|
||||
})
|
||||
}
|
||||
installed := map[string][]byte{}
|
||||
entries, _ := os.ReadDir(m.path(prefDir))
|
||||
for _, e := range entries {
|
||||
if e.IsDir() || !strings.HasSuffix(e.Name(), ".js") || (moduleFiles[e.Name()] && !withModule) {
|
||||
continue
|
||||
}
|
||||
if b, err := readBounded(m.path(filepath.Join(prefDir, e.Name()))); err == nil {
|
||||
installed[e.Name()] = b
|
||||
}
|
||||
}
|
||||
readDefaultsInto(out, installed, func(n string) (string, bool) {
|
||||
if moduleFiles[n] {
|
||||
return "the module's default (" + prefDir + "/" + n + ")", true
|
||||
}
|
||||
return "Betterbird's default (" + prefDir + "/" + n + ")", false
|
||||
})
|
||||
if len(problems) > 0 {
|
||||
return out, errors.New(strings.Join(problems, "; "))
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// policyPrefs are the Preferences entries of the placed policy: name to its value as a preference file writes
|
||||
// it, and whether it is locked. An unreadable policy has none.
|
||||
type policyPref struct {
|
||||
Value string
|
||||
Locked bool
|
||||
}
|
||||
|
||||
func policyPrefsOf(raw []byte) map[string]policyPref {
|
||||
var p struct {
|
||||
Policies struct {
|
||||
Preferences map[string]struct {
|
||||
Value any `json:"Value"`
|
||||
Status string `json:"Status"`
|
||||
} `json:"Preferences"`
|
||||
} `json:"policies"`
|
||||
}
|
||||
out := map[string]policyPref{}
|
||||
if json.Unmarshal(raw, &p) != nil {
|
||||
return out
|
||||
}
|
||||
for name, e := range p.Policies.Preferences {
|
||||
out[name] = policyPref{Value: literal(e.Value), Locked: e.Status == "locked"}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// locks are the preferences the placed policy locks: name to value.
|
||||
func (m *Machine) locks() map[string]string {
|
||||
raw, err := readBounded(m.path(policyFile))
|
||||
if err != nil {
|
||||
return map[string]string{}
|
||||
}
|
||||
out := map[string]string{}
|
||||
for name, p := range policyPrefsOf(raw) {
|
||||
if p.Locked {
|
||||
out[name] = p.Value
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// literal is a JSON value as a preference file writes it.
|
||||
func literal(v any) string {
|
||||
switch x := v.(type) {
|
||||
case bool:
|
||||
return strconv.FormatBool(x)
|
||||
case float64:
|
||||
return strconv.FormatFloat(x, 'f', -1, 64)
|
||||
case string:
|
||||
return quoted(x)
|
||||
case nil:
|
||||
return ""
|
||||
}
|
||||
b, _ := json.Marshal(v)
|
||||
return string(b)
|
||||
}
|
||||
|
||||
// statementsOf are the user_pref(…) statements of a user.js, several to a line allowed: name to value.
|
||||
func statementsOf(raw []byte) map[string]string {
|
||||
out := map[string]string{}
|
||||
for _, line := range strings.Split(string(raw), "\n") {
|
||||
for _, st := range splitStatements(strings.TrimSpace(line)) {
|
||||
if mm := prefLine.FindStringSubmatch(st); mm != nil {
|
||||
out[mm[1]] = mm[2]
|
||||
}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// ConfigPref is one managed preference through the layers.
|
||||
type ConfigPref struct {
|
||||
Name string `json:"name"`
|
||||
// Layer is locked (the policy), declared (the setting prefs), default (a value of the module's defaults file) or
|
||||
// "Betterbird's default" (a managed preference whose module value is Betterbird's own).
|
||||
Layer string `json:"layer"`
|
||||
Locked string `json:"locked,omitempty"`
|
||||
ModuleDefault string `json:"module_default,omitempty"`
|
||||
BetterbirdDefault string `json:"betterbird_default,omitempty"`
|
||||
// Declared is what the setting prefs says on this machine: mesh-wide, with the node's layer over it.
|
||||
Declared string `json:"declared,omitempty"`
|
||||
// Profile is what the profile saved in its prefs.js.
|
||||
Profile string `json:"profile,omitempty"`
|
||||
Effective string `json:"effective"`
|
||||
From string `json:"effective_from"`
|
||||
// Drift says how the profile's own value beats the module's without a node layer saying so.
|
||||
Drift string `json:"drift,omitempty"`
|
||||
// Remedy is how drift that clearing prefs.js cannot fix is fixed (the theme).
|
||||
Remedy string `json:"remedy,omitempty"`
|
||||
// module is the value the module means: its default, or Betterbird's.
|
||||
module string
|
||||
}
|
||||
|
||||
// ConfigAnswer is what betterbird_config answers.
|
||||
type ConfigAnswer struct {
|
||||
Profile string `json:"profile"`
|
||||
Defaults string `json:"module_defaults"`
|
||||
Prefs []ConfigPref `json:"prefs"`
|
||||
// Drifting are the names that drift.
|
||||
Drifting []string `json:"drifting"`
|
||||
// Refused are names the module's defaults list that the tools never manage: an account's or a credential's.
|
||||
Refused []string `json:"refused,omitempty"`
|
||||
// PrefsProblem says why the setting prefs on this machine is refused.
|
||||
PrefsProblem string `json:"prefs_problem,omitempty"`
|
||||
// Proposed is the controller's answer to a node layer proposed with keep.
|
||||
Proposed string `json:"proposed,omitempty"`
|
||||
Note string `json:"note,omitempty"`
|
||||
}
|
||||
|
||||
// config reads every managed preference through the layers: the policy's locks, the module's defaults, the
|
||||
// node layer (the declared user.js), the profile's prefs.js and user.js.
|
||||
func (m *Machine) config() (ConfigAnswer, Profile, error) {
|
||||
a := ConfigAnswer{Prefs: []ConfigPref{}, Drifting: []string{}}
|
||||
p, err := m.profile()
|
||||
if err != nil {
|
||||
return a, p, err
|
||||
}
|
||||
a.Profile = p.Dir
|
||||
sh, serr := m.shippedDefaults()
|
||||
if serr != nil {
|
||||
a.Defaults = "not placed: " + meshDefaults + " is missing; the module's push places it"
|
||||
} else {
|
||||
a.Defaults = fmt.Sprintf("placed: %s, %d preference(s) with a value and %d kept at Betterbird's default",
|
||||
meshDefaults, len(sh.values), len(sh.keeps))
|
||||
}
|
||||
locked := m.locks()
|
||||
own, _ := m.defaults(false)
|
||||
node, _, nerr := m.readDeclaredPrefs()
|
||||
if nerr != nil {
|
||||
a.PrefsProblem = nerr.Error()
|
||||
}
|
||||
prefsRaw, _ := readBounded(filepath.Join(p.path, "prefs.js"))
|
||||
saved := readPrefs(prefsRaw)
|
||||
userRaw, _ := readBounded(filepath.Join(p.path, "user.js"))
|
||||
placed := statementsOf(userRaw)
|
||||
|
||||
var names []string
|
||||
for n := range locked {
|
||||
names = append(names, n)
|
||||
}
|
||||
sort.Strings(names)
|
||||
listed := map[string]bool{}
|
||||
for _, n := range names {
|
||||
listed[n] = true
|
||||
}
|
||||
for _, n := range append(append([]string{}, sh.order...), sortedKeys(node)...) {
|
||||
if !listed[n] {
|
||||
listed[n] = true
|
||||
names = append(names, n)
|
||||
}
|
||||
}
|
||||
theme := activeTheme(p.path)
|
||||
for _, n := range names {
|
||||
if why := unmanageable(n); why != "" {
|
||||
a.Refused = append(a.Refused, n+": "+why)
|
||||
continue
|
||||
}
|
||||
c := ConfigPref{Name: n, Declared: node[n], Profile: saved[n]}
|
||||
if n == themePref && theme != "" {
|
||||
// The add-on database says which theme is active; the preference only mirrors it.
|
||||
c.Profile = quoted(theme)
|
||||
}
|
||||
bb, hasBB := own[n]
|
||||
if hasBB {
|
||||
c.BetterbirdDefault = bb.Value
|
||||
} else if v, ok := builtIn[n]; ok {
|
||||
c.BetterbirdDefault = v
|
||||
bb.From = "Betterbird's built-in default"
|
||||
}
|
||||
switch v, isLocked := locked[n]; {
|
||||
case isLocked:
|
||||
c.Layer, c.Locked, c.module = "locked", v, v
|
||||
case c.Declared != "":
|
||||
c.Layer, c.module = "declared", c.Declared
|
||||
case sh.keeps[n]:
|
||||
c.Layer = "Betterbird's default"
|
||||
c.module = c.BetterbirdDefault
|
||||
default:
|
||||
c.Layer, c.ModuleDefault, c.module = "default", sh.values[n], sh.values[n]
|
||||
}
|
||||
switch {
|
||||
case c.Locked != "":
|
||||
c.Effective, c.From = c.Locked, "locked by the module's policy"
|
||||
case placed[n] != "":
|
||||
c.Effective, c.From = placed[n], "the profile's user.js (wins at the start)"
|
||||
case c.Profile != "":
|
||||
c.Effective, c.From = c.Profile, "the profile's prefs.js"
|
||||
case c.ModuleDefault != "":
|
||||
c.Effective, c.From = c.ModuleDefault, "the module's default"
|
||||
case c.BetterbirdDefault != "":
|
||||
c.Effective, c.From = c.BetterbirdDefault, bb.From
|
||||
default:
|
||||
c.From = "Betterbird's built-in default (in no defaults file)"
|
||||
}
|
||||
if why := drift(c, placed[n]); why != "" {
|
||||
c.Drift = why
|
||||
switch {
|
||||
case n == themePref:
|
||||
c.Remedy = themeRemedy
|
||||
case c.Layer == "declared":
|
||||
c.Remedy = "betterbird_user_js with apply places the setting in the profile's user.js; Betterbird reads it " +
|
||||
"at its next start (betterbird_restart)"
|
||||
}
|
||||
a.Drifting = append(a.Drifting, n)
|
||||
}
|
||||
c.Declared, c.Profile, c.Effective = m.answer(n, c.Declared), m.answer(n, c.Profile), m.answer(n, c.Effective)
|
||||
a.Prefs = append(a.Prefs, c)
|
||||
}
|
||||
return a, p, nil
|
||||
}
|
||||
|
||||
// drift says how the profile differs from the module's configuration, or "": for a declared preference, the
|
||||
// profile's user.js not holding the declared value (so Betterbird does not start with it); for a preference kept at
|
||||
// Betterbird's default, a value in prefs.js that is not it. A lock holds whatever the profile says.
|
||||
func drift(c ConfigPref, placed string) string {
|
||||
if c.Locked != "" {
|
||||
return ""
|
||||
}
|
||||
if c.Declared != "" {
|
||||
if placed == c.Declared {
|
||||
return ""
|
||||
}
|
||||
return fmt.Sprintf("the setting prefs says %s, and the profile's user.js does not hold it yet", c.Declared)
|
||||
}
|
||||
if c.Profile == "" || c.Profile == c.module {
|
||||
return ""
|
||||
}
|
||||
want := c.module
|
||||
if want == "" {
|
||||
want = "Betterbird's built-in default"
|
||||
}
|
||||
where := "the profile's prefs.js holds"
|
||||
if c.Name == themePref {
|
||||
where = "the profile's active theme is"
|
||||
}
|
||||
return fmt.Sprintf("%s %s, which beats the module's %s, and the setting prefs does not say so", where, c.Profile, want)
|
||||
}
|
||||
|
||||
// Config is what betterbird_config answers; with keep it proposes a node layer of the setting prefs that holds
|
||||
// the profile's own values of the named managed preferences on this machine (novox/hq issue 345). The module
|
||||
// never writes user.js for it: the operator approves the layer, and the next push writes it.
|
||||
func (m *Machine) Config(keep []string) (ConfigAnswer, error) {
|
||||
a, p, err := m.config()
|
||||
if err != nil {
|
||||
return a, err
|
||||
}
|
||||
a.Note = "drift is a declared value the profile's user.js does not hold yet (betterbird_user_js with apply), or a " +
|
||||
"value the profile saved where the module keeps Betterbird's default: betterbird_config_apply clears that one " +
|
||||
"(Betterbird quit), or keep proposes a node layer of the setting prefs that holds it for this machine"
|
||||
if len(keep) == 0 {
|
||||
return a, nil
|
||||
}
|
||||
by := map[string]ConfigPref{}
|
||||
for _, c := range a.Prefs {
|
||||
by[c.Name] = c
|
||||
}
|
||||
prefsRaw, _ := readBounded(filepath.Join(p.path, "prefs.js"))
|
||||
saved := readPrefs(prefsRaw)
|
||||
kept := map[string]string{}
|
||||
for _, n := range keep {
|
||||
c, managed := by[n]
|
||||
switch {
|
||||
case !managed:
|
||||
return a, fmt.Errorf("nothing was proposed: %s is not a preference the module manages (betterbird_config lists them)", n)
|
||||
case c.Locked != "":
|
||||
return a, fmt.Errorf("nothing was proposed: %s is locked by the module's policy; a lock changes only in the catalogue", n)
|
||||
case n == themePref:
|
||||
return a, fmt.Errorf("nothing was proposed: a node layer cannot choose the theme; the add-on manager does (%s)", themeRemedy)
|
||||
case saved[n] == "":
|
||||
return a, fmt.Errorf("nothing was proposed: the profile saved no value of its own for %s", n)
|
||||
}
|
||||
kept[n] = saved[n]
|
||||
}
|
||||
node := nodeName()
|
||||
if node == "" {
|
||||
return a, errors.New("nothing was proposed: this runtime does not say which machine it serves (MESH_NODE)")
|
||||
}
|
||||
// A layer is replaced whole, so the proposal carries this node's layer as it stands, with the kept values in
|
||||
// the setting prefs beside what it held (novox/hq issue 586: one entry per preference).
|
||||
layer, err := readLayer(node)
|
||||
if err != nil {
|
||||
return a, fmt.Errorf("nothing was proposed: %v", err)
|
||||
}
|
||||
prefs, _ := layer["prefs"].(map[string]any)
|
||||
if prefs == nil {
|
||||
prefs = map[string]any{}
|
||||
}
|
||||
for _, n := range keep {
|
||||
v, ok := prefValue(kept[n])
|
||||
if !ok {
|
||||
return a, fmt.Errorf("nothing was proposed: the profile's value of %s is not one the setting can hold", n)
|
||||
}
|
||||
prefs[n] = v
|
||||
}
|
||||
layer["prefs"] = prefs
|
||||
said, err := proposeLayer(node, layer, false)
|
||||
a.Proposed = m.maskValue(said)
|
||||
if err != nil {
|
||||
return a, err
|
||||
}
|
||||
a.Note = "proposed for " + node + ": the operator approves it on a channel that proves who answers; the next " +
|
||||
"push writes it, and it is placed at Betterbird's next start from the launcher (or betterbird_restart). " +
|
||||
"The layer carries this node's layer as it stands, with the kept preferences in the setting prefs"
|
||||
return a, nil
|
||||
}
|
||||
|
||||
// Cleared is one value betterbird_config_apply took out of prefs.js.
|
||||
type Cleared struct {
|
||||
Name string `json:"name"`
|
||||
Was string `json:"was"`
|
||||
// Now is the value that holds once Betterbird starts again.
|
||||
Now string `json:"now"`
|
||||
}
|
||||
|
||||
// ConfigApplyAnswer is what betterbird_config_apply answers.
|
||||
type ConfigApplyAnswer struct {
|
||||
Profile string `json:"profile"`
|
||||
Applied bool `json:"applied"`
|
||||
Clears []Cleared `json:"clears"`
|
||||
// NotCleared is drift that clearing prefs.js cannot fix, each with how it is fixed.
|
||||
NotCleared []string `json:"not_cleared,omitempty"`
|
||||
Note string `json:"note"`
|
||||
}
|
||||
|
||||
// ConfigApply takes the drifting values out of the profile's prefs.js, so the module's value holds at the next
|
||||
// start. Without apply it lists them. It acts only while Betterbird is not running, because Betterbird
|
||||
// rewrites prefs.js when it quits. Every other line of prefs.js is kept as it was.
|
||||
func (m *Machine) ConfigApply(apply bool) (ConfigApplyAnswer, error) {
|
||||
c, p, err := m.config()
|
||||
a := ConfigApplyAnswer{Profile: c.Profile, Clears: []Cleared{}}
|
||||
if err != nil {
|
||||
return a, err
|
||||
}
|
||||
if strings.HasPrefix(c.Defaults, "not placed") {
|
||||
return a, fmt.Errorf("the module's defaults are %s: nothing to bring the profile back to", c.Defaults)
|
||||
}
|
||||
drifting := map[string]bool{}
|
||||
for _, cp := range c.Prefs {
|
||||
if cp.Drift != "" && cp.Remedy != "" {
|
||||
a.NotCleared = append(a.NotCleared, cp.Name+": "+cp.Remedy)
|
||||
continue
|
||||
}
|
||||
if cp.Drift != "" {
|
||||
drifting[cp.Name] = true
|
||||
now := m.answer(cp.Name, cp.module)
|
||||
if now == "" {
|
||||
now = "Betterbird's built-in default"
|
||||
}
|
||||
a.Clears = append(a.Clears, Cleared{Name: cp.Name, Was: cp.Profile, Now: now})
|
||||
}
|
||||
}
|
||||
if !apply {
|
||||
a.Note = "a dry run: these values would be taken out of prefs.js. Quit Betterbird, then call with apply"
|
||||
return a, nil
|
||||
}
|
||||
if ps := m.mainProcs(); len(ps) > 0 {
|
||||
return a, fmt.Errorf("Betterbird is running (pid %d): it rewrites prefs.js when it quits, so nothing was cleared; quit it, then apply again", ps[0].PID)
|
||||
}
|
||||
// The profile's own lock is Gecko's sign that a Betterbird holds it, whichever installation started it.
|
||||
if _, err := os.Lstat(filepath.Join(p.path, "lock")); err == nil {
|
||||
return a, errors.New("the profile is locked (its lock file is there): a Betterbird holds it, or one ended " +
|
||||
"without quitting; nothing was cleared. Quit Betterbird (or start and quit it once), then apply again")
|
||||
}
|
||||
if len(drifting) == 0 {
|
||||
a.Applied, a.Note = true, "nothing in prefs.js drifts: it was left as it was"
|
||||
return a, nil
|
||||
}
|
||||
target := filepath.Join(p.path, "prefs.js")
|
||||
raw, err := readBounded(target)
|
||||
if err != nil {
|
||||
return a, err
|
||||
}
|
||||
st, err := os.Stat(target)
|
||||
if err != nil {
|
||||
return a, err
|
||||
}
|
||||
lines := strings.SplitAfter(string(raw), "\n")
|
||||
var kept strings.Builder
|
||||
for _, l := range lines {
|
||||
if mm := prefLine.FindStringSubmatch(strings.TrimRight(l, "\n")); mm != nil && drifting[mm[1]] {
|
||||
continue
|
||||
}
|
||||
kept.WriteString(l)
|
||||
}
|
||||
tmp := target + ".mesh-new"
|
||||
if err := writeSynced(tmp, []byte(kept.String()), st.Mode().Perm()); err != nil {
|
||||
os.Remove(tmp)
|
||||
return a, err
|
||||
}
|
||||
if err := os.Rename(tmp, target); err != nil {
|
||||
os.Remove(tmp)
|
||||
return a, err
|
||||
}
|
||||
a.Applied = true
|
||||
a.Note = fmt.Sprintf("cleared %d value(s) from prefs.js; the module's values hold from Betterbird's next start", len(a.Clears))
|
||||
return a, nil
|
||||
}
|
||||
|
||||
// writeSynced writes a file and syncs it to disk before it is renamed into place, so a crash never leaves an
|
||||
// empty prefs.js.
|
||||
func writeSynced(path string, data []byte, mode os.FileMode) error {
|
||||
f, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, mode)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := f.Write(data); err != nil {
|
||||
f.Close()
|
||||
return err
|
||||
}
|
||||
if err := f.Sync(); err != nil {
|
||||
f.Close()
|
||||
return err
|
||||
}
|
||||
return f.Close()
|
||||
}
|
||||
@@ -0,0 +1,541 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"archive/zip"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// The configuration the module ships (novox/hq issue 345): the policy's locks, the module's defaults file, and the
|
||||
// tools that show the layers, flag drift, clear it and propose a node layer.
|
||||
|
||||
// shippedDefaultsWant is the setup the operator approved on 2026-10-12 (the laptop's, with reminders missed while
|
||||
// asleep shown at the next start and chat off), which the module shipped as its defaults file until issue 586 moved
|
||||
// it into the mesh-wide layer of the setting prefs: the catalogue holds none of it now.
|
||||
var shippedDefaultsWant = map[string]string{
|
||||
"mail.uidensity": "0",
|
||||
"mail.uifontsize": "14",
|
||||
"mail.dark-reader.enabled": "false",
|
||||
"mail.pane_config.dynamic": "2",
|
||||
"mail.pane_config.multiline_all": "true",
|
||||
"mail.threadpane.listview": "0",
|
||||
"mailnews.default_view_flags": "1",
|
||||
"mail.warn_on_shift_delete": "false",
|
||||
"mail.biff.show_alert": "false",
|
||||
"mail.biff.play_sound": "false",
|
||||
"mail.biff.show_tray_icon": "false",
|
||||
"mail.biff.use_system_alert": "true",
|
||||
"calendar.view.daystarthour": "9",
|
||||
"calendar.view.dayendhour": "20",
|
||||
"calendar.item.promptDelete": "false",
|
||||
"calendar.timezone.useSystemTimezone": "true",
|
||||
"calendar.alarms.show": "true",
|
||||
"calendar.alarms.showmissed": "true",
|
||||
"mail.chat.enabled": "false",
|
||||
"mailnews.message_display.disable_remote_image": "true",
|
||||
"privacy.donottrackheader.enabled": "true",
|
||||
}
|
||||
|
||||
var shippedKeepsWant = []string{"extensions.activeThemeID", "browser.display.use_system_colors",
|
||||
"layout.css.always_underline_links", "mail.spam.manualMark", "mail.compose.show_attachment_pane"}
|
||||
|
||||
func repoFile(t *testing.T, rel string) []byte {
|
||||
t.Helper()
|
||||
raw, err := os.ReadFile(filepath.Join("..", "..", rel))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return raw
|
||||
}
|
||||
|
||||
// withDefaults places the module's defaults file as the push does.
|
||||
func (f *fake) withDefaults() {
|
||||
f.t.Helper()
|
||||
f.write(meshDefaults, string(repoFile(f.t, "install/defaults-pref/mesh-defaults.js")))
|
||||
}
|
||||
|
||||
// omni places a packed application holding the given files.
|
||||
func (f *fake) omni(files map[string]string) {
|
||||
f.t.Helper()
|
||||
full := filepath.Join(f.Root, omniJa)
|
||||
if err := os.MkdirAll(filepath.Dir(full), 0o755); err != nil {
|
||||
f.t.Fatal(err)
|
||||
}
|
||||
out, err := os.Create(full)
|
||||
if err != nil {
|
||||
f.t.Fatal(err)
|
||||
}
|
||||
w := zip.NewWriter(out)
|
||||
for name, body := range files {
|
||||
e, _ := w.Create(name)
|
||||
e.Write([]byte(body))
|
||||
}
|
||||
w.Close()
|
||||
out.Close()
|
||||
}
|
||||
|
||||
// betterbirdsOwn is a small omni.ja: two files that contradict each other, as all-thunderbird.js and
|
||||
// mailnews.js do in the pinned release.
|
||||
func (f *fake) betterbirdsOwn() {
|
||||
f.omni(map[string]string{
|
||||
"greprefs.js": `pref("gfx.webrender.software", false);` + "\n",
|
||||
"defaults/pref/all-thunderbird.js": `pref("offline.autoDetect", true);` + "\n" +
|
||||
`pref("mail.threadpane.listview", 1);` + "\n" + `pref("mail.uidensity", 1);` + "\n" +
|
||||
`pref("mail.shell.checkDefaultClient", true);` + "\n",
|
||||
"defaults/pref/mailnews.js": `pref("offline.autoDetect", false);` + "\n" +
|
||||
`pref("mail.biff.show_alert", true); // a comment after it` + "\n" + `pref("mail.spam.manualMark", false);` + "\n" +
|
||||
`pref("mailnews.start_page.enabled", true);` + "\n",
|
||||
"defaults/pref/calendar.js": `pref("calendar.alarms.showmissed", false);` + "\n",
|
||||
"chrome/other.js": `pref("not.a.default", 1);` + "\n",
|
||||
})
|
||||
}
|
||||
|
||||
func stopBetterbird(t *testing.T, f *fake) {
|
||||
t.Helper()
|
||||
for _, pid := range []string{"4000", "4001"} {
|
||||
if err := os.RemoveAll(filepath.Join(f.Root, "proc", pid)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheModuleShipsTheApprovedLocksAndDefaults(t *testing.T) {
|
||||
m, _ := readManifest(t)
|
||||
if a := artifact(t, m, "defaults"); a["from"] != "install/defaults-pref" {
|
||||
t.Fatalf("the defaults file ships through the defaults artifact: %v", a)
|
||||
}
|
||||
if r := resource(t, m, "spelling"); r["path"] != prefDir || r["artifact"] != "defaults" {
|
||||
t.Fatalf("placed in Betterbird's own defaults directory: %v", r)
|
||||
}
|
||||
raw := repoFile(t, "install/distribution/policies.json")
|
||||
if why := policyProblem(raw); why != "" {
|
||||
t.Fatal(why)
|
||||
}
|
||||
var p struct {
|
||||
Policies map[string]json.RawMessage `json:"policies"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &p); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for k, want := range map[string]string{"DisableAppUpdate": "true", "DisableTelemetry": "true", "ExtensionUpdate": "false"} {
|
||||
if string(p.Policies[k]) != want {
|
||||
t.Errorf("%s is %s, not %s", k, p.Policies[k], want)
|
||||
}
|
||||
}
|
||||
prefs := policyPrefsOf(raw)
|
||||
for _, n := range []string{"mail.shell.checkDefaultClient", "mail.inappnotifications.enabled", "mailnews.start_page.enabled"} {
|
||||
if prefs[n] != (policyPref{Value: "false", Locked: true}) {
|
||||
t.Errorf("%s is locked off: %+v", n, prefs[n])
|
||||
}
|
||||
}
|
||||
// Betterbird's Preferences policy accepts only these prefixes (Policies.sys.mjs); privacy. and toolkit. are not.
|
||||
allowed := []string{"accessibility.", "app.update.", "browser.", "calendar.", "chat.", "datareporting.policy.", "dom.",
|
||||
"extensions.", "general.autoScroll", "general.smoothScroll", "geo.", "gfx.", "intl.", "layers.", "layout.", "mail.",
|
||||
"mailnews.", "media.", "network.", "pdfjs.", "places.", "print.", "signon.", "spellchecker.", "ui.", "widget."}
|
||||
for n := range prefs {
|
||||
ok := false
|
||||
for _, a := range allowed {
|
||||
ok = ok || strings.HasPrefix(n, a)
|
||||
}
|
||||
if !ok {
|
||||
t.Errorf("the policy cannot lock %s: its prefix is not one Betterbird accepts", n)
|
||||
}
|
||||
}
|
||||
|
||||
text := string(repoFile(t, "install/defaults-pref/mesh-defaults.js"))
|
||||
sh := parseShipped([]byte(text))
|
||||
if len(sh.values) != 0 {
|
||||
t.Errorf("the module's defaults hold no value of the operator's: they are the setting prefs (issue 586): %v", sh.values)
|
||||
}
|
||||
for n := range shippedDefaultsWant {
|
||||
if strings.Contains(text, n) {
|
||||
t.Errorf("%s is the operator's choice and lives in the setting prefs, never in the catalogue", n)
|
||||
}
|
||||
}
|
||||
var keeps []string
|
||||
for _, n := range sh.order {
|
||||
if sh.keeps[n] {
|
||||
keeps = append(keeps, n)
|
||||
}
|
||||
}
|
||||
if !reflect.DeepEqual(keeps, shippedKeepsWant) {
|
||||
t.Errorf("the preferences kept at Betterbird's default: %v", keeps)
|
||||
}
|
||||
lines := strings.Split(text, "\n")
|
||||
for i, l := range lines {
|
||||
if defaultLine.MatchString(l) && (i == 0 || !strings.HasPrefix(lines[i-1], "// ")) {
|
||||
t.Errorf("line %d has no comment saying why: %s", i+1, l)
|
||||
}
|
||||
if t2 := strings.TrimSpace(l); t2 != "" && !strings.HasPrefix(t2, "//") && !defaultLine.MatchString(l) {
|
||||
t.Errorf("line %d is neither a comment nor a pref(…) line: %s", i+1, l)
|
||||
}
|
||||
}
|
||||
for n := range sh.values {
|
||||
if withheld(n) != "" || passwordSecret.MatchString(n) {
|
||||
t.Errorf("%s is an account's or a credential's preference, never a default of the module", n)
|
||||
}
|
||||
if _, locked := prefs[n]; locked {
|
||||
t.Errorf("%s is both locked and a default", n)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// T2: Betterbird's own defaults are values read from omni.ja, in Gecko's order, and the module's beside them.
|
||||
func TestPrefsSayTheDefaultAndTheFileThatGivesIt(t *testing.T) {
|
||||
f := newFake(t)
|
||||
f.betterbird()
|
||||
f.placedByTheMesh()
|
||||
f.withDefaults()
|
||||
f.betterbirdsOwn()
|
||||
links(t, nil)
|
||||
got, err := f.Prefs([]string{"offline.autoDetect", "mail.threadpane.listview", "mail.uidensity", "gfx.webrender.software",
|
||||
"mailnews.start_page.enabled", "layout.css.always_underline_links", "not.a.default"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
by := map[string]Pref{}
|
||||
for _, p := range got["prefs"].([]Pref) {
|
||||
by[p.Name] = p
|
||||
}
|
||||
if p := by["offline.autoDetect"]; p.Value != "true" || p.Default != "true" || !strings.Contains(p.DefaultFrom, "all-thunderbird.js") {
|
||||
t.Errorf("of two files, the one first by name holds, as Gecko reads them: %+v", p)
|
||||
}
|
||||
if p := by["mail.threadpane.listview"]; p.Value != "1" || strings.Contains(p.From, "module") || strings.Contains(p.DefaultFrom, "module") {
|
||||
t.Errorf("the module ships no value of the operator's, so Betterbird's own default holds: %+v", p)
|
||||
}
|
||||
if p := by["gfx.webrender.software"]; p.Value != "false" || !strings.Contains(p.From, "greprefs.js") {
|
||||
t.Errorf("%+v", p)
|
||||
}
|
||||
if p := by["mailnews.start_page.enabled"]; p.Value != "false" || p.From != "locked by the module's policy" || p.Default != "true" {
|
||||
t.Errorf("a locked preference says its lock and Betterbird's default under it: %+v", p)
|
||||
}
|
||||
if p := by["layout.css.always_underline_links"]; p.Value != "" || !strings.Contains(p.From, "built-in") {
|
||||
t.Errorf("a preference in no file is said as such: %+v", p)
|
||||
}
|
||||
if p := by["not.a.default"]; p.Default != "" {
|
||||
t.Errorf("only defaults/pref and greprefs.js are defaults: %+v", p)
|
||||
}
|
||||
}
|
||||
|
||||
// T4: of mailnews.* only what names an account, a server or the operator's tags is withheld.
|
||||
func TestMailnewsIsWithheldOnlyWhereItNamesAnAccount(t *testing.T) {
|
||||
for name, held := range map[string]bool{
|
||||
"mailnews.start_page.enabled": false,
|
||||
"mailnews.default_view_flags": false,
|
||||
"mailnews.message_display.disable_remote_image": false,
|
||||
"mailnews.tags.$label1.tag": true,
|
||||
"mailnews.smtp.loglevel": true,
|
||||
"mailnews.reply_to_self_check_all_ident": true,
|
||||
"mailnews.auto_config.fetchFromISP.sendEmailAddress": true,
|
||||
"mailnews.oauth.useSchemeRedirect": true,
|
||||
"mail.server.server1.hostname": true,
|
||||
} {
|
||||
if got := withheld(name) != ""; got != held {
|
||||
t.Errorf("%s: withheld %v, want %v", name, got, held)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// T1: every managed preference through the layers, and drift: a declared value the profile's user.js does not hold
|
||||
// yet, or a value the profile saved where the module keeps Betterbird's default.
|
||||
func TestTheConfigShowsTheLayersAndTheDrift(t *testing.T) {
|
||||
f := newFake(t)
|
||||
f.betterbird()
|
||||
f.placedByTheMesh()
|
||||
f.withDefaults()
|
||||
f.betterbirdsOwn()
|
||||
links(t, nil)
|
||||
f.declare(`user_pref("mail.uifontsize", 16); user_pref("mail.biff.show_alert", false); user_pref("calendar.alarms.showmissed", true);`)
|
||||
f.write(profileDir+"/user.js", mark+")\n"+`user_pref("mail.uifontsize", 16);`+"\n"+`user_pref("calendar.alarms.showmissed", true);`+"\n")
|
||||
f.write(profileDir+"/prefs.js", "// Mozilla User Preferences\n"+
|
||||
`user_pref("extensions.activeThemeID", "thunderbird-compact-dark@mozilla.org");`+"\n"+
|
||||
`user_pref("mail.uifontsize", 12);`+"\n"+
|
||||
`user_pref("mail.biff.show_alert", true);`+"\n"+
|
||||
`user_pref("mail.spam.manualMark", true);`+"\n"+
|
||||
`user_pref("browser.display.use_system_colors", false);`+"\n"+
|
||||
`user_pref("mailnews.start_page.enabled", true);`+"\n")
|
||||
c, err := f.Config(nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
by := map[string]ConfigPref{}
|
||||
for _, p := range c.Prefs {
|
||||
by[p.Name] = p
|
||||
}
|
||||
if len(c.Prefs) != 3+3+len(shippedKeepsWant) || !strings.HasPrefix(c.Defaults, "placed") {
|
||||
t.Fatalf("every lock, declared preference and kept default: %d, %s", len(c.Prefs), c.Defaults)
|
||||
}
|
||||
if !reflect.DeepEqual(c.Drifting, []string{"extensions.activeThemeID", "mail.spam.manualMark", "mail.biff.show_alert"}) {
|
||||
t.Fatalf("drift: %v", c.Drifting)
|
||||
}
|
||||
if p := by["mail.biff.show_alert"]; p.Layer != "declared" || p.Declared != "false" || p.Profile != "true" ||
|
||||
p.Effective != "true" || !strings.Contains(p.Drift, "the setting prefs says false") || !strings.Contains(p.Remedy, "betterbird_user_js") {
|
||||
t.Errorf("a declared value the profile's user.js does not hold yet: %+v", p)
|
||||
}
|
||||
if p := by["mail.spam.manualMark"]; p.Layer != "Betterbird's default" || p.BetterbirdDefault != "false" ||
|
||||
!strings.Contains(p.Drift, "beats the module's false") || p.Remedy != "" {
|
||||
t.Errorf("a saved value where the module keeps Betterbird's default: %+v", p)
|
||||
}
|
||||
if p := by["extensions.activeThemeID"]; p.Layer != "Betterbird's default" || !strings.Contains(p.Drift, `"default-theme@mozilla.org"`) ||
|
||||
!strings.Contains(p.Remedy, "add-on manager") || p.Profile != `"thunderbird-compact-dark@mozilla.org"` {
|
||||
t.Errorf("another theme is drift whose remedy is the add-on manager, its id answered whole: %+v", p)
|
||||
}
|
||||
if p := by["browser.display.use_system_colors"]; p.Drift != "" {
|
||||
t.Errorf("a saved value equal to Betterbird's default is not drift: %+v", p)
|
||||
}
|
||||
if p := by["mail.uifontsize"]; p.Drift != "" || p.Declared != "16" || p.Effective != "16" || !strings.Contains(p.From, "user.js") {
|
||||
t.Errorf("a declared value the user.js holds wins at the start, whatever prefs.js saved: %+v", p)
|
||||
}
|
||||
if p := by["mailnews.start_page.enabled"]; p.Layer != "locked" || p.Effective != "false" || p.Drift != "" {
|
||||
t.Errorf("a lock wins over the profile: %+v", p)
|
||||
}
|
||||
if p := by["calendar.alarms.showmissed"]; p.Effective != "true" || p.Layer != "declared" {
|
||||
t.Errorf("%+v", p)
|
||||
}
|
||||
// T5: the check says the drift, with both remedies.
|
||||
ch, err := f.Check()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
found := false
|
||||
for _, x := range ch.Findings {
|
||||
found = found || (strings.Contains(x.What, "1 managed preference(s) drift") && strings.Contains(x.What, "mail.spam.manualMark") &&
|
||||
strings.Contains(x.Do, "betterbird_config_apply") && strings.Contains(x.Do, "keep"))
|
||||
}
|
||||
theme := false
|
||||
for _, x := range ch.Findings {
|
||||
theme = theme || (strings.HasPrefix(x.What, "extensions.activeThemeID drifts") && strings.Contains(x.Do, "add-on manager") &&
|
||||
!strings.Contains(x.Do, "betterbird_config_apply"))
|
||||
}
|
||||
declared := false
|
||||
for _, x := range ch.Findings {
|
||||
declared = declared || (strings.HasPrefix(x.What, "mail.biff.show_alert drifts") && strings.Contains(x.Do, "betterbird_user_js"))
|
||||
}
|
||||
if !found || !theme || !declared {
|
||||
t.Fatalf("the check reports the drift: %+v", ch.Findings)
|
||||
}
|
||||
os.Remove(filepath.Join(f.Root, meshDefaults))
|
||||
ch, _ = f.Check()
|
||||
found = false
|
||||
for _, x := range ch.Findings {
|
||||
found = found || strings.Contains(x.What, "the module's defaults are not placed")
|
||||
}
|
||||
if !found {
|
||||
t.Fatalf("defaults missing on a node of the mesh's release is a finding: %+v", ch.Findings)
|
||||
}
|
||||
}
|
||||
|
||||
// T3: the drifting values are cleared from prefs.js only while Betterbird is not running, and nothing else.
|
||||
func TestConfigApplyClearsTheDriftOnlyWhileBetterbirdIsQuit(t *testing.T) {
|
||||
f := newFake(t)
|
||||
f.betterbird()
|
||||
f.placedByTheMesh()
|
||||
f.withDefaults()
|
||||
f.betterbirdsOwn()
|
||||
links(t, nil)
|
||||
f.declare("// none")
|
||||
prefs := "// Mozilla User Preferences\n" +
|
||||
`user_pref("extensions.activeThemeID", "thunderbird-compact-dark@mozilla.org");` + "\n" +
|
||||
`user_pref("mail.identity.id1.useremail", "someone@example.org");` + "\n" +
|
||||
`user_pref("mail.spam.manualMark", true);` + "\n" +
|
||||
`user_pref("mail.uidensity", 0);` + "\n"
|
||||
f.write(profileDir+"/prefs.js", prefs)
|
||||
dry, err := f.ConfigApply(false)
|
||||
if err != nil || dry.Applied || len(dry.Clears) != 1 || len(dry.NotCleared) != 1 {
|
||||
t.Fatalf("a dry run lists and changes nothing: %+v %v", dry, err)
|
||||
}
|
||||
if f.read(profileDir+"/prefs.js") != prefs {
|
||||
t.Fatal("a dry run wrote prefs.js")
|
||||
}
|
||||
if _, err := f.ConfigApply(true); err == nil || !strings.Contains(err.Error(), "running") {
|
||||
t.Fatalf("refused while Betterbird runs: %v", err)
|
||||
}
|
||||
if f.read(profileDir+"/prefs.js") != prefs {
|
||||
t.Fatal("prefs.js changed while Betterbird runs")
|
||||
}
|
||||
stopBetterbird(t, f)
|
||||
f.write(profileDir+"/lock", "")
|
||||
if _, err := f.ConfigApply(true); err == nil || !strings.Contains(err.Error(), "locked") {
|
||||
t.Fatalf("refused while the profile's own lock is there: %v", err)
|
||||
}
|
||||
if err := os.Remove(filepath.Join(f.Root, profileDir, "lock")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
a, err := f.ConfigApply(true)
|
||||
if err != nil || !a.Applied {
|
||||
t.Fatalf("%+v %v", a, err)
|
||||
}
|
||||
want := "// Mozilla User Preferences\n" +
|
||||
`user_pref("extensions.activeThemeID", "thunderbird-compact-dark@mozilla.org");` + "\n" +
|
||||
`user_pref("mail.identity.id1.useremail", "someone@example.org");` + "\n" +
|
||||
`user_pref("mail.uidensity", 0);` + "\n"
|
||||
if got := f.read(profileDir + "/prefs.js"); got != want {
|
||||
t.Fatalf("only the drifting lines go:\n%s", got)
|
||||
}
|
||||
said := map[string]Cleared{}
|
||||
for _, c := range a.Clears {
|
||||
said[c.Name] = c
|
||||
}
|
||||
if c := said["mail.spam.manualMark"]; c.Was != "true" || c.Now != "false" {
|
||||
t.Errorf("it says what it cleared and what holds now: %+v", c)
|
||||
}
|
||||
if _, cleared := said["extensions.activeThemeID"]; cleared || len(a.NotCleared) != 1 || !strings.Contains(a.NotCleared[0], "add-on manager") {
|
||||
t.Errorf("the theme is never cleared, and said with its remedy: %+v", a)
|
||||
}
|
||||
raw, _ := json.Marshal(a)
|
||||
if strings.Contains(string(raw), "example.org") {
|
||||
t.Fatalf("no account value leaves: %s", raw)
|
||||
}
|
||||
if c, _ := f.Config(nil); !reflect.DeepEqual(c.Drifting, []string{"extensions.activeThemeID"}) {
|
||||
t.Fatalf("once applied, only the theme drifts, until the add-on manager changes it: %v", c.Drifting)
|
||||
}
|
||||
os.Remove(filepath.Join(f.Root, meshDefaults))
|
||||
if _, err := f.ConfigApply(true); err == nil || !strings.Contains(err.Error(), "not placed") {
|
||||
t.Fatalf("without the module's defaults there is nothing to come back to: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// T6: keeping a value on one node is that node's layer of the setting prefs proposed through the controller, one
|
||||
// entry per preference beside what the layer holds (issue 586), never a write to user.js.
|
||||
func TestKeepProposesANodeLayerThroughTheController(t *testing.T) {
|
||||
f := newFake(t)
|
||||
f.betterbird()
|
||||
f.placedByTheMesh()
|
||||
f.withDefaults()
|
||||
f.betterbirdsOwn()
|
||||
links(t, nil)
|
||||
f.declare(`user_pref("calendar.alarms.playsound", false); user_pref("mail.uifontsize", 12);`)
|
||||
f.write(profileDir+"/prefs.js", `user_pref("mail.uifontsize", 16);`+"\n"+`user_pref("mail.spam.manualMark", true);`+"\n")
|
||||
oldNode, oldAsk := nodeName, asker
|
||||
t.Cleanup(func() { nodeName, asker = oldNode, oldAsk })
|
||||
nodeName = func() string { return "shanks" }
|
||||
var asked []map[string]any
|
||||
asker = func(v string, a any) (json.RawMessage, error) {
|
||||
args := a.(map[string]any)
|
||||
asked = append(asked, args)
|
||||
if args["propose"] == nil {
|
||||
return json.RawMessage(`{"ok": true, "output": "{\"prefs\": {\"calendar.alarms.playsound\": false}, \"identities\": {\"mail\": {\"account\": \"mail\"}}}\n set at the terminal\nbetterbird on shanks, every value"}`), nil
|
||||
}
|
||||
return json.RawMessage(`{"ok": true, "output": "proposed: p-1"}`), nil
|
||||
}
|
||||
before := f.read(testHome + "/" + prefsFile)
|
||||
c, err := f.Config([]string{"mail.uifontsize", "mail.spam.manualMark"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(asked) != 2 || asked[0]["node"] != "shanks" || asked[1]["node"] != "shanks" || asked[1]["propose"] != "true" ||
|
||||
asked[1]["module"] != "betterbird" {
|
||||
t.Fatalf("the node's layer read, then proposed: %v", asked)
|
||||
}
|
||||
var values map[string]any
|
||||
if err := json.Unmarshal([]byte(asked[1]["values"].(string)), &values); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := map[string]any{"calendar.alarms.playsound": false, "mail.uifontsize": float64(16), "mail.spam.manualMark": true}
|
||||
if !reflect.DeepEqual(values["prefs"], want) || values["identities"] == nil {
|
||||
t.Fatalf("the layer as it stands with the kept values, one entry each:\n got %v\nwant %v", values, want)
|
||||
}
|
||||
if c.Proposed != "proposed: p-1" || f.read(testHome+"/"+prefsFile) != before {
|
||||
t.Fatalf("proposed, and nothing written: %+v", c)
|
||||
}
|
||||
for _, bad := range [][]string{{"mailnews.start_page.enabled"}, {"some.other.pref"}, {"calendar.alarms.playsound"}} {
|
||||
asked = nil
|
||||
if _, err := f.Config(bad); err == nil || len(asked) != 0 {
|
||||
t.Errorf("%v: a lock, an unmanaged name or a value the profile did not save is refused: %v", bad, err)
|
||||
}
|
||||
}
|
||||
nodeName = func() string { return "" }
|
||||
if _, err := f.Config([]string{"mail.uifontsize"}); err == nil || !strings.Contains(err.Error(), "MESH_NODE") {
|
||||
t.Fatalf("%v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// The review of pull request 253: whether Betterbird asks to be the default client is decided from the policy's
|
||||
// own Preferences entry, never from what else the policy lacks.
|
||||
func TestTheDefaultClientAskIsDecidedFromThePreferencesEntry(t *testing.T) {
|
||||
f := newFake(t)
|
||||
f.betterbird()
|
||||
f.placedByTheMesh()
|
||||
links(t, nil)
|
||||
f.write(policyFile, `{"policies": {"ExtensionSettings": {}}}`)
|
||||
if why := f.defaultClientAsk(filepath.Join(f.Root, profileDir)); why == "" {
|
||||
t.Fatal("a policy lacking the updater and the lock still asks")
|
||||
}
|
||||
f.write(policyFile, `{"policies": {"Preferences": {"mail.shell.checkDefaultClient": {"Value": false, "Status": "locked"}}}}`)
|
||||
if why := f.defaultClientAsk(filepath.Join(f.Root, profileDir)); why != "" {
|
||||
t.Fatalf("the lock holds whatever else the policy lacks: %s", why)
|
||||
}
|
||||
}
|
||||
|
||||
// The review of pull request 256: the theme is the add-on database's, and a saved default is not drift.
|
||||
func TestTheThemeIsTheAddOnDatabasesAndASavedDefaultIsNotDrift(t *testing.T) {
|
||||
f := newFake(t)
|
||||
f.betterbird()
|
||||
f.placedByTheMesh()
|
||||
f.withDefaults()
|
||||
links(t, nil)
|
||||
f.write(profileDir+"/prefs.js", `user_pref("browser.display.use_system_colors", false);`+"\n"+
|
||||
`user_pref("layout.css.always_underline_links", false);`+"\n")
|
||||
f.write(profileDir+"/extensions.json", `{"addons":[{"id":"thunderbird-compact-dark@mozilla.org","type":"theme","active":true,"location":"app-builtin"},`+
|
||||
`{"id":"default-theme@mozilla.org","type":"theme","active":false,"location":"app-builtin"}]}`)
|
||||
c, err := f.Config(nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !reflect.DeepEqual(c.Drifting, []string{"extensions.activeThemeID"}) {
|
||||
t.Fatalf("the active theme from the add-on database, with no preference saved, is drift; a saved default is not: %v", c.Drifting)
|
||||
}
|
||||
f.write(profileDir+"/prefs.js", `user_pref("extensions.activeThemeID", "default-theme@mozilla.org");`+"\n")
|
||||
f.write(profileDir+"/extensions.json", `{"addons":[{"id":"default-theme@mozilla.org","type":"theme","active":true,"location":"app-builtin"}]}`)
|
||||
if c, _ := f.Config(nil); len(c.Drifting) != 0 {
|
||||
t.Fatalf("a profile back on the default theme does not drift: %v", c.Drifting)
|
||||
}
|
||||
if _, err := f.Config([]string{"extensions.activeThemeID"}); err == nil || !strings.Contains(err.Error(), "add-on manager") {
|
||||
t.Fatalf("a node layer cannot choose the theme: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// The review of pull request 256: a credential's or an account's name in the defaults file is never managed.
|
||||
func TestAnAccountsNameInTheDefaultsIsNeverManaged(t *testing.T) {
|
||||
f := newFake(t)
|
||||
f.betterbird()
|
||||
f.placedByTheMesh()
|
||||
links(t, nil)
|
||||
f.write(meshDefaults, string(repoFile(t, "install/defaults-pref/mesh-defaults.js"))+
|
||||
`pref("mail.identity.id1.useremail", "x");`+"\n"+"// keeps Betterbird's default: mail.server.server1.hostname\n"+
|
||||
`pref("mail.smtpservers", "");`+"\n")
|
||||
prefs := `user_pref("mail.identity.id1.useremail", "someone@example.org");` + "\n" +
|
||||
`user_pref("mail.server.server1.hostname", "imap.example.org");` + "\n" +
|
||||
`user_pref("mail.smtpservers", "smtp1");` + "\n"
|
||||
f.write(profileDir+"/prefs.js", prefs)
|
||||
c, err := f.Config(nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(c.Refused) != 3 || len(c.Drifting) != 0 {
|
||||
t.Fatalf("refused, never managed: %+v", c)
|
||||
}
|
||||
raw, _ := json.Marshal(c)
|
||||
if strings.Contains(string(raw), "example.org") {
|
||||
t.Fatalf("no account value is answered: %s", raw)
|
||||
}
|
||||
stopBetterbird(t, f)
|
||||
if _, err := f.ConfigApply(true); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if f.read(profileDir+"/prefs.js") != prefs {
|
||||
t.Fatal("prefs.js keeps every account's value")
|
||||
}
|
||||
ch, _ := f.Check()
|
||||
found := false
|
||||
for _, x := range ch.Findings {
|
||||
found = found || (strings.Contains(x.What, "3 preference(s) the tools never manage") && strings.Contains(x.Do, "mesh-defaults.js"))
|
||||
}
|
||||
if !found {
|
||||
t.Fatalf("the check says so: %+v", ch.Findings)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,471 @@
|
||||
package main
|
||||
|
||||
// Betterbird's whole configuration, beside the accounts (novox/hq ADR 0329 §4–§6, issue 586).
|
||||
//
|
||||
// Five settings, each mesh-wide with a node's layer over it, reach the machine as two files the node-engine
|
||||
// merges from the module's layers key by key:
|
||||
//
|
||||
// - ~/.config/betterbird/configuration.json: identities, calendars, address-books and filters, trusted as
|
||||
// accounts is (§5), because each names where a password, a reply or forwarded mail goes;
|
||||
// - ~/.config/betterbird/tags.json: tags, which names no server and sends no mail, and is not trusted.
|
||||
//
|
||||
// An entry is keyed by a name of the operator's (lower-case letters and digits, as an account's), a tag by its
|
||||
// own key. A node's layer may set an entry to null to take it out on that node, and may give an identity, a
|
||||
// calendar or an address book its `key`: the profile's key to adopt there (§4, §7). No value holds a password,
|
||||
// a token or a cookie: a key shaped like one is refused whole, and so is an address whose user part carries a
|
||||
// password (§2). A setting holds no line break (novox/hq issue 339), so a signature is a list of its lines.
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"os"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
)
|
||||
|
||||
const (
|
||||
// configurationFile holds identities, calendars, address-books and filters; tagsFile holds tags.
|
||||
configurationFile = ".config/betterbird/configuration.json"
|
||||
tagsFile = ".config/betterbird/tags.json"
|
||||
)
|
||||
|
||||
// The five kinds, in the order the tools answer them.
|
||||
const (
|
||||
kindIdentities = "identities"
|
||||
kindCalendars = "calendars"
|
||||
kindAddressBooks = "address-books"
|
||||
kindFilters = "filters"
|
||||
kindTags = "tags"
|
||||
)
|
||||
|
||||
var allKinds = []string{kindIdentities, kindCalendars, kindAddressBooks, kindFilters, kindTags}
|
||||
|
||||
// Identity is one identity of a declared account. The account's default identity is the entry that gives no
|
||||
// address: its name and address are the ones the setting accounts gives (§4). Any other entry is another
|
||||
// identity of that account, matched by its address.
|
||||
type Identity struct {
|
||||
Account string `json:"account"`
|
||||
Address string `json:"address,omitempty"`
|
||||
Name string `json:"name,omitempty"`
|
||||
ReplyTo string `json:"reply-to,omitempty"`
|
||||
Organization string `json:"organization,omitempty"`
|
||||
// Signature is the signature's text, a line to an element.
|
||||
Signature []string `json:"signature,omitempty"`
|
||||
SignatureHTML bool `json:"signature-html,omitempty"`
|
||||
// SignatureAboveQuote puts the reply and its signature above the quoted message (Betterbird's "start my
|
||||
// reply above the quote", signature below the reply); false is below the quote.
|
||||
SignatureAboveQuote bool `json:"signature-above-quote,omitempty"`
|
||||
Key string `json:"key,omitempty"`
|
||||
}
|
||||
|
||||
// Calendar is one CalDAV calendar.
|
||||
type Calendar struct {
|
||||
Name string `json:"name"`
|
||||
URL string `json:"url"`
|
||||
Color string `json:"color,omitempty"`
|
||||
ReadOnly bool `json:"read-only,omitempty"`
|
||||
// Reminders absent is on.
|
||||
Reminders *bool `json:"reminders,omitempty"`
|
||||
User string `json:"user,omitempty"`
|
||||
// Invitations is the identities entry whose address answers invitations; absent, none.
|
||||
Invitations string `json:"invitations,omitempty"`
|
||||
Key string `json:"key,omitempty"`
|
||||
}
|
||||
|
||||
// AddressBook is one CardDAV address book.
|
||||
type AddressBook struct {
|
||||
Name string `json:"name"`
|
||||
URL string `json:"url"`
|
||||
User string `json:"user,omitempty"`
|
||||
Key string `json:"key,omitempty"`
|
||||
}
|
||||
|
||||
// Tag is one tag, keyed in the setting by its own key.
|
||||
type Tag struct {
|
||||
Name string `json:"name"`
|
||||
Color string `json:"color,omitempty"`
|
||||
// Order is its place among the tags, from 1.
|
||||
Order int `json:"order,omitempty"`
|
||||
}
|
||||
|
||||
// Filter is one message filter of an account, in its place in the account's list.
|
||||
type Filter struct {
|
||||
Name string `json:"name"`
|
||||
Enabled bool `json:"enabled"`
|
||||
// When it runs: new-mail, manual, after-junk, sending, archiving, periodic.
|
||||
When []string `json:"when"`
|
||||
// Match is all (every condition holds), any (one does) or every-message (no condition).
|
||||
Match string `json:"match"`
|
||||
Conditions []Condition `json:"conditions,omitempty"`
|
||||
Actions []Action `json:"actions"`
|
||||
}
|
||||
|
||||
// Condition is one of a filter's conditions, in Betterbird's own words: a field, an operator, a value.
|
||||
type Condition struct {
|
||||
Field string `json:"field"`
|
||||
Op string `json:"op"`
|
||||
Value string `json:"value,omitempty"`
|
||||
}
|
||||
|
||||
// Action is one of a filter's actions, in Betterbird's own words. A folder is named by its path within the
|
||||
// account (INBOX/Receipts), and becomes that folder's address on each node (§8).
|
||||
type Action struct {
|
||||
Action string `json:"action"`
|
||||
Folder string `json:"folder,omitempty"`
|
||||
Value string `json:"value,omitempty"`
|
||||
}
|
||||
|
||||
// Configuration is the five settings as the machine has them.
|
||||
type Configuration struct {
|
||||
Identities map[string]*Identity `json:"identities,omitempty"`
|
||||
Calendars map[string]*Calendar `json:"calendars,omitempty"`
|
||||
AddressBooks map[string]*AddressBook `json:"address-books,omitempty"`
|
||||
Filters map[string][]Filter `json:"filters,omitempty"`
|
||||
Tags map[string]*Tag `json:"tags,omitempty"`
|
||||
}
|
||||
|
||||
var (
|
||||
entryName = regexp.MustCompile(`^[a-z][a-z0-9]{0,30}$`)
|
||||
tagKey = regexp.MustCompile(`^[$A-Za-z0-9_.\-]{1,64}$`)
|
||||
colorWord = regexp.MustCompile(`^#[0-9A-Fa-f]{6}$`)
|
||||
// secretName is a field's name that says it holds a credential: such a field is withheld from every answer (§2).
|
||||
secretName = regexp.MustCompile(`(?i)(password|passwd|token|secret|cookie)`)
|
||||
// userPassword is an address whose user part carries a password: scheme://user:password@host.
|
||||
userPassword = regexp.MustCompile(`(?i)\b([a-z][a-z0-9+.\-]*://[^/:@\s]*):[^@/\s]+@`)
|
||||
)
|
||||
|
||||
// The words a filter is written in.
|
||||
var (
|
||||
filterWhen = map[string]int{"new-mail": 0x1, "manual": 0x10, "after-junk": 0x20, "sending": 0x40,
|
||||
"archiving": 0x80, "periodic": 0x100}
|
||||
filterFields = map[string]bool{"subject": true, "from": true, "to": true, "cc": true, "to or cc": true,
|
||||
"all addresses": true, "body": true, "date": true, "priority": true, "status": true, "age in days": true,
|
||||
"size": true, "tag": true, "junk status": true, "junk percent": true, "junk score origin": true,
|
||||
"has attachment status": true, "date received": true}
|
||||
filterOps = map[string]bool{"contains": true, "doesn't contain": true, "is": true, "isn't": true,
|
||||
"begins with": true, "ends with": true, "is empty": true, "isn't empty": true, "is before": true,
|
||||
"is after": true, "is greater than": true, "is less than": true, "is higher than": true,
|
||||
"is lower than": true, "is in ab": true, "isn't in ab": true, "matches": true, "doesn't match": true}
|
||||
// filterActions are the actions a declared filter may take, and whether each names a folder or a value.
|
||||
filterActions = map[string]string{"Move to folder": "folder", "Copy to folder": "folder",
|
||||
"Change priority": "value", "Delete": "", "Mark read": "", "Mark unread": "", "Mark flagged": "",
|
||||
"Ignore thread": "", "Ignore subthread": "", "Watch thread": "", "AddTag": "value", "JunkScore": "value",
|
||||
"Forward": "value", "Stop execution": "", "Delete from Pop3 server": "", "Leave on Pop3 server": "",
|
||||
"Fetch body from Pop3Server": ""}
|
||||
)
|
||||
|
||||
// readConfiguration reads the five settings. A missing file holds none of its kinds (the module is older
|
||||
// here); a file that carries a key shaped like a credential, an address with a password, an unknown field or a
|
||||
// value Betterbird cannot be given is refused whole, naming the entry and never a value.
|
||||
func (m *Machine) readConfiguration() (Configuration, error) {
|
||||
var c Configuration
|
||||
var problems []string
|
||||
for _, file := range []string{configurationFile, tagsFile} {
|
||||
raw, err := readBounded(m.home(file))
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
return Configuration{}, err
|
||||
}
|
||||
if err := decodeInto(raw, file, &c); err != nil {
|
||||
problems = append(problems, err.Error())
|
||||
}
|
||||
}
|
||||
if len(problems) > 0 {
|
||||
return Configuration{}, errors.New(strings.Join(problems, "; "))
|
||||
}
|
||||
if p := configurationProblems(c); len(p) > 0 {
|
||||
return Configuration{}, fmt.Errorf("the configuration is refused: %s", strings.Join(p, "; "))
|
||||
}
|
||||
c.dropNulls()
|
||||
for _, cal := range c.Calendars {
|
||||
cal.Color = strings.ToLower(cal.Color)
|
||||
}
|
||||
for _, t := range c.Tags {
|
||||
t.Color = strings.ToUpper(t.Color)
|
||||
}
|
||||
return c, nil
|
||||
}
|
||||
|
||||
// decodeInto reads one settings file into c, refusing a credential-shaped key or an unknown field.
|
||||
func decodeInto(raw []byte, file string, c *Configuration) error {
|
||||
var tree any
|
||||
if err := json.Unmarshal(raw, &tree); err != nil {
|
||||
return fmt.Errorf("~/%s is not JSON: %v", file, err)
|
||||
}
|
||||
if where := secretKeyIn(tree, ""); where != "" {
|
||||
return fmt.Errorf("~/%s carries %s, a key shaped like a credential: the configuration is declared without "+
|
||||
"any password, token or cookie (novox/hq ADR 0329 §2); take it out at the controller's terminal", file, where)
|
||||
}
|
||||
dec := json.NewDecoder(bytes.NewReader(raw))
|
||||
dec.DisallowUnknownFields()
|
||||
var into Configuration
|
||||
if err := dec.Decode(&into); err != nil {
|
||||
return fmt.Errorf("~/%s does not hold the shape of the settings it takes: %v", file, err)
|
||||
}
|
||||
if file == tagsFile {
|
||||
if into.Identities != nil || into.Calendars != nil || into.AddressBooks != nil || into.Filters != nil {
|
||||
return fmt.Errorf("~/%s holds only tags", file)
|
||||
}
|
||||
c.Tags = into.Tags
|
||||
return nil
|
||||
}
|
||||
if into.Tags != nil {
|
||||
return fmt.Errorf("~/%s holds no tags: tags are their own setting", file)
|
||||
}
|
||||
c.Identities, c.Calendars, c.AddressBooks, c.Filters = into.Identities, into.Calendars, into.AddressBooks, into.Filters
|
||||
return nil
|
||||
}
|
||||
|
||||
// dropNulls takes out the entries a node's layer set to null: not on this node.
|
||||
func (c *Configuration) dropNulls() {
|
||||
for k, v := range c.Identities {
|
||||
if v == nil {
|
||||
delete(c.Identities, k)
|
||||
}
|
||||
}
|
||||
for k, v := range c.Calendars {
|
||||
if v == nil {
|
||||
delete(c.Calendars, k)
|
||||
}
|
||||
}
|
||||
for k, v := range c.AddressBooks {
|
||||
if v == nil {
|
||||
delete(c.AddressBooks, k)
|
||||
}
|
||||
}
|
||||
for k, v := range c.Filters {
|
||||
if v == nil {
|
||||
delete(c.Filters, k)
|
||||
}
|
||||
}
|
||||
for k, v := range c.Tags {
|
||||
if v == nil {
|
||||
delete(c.Tags, k)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// addressPassword says whether an address carries a password in its user part.
|
||||
func addressPassword(s string) bool {
|
||||
if userPassword.MatchString(s) {
|
||||
return true
|
||||
}
|
||||
if u, err := url.Parse(s); err == nil && u.User != nil {
|
||||
_, has := u.User.Password()
|
||||
return has
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// withoutPassword is an address with the password of its user part as ‹withheld› (§2).
|
||||
func withoutPassword(s string) string {
|
||||
return userPassword.ReplaceAllString(s, "${1}:‹withheld›@")
|
||||
}
|
||||
|
||||
// davURL says what is wrong with a calendar's or an address book's address, or "".
|
||||
func davURL(s string) string {
|
||||
if addressPassword(s) {
|
||||
return "its address carries a password in its user part, which no setting holds (novox/hq ADR 0329 §2): " +
|
||||
"Betterbird asks for the password at the first sign-in"
|
||||
}
|
||||
u, err := url.Parse(s)
|
||||
if err != nil || (u.Scheme != "https" && u.Scheme != "http") || u.Host == "" {
|
||||
return "its address is an https:// (or http://) address of the server"
|
||||
}
|
||||
if strings.ContainsAny(s, lineEnds+`"\`) {
|
||||
return "its address holds a quote, a backslash or a line break"
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// configurationProblems is everything wrong with the five settings, each said by its kind and entry name.
|
||||
func configurationProblems(c Configuration) []string {
|
||||
var out []string
|
||||
say := func(kind, name, format string, args ...any) {
|
||||
out = append(out, fmt.Sprintf("%s %s: ", kind, name)+fmt.Sprintf(format, args...))
|
||||
}
|
||||
text := func(kind, name, field, v string) {
|
||||
if !plainText.MatchString(v) {
|
||||
say(kind, name, "%s holds a quote, a backslash or a line break", field)
|
||||
}
|
||||
}
|
||||
for _, name := range sortedKeys(c.Identities) {
|
||||
id := c.Identities[name]
|
||||
if id == nil {
|
||||
continue
|
||||
}
|
||||
if !entryName.MatchString(name) {
|
||||
say(kindIdentities, name, "a name is lower-case letters and digits, starting with a letter")
|
||||
}
|
||||
if !accountName.MatchString(id.Account) {
|
||||
say(kindIdentities, name, "account names an entry of the setting accounts")
|
||||
}
|
||||
if id.Address == "" && id.Name != "" {
|
||||
say(kindIdentities, name, "the account's default identity (no address) takes its name from the setting accounts")
|
||||
}
|
||||
if id.Address != "" && !addressShape.MatchString(id.Address) {
|
||||
say(kindIdentities, name, "the address is not a mail address")
|
||||
}
|
||||
if id.ReplyTo != "" && !addressShape.MatchString(id.ReplyTo) {
|
||||
say(kindIdentities, name, "reply-to is not a mail address")
|
||||
}
|
||||
for field, v := range map[string]string{"name": id.Name, "organization": id.Organization} {
|
||||
text(kindIdentities, name, field, v)
|
||||
}
|
||||
for _, line := range id.Signature {
|
||||
if strings.ContainsAny(line, lineEnds) {
|
||||
say(kindIdentities, name, "a signature's line holds a line break: each line is its own element")
|
||||
break
|
||||
}
|
||||
}
|
||||
if id.Key != "" && !profileKey.MatchString(id.Key) {
|
||||
say(kindIdentities, name, "key is an identity key of the profile, such as id3")
|
||||
}
|
||||
}
|
||||
for _, name := range sortedKeys(c.Calendars) {
|
||||
cal := c.Calendars[name]
|
||||
if cal == nil {
|
||||
continue
|
||||
}
|
||||
if !entryName.MatchString(name) {
|
||||
say(kindCalendars, name, "a name is lower-case letters and digits, starting with a letter")
|
||||
}
|
||||
if why := davURL(cal.URL); why != "" {
|
||||
say(kindCalendars, name, "%s", why)
|
||||
}
|
||||
if cal.Color != "" && !colorWord.MatchString(cal.Color) {
|
||||
say(kindCalendars, name, "a colour is #rrggbb")
|
||||
}
|
||||
if cal.Invitations != "" && c.Identities[cal.Invitations] == nil {
|
||||
say(kindCalendars, name, "invitations names %s, which the setting identities does not hold", cal.Invitations)
|
||||
}
|
||||
text(kindCalendars, name, "name", cal.Name)
|
||||
text(kindCalendars, name, "user", cal.User)
|
||||
if cal.Key != "" && !tagKey.MatchString(cal.Key) {
|
||||
say(kindCalendars, name, "key is a calendar key of the profile")
|
||||
}
|
||||
}
|
||||
for _, name := range sortedKeys(c.AddressBooks) {
|
||||
ab := c.AddressBooks[name]
|
||||
if ab == nil {
|
||||
continue
|
||||
}
|
||||
if !entryName.MatchString(name) {
|
||||
say(kindAddressBooks, name, "a name is lower-case letters and digits, starting with a letter")
|
||||
}
|
||||
if why := davURL(ab.URL); why != "" {
|
||||
say(kindAddressBooks, name, "%s", why)
|
||||
}
|
||||
text(kindAddressBooks, name, "name", ab.Name)
|
||||
text(kindAddressBooks, name, "user", ab.User)
|
||||
if ab.Key != "" && !profileKey.MatchString(ab.Key) {
|
||||
say(kindAddressBooks, name, "key is an address book key of the profile")
|
||||
}
|
||||
}
|
||||
for _, key := range sortedKeys(c.Tags) {
|
||||
t := c.Tags[key]
|
||||
if t == nil {
|
||||
continue
|
||||
}
|
||||
if !tagKey.MatchString(key) {
|
||||
say(kindTags, key, "a tag's key is letters, digits, $, _, . or -")
|
||||
}
|
||||
if t.Color != "" && !colorWord.MatchString(t.Color) {
|
||||
say(kindTags, key, "a colour is #rrggbb")
|
||||
}
|
||||
text(kindTags, key, "name", t.Name)
|
||||
}
|
||||
for _, account := range sortedKeys(c.Filters) {
|
||||
if !accountName.MatchString(account) {
|
||||
say(kindFilters, account, "filters are keyed by an account's name in the setting accounts")
|
||||
}
|
||||
for i, f := range c.Filters[account] {
|
||||
for _, p := range filterProblems(f) {
|
||||
say(kindFilters, account, "filter %d: %s", i+1, p)
|
||||
}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// filterValue is what a filter's name, value or folder may hold: what its file can carry without quoting.
|
||||
var filterValue = regexp.MustCompile(`^[^"\\()\n\r]*$`)
|
||||
|
||||
func filterProblems(f Filter) []string {
|
||||
var out []string
|
||||
if f.Name == "" || !filterValue.MatchString(f.Name) {
|
||||
out = append(out, "a name, without a quote, a backslash or a parenthesis")
|
||||
}
|
||||
if len(f.When) == 0 {
|
||||
out = append(out, "when says when it runs: new-mail, manual, after-junk, sending, archiving or periodic")
|
||||
}
|
||||
for _, w := range f.When {
|
||||
if _, ok := filterWhen[w]; !ok {
|
||||
out = append(out, fmt.Sprintf("when %q is new-mail, manual, after-junk, sending, archiving or periodic", w))
|
||||
}
|
||||
}
|
||||
switch f.Match {
|
||||
case "all", "any":
|
||||
if len(f.Conditions) == 0 {
|
||||
out = append(out, "match all or any needs a condition")
|
||||
}
|
||||
case "every-message":
|
||||
if len(f.Conditions) > 0 {
|
||||
out = append(out, "match every-message takes no condition")
|
||||
}
|
||||
default:
|
||||
out = append(out, "match is all, any or every-message")
|
||||
}
|
||||
for i, c := range f.Conditions {
|
||||
if !filterFields[c.Field] && !(strings.HasPrefix(c.Field, `"`) && strings.HasSuffix(c.Field, `"`) && len(c.Field) > 2) {
|
||||
out = append(out, fmt.Sprintf("condition %d: the field %q is not one Betterbird filters on", i+1, c.Field))
|
||||
}
|
||||
if !filterOps[c.Op] {
|
||||
out = append(out, fmt.Sprintf("condition %d: the operator %q is not one of Betterbird's", i+1, c.Op))
|
||||
}
|
||||
if !filterValue.MatchString(c.Value) {
|
||||
out = append(out, fmt.Sprintf("condition %d: the value holds a quote, a backslash, a parenthesis or a line break", i+1))
|
||||
}
|
||||
}
|
||||
if len(f.Actions) == 0 {
|
||||
out = append(out, "a filter takes an action")
|
||||
}
|
||||
for i, a := range f.Actions {
|
||||
takes, ok := filterActions[a.Action]
|
||||
switch {
|
||||
case !ok:
|
||||
out = append(out, fmt.Sprintf("action %d: %q is not an action a declared filter takes", i+1, a.Action))
|
||||
case takes == "folder" && (a.Folder == "" || a.Value != ""):
|
||||
out = append(out, fmt.Sprintf("action %d: %s names a folder, by its path within the account", i+1, a.Action))
|
||||
case takes == "value" && (a.Value == "" || a.Folder != ""):
|
||||
out = append(out, fmt.Sprintf("action %d: %s takes a value", i+1, a.Action))
|
||||
case takes == "" && (a.Value != "" || a.Folder != ""):
|
||||
out = append(out, fmt.Sprintf("action %d: %s takes no value", i+1, a.Action))
|
||||
}
|
||||
if !filterValue.MatchString(a.Value) || !filterValue.MatchString(a.Folder) || strings.HasPrefix(a.Folder, "/") ||
|
||||
strings.Contains(a.Folder, "://") {
|
||||
out = append(out, fmt.Sprintf("action %d: a folder is a path within the account, and a value holds no quote, "+
|
||||
"backslash, parenthesis or line break", i+1))
|
||||
}
|
||||
if a.Action == "Forward" && !addressShape.MatchString(a.Value) {
|
||||
out = append(out, fmt.Sprintf("action %d: Forward names a mail address", i+1))
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// sortedFilterAccounts are the accounts filters are declared for.
|
||||
func (c Configuration) sortedFilterAccounts() []string {
|
||||
out := make([]string, 0, len(c.Filters))
|
||||
for a := range c.Filters {
|
||||
out = append(out, a)
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,518 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// Betterbird's whole configuration (novox/hq ADR 0329, issue 586): identities, calendars, address books, filters
|
||||
// and tags declared mesh-wide, adopted or created as accounts are, filters owned whole per declared account, what
|
||||
// Betterbird rewrites written only while it is closed, derived from the nodes' profiles one layer at a time, and
|
||||
// shown with the operator's own values but never a credential.
|
||||
|
||||
// profileWithConfiguration adds to profileWithAccounts' profile: account1's directory and signature, CalDAV
|
||||
// calendars (one address twice), an ICS and an Owl calendar, a local and a CardDAV address book, two tags, and
|
||||
// account1's filter file holding one filter of the person's.
|
||||
func (f *fake) profileWithConfiguration() {
|
||||
f.t.Helper()
|
||||
prefs := f.read(profileDir+"/prefs.js") +
|
||||
`user_pref("mail.server.server1.directory-rel", "[ProfD]ImapMail/imap.example.org");` + "\n" +
|
||||
`user_pref("mail.server.server2.directory-rel", "[ProfD]Mail/Local Folders");` + "\n" +
|
||||
`user_pref("mail.identity.id1.htmlSigText", "Regards");` + "\n" +
|
||||
`user_pref("calendar.registry.c1.type", "caldav");` + "\n" +
|
||||
`user_pref("calendar.registry.c1.uri", "https://dav.example.org/cal/home/");` + "\n" +
|
||||
`user_pref("calendar.registry.c1.name", "Home");` + "\n" +
|
||||
`user_pref("calendar.registry.c1.color", "#3366ff");` + "\n" +
|
||||
`user_pref("calendar.registry.c2.type", "caldav");` + "\n" +
|
||||
`user_pref("calendar.registry.c2.uri", "https://dav.example.org/cal/shared/");` + "\n" +
|
||||
`user_pref("calendar.registry.c2.name", "Shared A");` + "\n" +
|
||||
`user_pref("calendar.registry.c3.type", "caldav");` + "\n" +
|
||||
`user_pref("calendar.registry.c3.uri", "https://dav.example.org/cal/shared/");` + "\n" +
|
||||
`user_pref("calendar.registry.c3.name", "Shared B");` + "\n" +
|
||||
`user_pref("calendar.registry.c4.type", "ics");` + "\n" +
|
||||
`user_pref("calendar.registry.c4.uri", "https://feeds.example.org/secret-token-abc.ics");` + "\n" +
|
||||
`user_pref("calendar.registry.c4.name", "Holidays");` + "\n" +
|
||||
`user_pref("calendar.registry.c5.type", "owl");` + "\n" +
|
||||
`user_pref("calendar.registry.c5.uri", "owl://work@corp.example.com/calendar");` + "\n" +
|
||||
`user_pref("calendar.registry.c5.name", "Work");` + "\n" +
|
||||
`user_pref("calendar.list.sortOrder", "c1 c2 c3 c4 c5");` + "\n" +
|
||||
`user_pref("ldap_2.servers.pab.description", "Personal Address Book");` + "\n" +
|
||||
`user_pref("ldap_2.servers.pab.dirType", 101);` + "\n" +
|
||||
`user_pref("ldap_2.servers.ab1.description", "Contacts");` + "\n" +
|
||||
`user_pref("ldap_2.servers.ab1.dirType", 102);` + "\n" +
|
||||
`user_pref("ldap_2.servers.ab1.carddav.url", "https://dav.example.org/card/me/");` + "\n" +
|
||||
`user_pref("ldap_2.servers.ab1.carddav.username", "me");` + "\n" +
|
||||
`user_pref("mailnews.tags.$label1.tag", "Important");` + "\n" +
|
||||
`user_pref("mailnews.tags.$label1.color", "#FF0000");` + "\n" +
|
||||
`user_pref("mailnews.tags.personal.tag", "Personal");` + "\n" +
|
||||
`user_pref("mailnews.tags.personal.color", "#00AA00");` + "\n"
|
||||
f.write(profileDir+"/prefs.js", prefs)
|
||||
f.write(profileDir+"/ImapMail/imap.example.org/"+filterFileName, "version=\"9\"\nlogging=\"no\"\nname=\"Old\"\n"+
|
||||
"enabled=\"yes\"\ntype=\"17\"\naction=\"Mark read\"\ncondition=\"AND (from,contains,news)\"\n")
|
||||
f.write(profileDir+"/Mail/Local Folders/"+filterFileName, "version=\"9\"\nlogging=\"no\"\n")
|
||||
f.write(profileDir+"/calendar-data/cache.sqlite", "events")
|
||||
}
|
||||
|
||||
const declaredConfiguration = `{
|
||||
"identities": {
|
||||
"mail": {"account": "mail", "signature": ["-- ", "Other"], "signature-above-quote": true},
|
||||
"mailalias": {"account": "mail", "address": "alias@mail.example.org", "name": "Alias", "reply-to": "other@mail.example.org"}
|
||||
},
|
||||
"calendars": {
|
||||
"home": {"name": "Home", "url": "https://dav.example.org/cal/home", "color": "#3366CC", "user": "other", "invitations": "mail"},
|
||||
"work": {"name": "Team", "url": "https://dav.example.org/cal/team/"},
|
||||
"shared": {"name": "Shared", "url": "https://dav.example.org/cal/shared/"}
|
||||
},
|
||||
"address-books": {
|
||||
"contacts": {"name": "Contacts", "url": "https://dav.example.org/card/me/", "user": "other"},
|
||||
"team": {"name": "Team", "url": "https://dav.example.org/card/team/"}
|
||||
},
|
||||
"filters": {}
|
||||
}`
|
||||
|
||||
const declaredTags = `{"tags": {"$label1": {"name": "Important", "color": "#ff0000", "order": 1}, "todo": {"name": "To do", "color": "#0000ff", "order": 2}}}`
|
||||
|
||||
func (f *fake) declareConfiguration(body, tags string) {
|
||||
f.write(testHome+"/"+configurationFile, body)
|
||||
f.write(testHome+"/"+tagsFile, tags)
|
||||
}
|
||||
|
||||
func entriesBy(u UserJS) map[string]resolvedEntry {
|
||||
out := map[string]resolvedEntry{}
|
||||
for _, e := range u.Configuration {
|
||||
out[e.Kind+"/"+e.Name] = e
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// §2, §4, §6: the settings' shapes; a credential, an address with a password, a key on a tag and the four new
|
||||
// prefixes in prefs are refused, said without the value.
|
||||
func TestTheSettingsRefuseACredentialAPasswordedAddressATagsKeyAndTheirPrefixesInPrefs(t *testing.T) {
|
||||
f := newFake(t)
|
||||
f.declareConfiguration(declaredConfiguration, declaredTags)
|
||||
c, err := f.readConfiguration()
|
||||
if err != nil || len(c.Identities) != 2 || len(c.Calendars) != 3 || len(c.AddressBooks) != 2 || len(c.Tags) != 2 {
|
||||
t.Fatalf("the declared shapes are taken: %+v %v", c, err)
|
||||
}
|
||||
if c.Calendars["home"].Color != "#3366cc" || c.Tags["todo"].Color != "#0000FF" {
|
||||
t.Errorf("colours are said as Betterbird keeps them: %+v %+v", c.Calendars["home"], c.Tags["todo"])
|
||||
}
|
||||
for _, bad := range []struct{ conf, tags, says string }{
|
||||
{`{"calendars": {"home": {"name": "H", "url": "https://dav.example.org/c/", "password": "hunter2"}}}`, `{}`, "shaped like a credential"},
|
||||
{`{"calendars": {"home": {"name": "H", "url": "https://me:hunter2@dav.example.org/c/"}}}`, `{}`, "carries a password"},
|
||||
{`{"address-books": {"b": {"name": "B", "url": "https://me:hunter2@dav.example.org/b/"}}}`, `{}`, "carries a password"},
|
||||
{`{}`, `{"tags": {"todo": {"name": "To do", "key": "x"}}}`, "does not hold the shape"},
|
||||
{`{"tags": {"todo": {"name": "To do"}}}`, `{}`, "holds no tags"},
|
||||
{`{"identities": {"x": {"account": "mail", "name": "Me"}}}`, `{}`, "default identity"},
|
||||
{`{"filters": {"mail": [{"name": "F", "enabled": true, "when": ["manual"], "match": "all", "actions": [{"action": "Delete"}]}]}}`, `{}`, "needs a condition"},
|
||||
{`{"filters": {"mail": [{"name": "F", "enabled": true, "when": ["manual"], "match": "every-message", "actions": [{"action": "Move to folder", "folder": "imap://x@y/INBOX"}]}]}}`, `{}`, "path within the account"},
|
||||
} {
|
||||
f.declareConfiguration(bad.conf, bad.tags)
|
||||
_, err := f.readConfiguration()
|
||||
if err == nil || !strings.Contains(err.Error(), bad.says) || strings.Contains(err.Error(), "hunter2") {
|
||||
t.Errorf("%s %s: %v", bad.conf, bad.tags, err)
|
||||
}
|
||||
}
|
||||
// A key is an identity's, a calendar's or an address book's.
|
||||
f.declareConfiguration(`{"identities": {"x": {"account": "mail", "address": "x@mail.example.org", "key": "id9"}},
|
||||
"calendars": {"c": {"name": "C", "url": "https://dav.example.org/c/", "key": "c9"}},
|
||||
"address-books": {"b": {"name": "B", "url": "https://dav.example.org/b/", "key": "ab9"}}}`, `{}`)
|
||||
if _, err := f.readConfiguration(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// prefs refuses the four prefixes the new settings own, and the account's and a credential's as before.
|
||||
for _, name := range []string{"calendar.registry.c1.color", "calendar.list.sortOrder", "ldap_2.servers.ab1.description",
|
||||
"mailnews.tags.$label1.color", "mail.identity.id1.htmlSigText", "signon.rememberSignons"} {
|
||||
body, _ := json.Marshal(map[string]any{"prefs": map[string]any{name: "x"}})
|
||||
if _, err := parseDeclaredPrefs(body); err == nil || !strings.Contains(err.Error(), name) {
|
||||
t.Errorf("%s: %v", name, err)
|
||||
}
|
||||
}
|
||||
got, err := parseDeclaredPrefs([]byte(`{"prefs": {"mail.uifontsize": 14, "mail.chat.enabled": false, "calendar.alarms.show": null,
|
||||
"intl.date_time.pattern_override.date_short": "yyyy-MM-dd"}}`))
|
||||
if err != nil || !reflect.DeepEqual(got, map[string]string{"mail.uifontsize": "14", "mail.chat.enabled": "false",
|
||||
"intl.date_time.pattern_override.date_short": `"yyyy-MM-dd"`}) {
|
||||
t.Fatalf("one entry per preference, null taking one out: %v %v", got, err)
|
||||
}
|
||||
if _, err := parseDeclaredPrefs([]byte(`{"prefs": {"mail.uifontsize": 14.5}}`)); err == nil {
|
||||
t.Fatal("a number Betterbird does not keep is refused")
|
||||
}
|
||||
}
|
||||
|
||||
// §7, §9: each kind is adopted, created or refused as accounts are; the declared fields go into the user.js, and
|
||||
// what Betterbird rewrites waits for it to close; leaving the setting releases or removes, never events.
|
||||
func TestEachKindIsAdoptedOrCreatedAndWhatBetterbirdRewritesWaitsForItToClose(t *testing.T) {
|
||||
f := newFake(t)
|
||||
f.profileWithAccounts()
|
||||
f.profileWithConfiguration()
|
||||
f.noPackage()
|
||||
links(t, nil)
|
||||
f.declare("// none")
|
||||
f.declareAccounts(adoptable)
|
||||
f.declareConfiguration(declaredConfiguration, declaredTags)
|
||||
before := f.read(profileDir + "/prefs.js")
|
||||
|
||||
// Betterbird runs: the user.js is placed, prefs.js and the filter files wait (§9).
|
||||
u, err := f.userJS(true)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
by := entriesBy(u)
|
||||
for want, how := range map[string][2]string{
|
||||
"identities/mail": {adopted, "id1"}, "identities/mailalias": {created, "ident_mesh_mailalias"},
|
||||
"calendars/home": {adopted, "c1"}, "calendars/work": {created, "mesh_work"}, "calendars/shared": {refused, ""},
|
||||
"address-books/contacts": {adopted, "ab1"}, "address-books/team": {created, "mesh_team"},
|
||||
"tags/$label1": {adopted, "$label1"}, "tags/todo": {created, "todo"},
|
||||
} {
|
||||
if e := by[want]; e.How != how[0] || e.Key != how[1] {
|
||||
t.Errorf("%s: %+v, want %v", want, e, how)
|
||||
}
|
||||
}
|
||||
if why := by["calendars/shared"].Why; !strings.Contains(why, "2 entries") || !strings.Contains(why, "c2, c3") ||
|
||||
!strings.Contains(why, "key in this node's layer") {
|
||||
t.Errorf("an ambiguous match is refused with the keys and the remedy: %s", why)
|
||||
}
|
||||
placed := f.read(profileDir + "/user.js")
|
||||
for _, want := range []string{
|
||||
`user_pref("mail.identity.id1.htmlSigText", "-- \nOther");`,
|
||||
`user_pref("mail.identity.id1.reply_on_top", 1);`,
|
||||
`user_pref("mail.identity.id1.sig_bottom", false);`,
|
||||
`user_pref("mail.identity.ident_mesh_mailalias.useremail", "alias@mail.example.org");`,
|
||||
`user_pref("mail.identity.ident_mesh_mailalias.smtpServer", "smtp1");`,
|
||||
`user_pref("calendar.registry.c1.color", "#3366cc");`,
|
||||
`user_pref("calendar.registry.c1.imip.identity.key", "id1");`,
|
||||
`user_pref("calendar.registry.mesh_work.type", "caldav");`,
|
||||
`user_pref("ldap_2.servers.ab1.carddav.username", "other");`,
|
||||
`user_pref("ldap_2.servers.mesh_team.dirType", 102);`,
|
||||
`user_pref("mailnews.tags.todo.color", "#0000FF");`,
|
||||
`user_pref("mailnews.tags.todo.ordinal", "1002");`,
|
||||
} {
|
||||
if !strings.Contains(placed, want) {
|
||||
t.Errorf("the user.js lacks %s", want)
|
||||
}
|
||||
}
|
||||
for _, never := range []string{"calendar.registry.c2.", "calendar.registry.c3.", "calendar.registry.c4.", "calendar.registry.c5.",
|
||||
"ldap_2.servers.pab.", "mailnews.tags.personal.", "calendar.list.sortOrder"} {
|
||||
if strings.Contains(placed, never) {
|
||||
t.Errorf("the user.js writes %s, which nothing declares or which is refused", never)
|
||||
}
|
||||
}
|
||||
if !strings.HasPrefix(u.Closed, "not written") || !strings.Contains(u.Closed, "running") ||
|
||||
u.Waiting[kindIdentities] != 1 || u.Waiting[kindCalendars] != 1 {
|
||||
t.Fatalf("while Betterbird runs prefs.js is not written, and the waiting changes are counted: %q %v", u.Closed, u.Waiting)
|
||||
}
|
||||
if f.read(profileDir+"/prefs.js") != before {
|
||||
t.Fatal("prefs.js changed while Betterbird runs")
|
||||
}
|
||||
ch, _ := f.Check()
|
||||
waits := false
|
||||
for _, x := range ch.Findings {
|
||||
waits = waits || (strings.Contains(x.What, "change(s) to calendars are not written yet") && strings.Contains(x.Do, "betterbird_restart"))
|
||||
}
|
||||
if !waits {
|
||||
t.Fatalf("the check counts the waiting changes per kind: %+v", ch.Findings)
|
||||
}
|
||||
|
||||
// A key in the node's layer settles the ambiguous calendar; one the profile does not hold is refused.
|
||||
f.declareConfiguration(strings.Replace(declaredConfiguration, `"url": "https://dav.example.org/cal/shared/"}`,
|
||||
`"url": "https://dav.example.org/cal/shared/", "key": "c3"}`, 1), declaredTags)
|
||||
if u, _ = f.userJS(false); entriesBy(u)["calendars/shared"].Key != "c3" || entriesBy(u)["calendars/shared"].How != adopted {
|
||||
t.Fatalf("the named key is adopted: %+v", entriesBy(u)["calendars/shared"])
|
||||
}
|
||||
f.declareConfiguration(strings.Replace(declaredConfiguration, `"url": "https://dav.example.org/cal/shared/"}`,
|
||||
`"url": "https://dav.example.org/cal/shared/", "key": "c9"}`, 1), declaredTags)
|
||||
if u, _ = f.userJS(false); entriesBy(u)["calendars/shared"].How != refused || !strings.Contains(entriesBy(u)["calendars/shared"].Why, "c9") {
|
||||
t.Fatalf("a key the profile does not hold is refused: %+v", entriesBy(u)["calendars/shared"])
|
||||
}
|
||||
f.declareConfiguration(strings.Replace(declaredConfiguration, `"url": "https://dav.example.org/cal/shared/"}`,
|
||||
`"url": "https://dav.example.org/cal/shared/", "key": "c1"}`, 1), declaredTags)
|
||||
if u, _ = f.userJS(false); entriesBy(u)["calendars/shared"].How != refused {
|
||||
t.Fatalf("a key whose entry does not match is refused: %+v", entriesBy(u)["calendars/shared"])
|
||||
}
|
||||
f.declareConfiguration(declaredConfiguration, declaredTags)
|
||||
|
||||
// Closed: the lists gain the module's own keys at their end, every other line as it was.
|
||||
stopBetterbird(t, f)
|
||||
f.write(profileDir+"/lock", "")
|
||||
if err := f.Prepare(); err == nil || !strings.Contains(err.Error(), "locked") {
|
||||
t.Fatalf("the profile's lock holds the writes back: %v", err)
|
||||
}
|
||||
os.Remove(filepath.Join(f.Root, profileDir, "lock"))
|
||||
if err := f.Prepare(); err != nil && !strings.Contains(err.Error(), "calendars shared is refused") {
|
||||
t.Fatal(err)
|
||||
}
|
||||
after := f.read(profileDir + "/prefs.js")
|
||||
for _, want := range []string{`user_pref("mail.account.account1.identities", "id1,ident_mesh_mailalias");`,
|
||||
`user_pref("calendar.list.sortOrder", "c1 c2 c3 c4 c5 mesh_work");`} {
|
||||
if !strings.Contains(after, want) {
|
||||
t.Errorf("prefs.js lacks %s:\n%s", want, after)
|
||||
}
|
||||
}
|
||||
bl, al := strings.Split(before, "\n"), strings.Split(after, "\n")
|
||||
for i := range bl {
|
||||
if bl[i] != al[i] && !strings.Contains(bl[i], ".identities") && !strings.Contains(bl[i], "sortOrder") {
|
||||
t.Errorf("line %d changed: %q to %q", i+1, bl[i], al[i])
|
||||
}
|
||||
}
|
||||
|
||||
// Leaving: an adopted entry is released (its lines go, the profile keeps it), a created one's preferences are
|
||||
// removed; the calendar's events stay.
|
||||
f.write(profileDir+"/prefs.js", f.read(profileDir+"/prefs.js")+
|
||||
`user_pref("calendar.registry.mesh_work.calendarSyncState", "x");`+"\n"+`user_pref("mailnews.tags.todo.tag", "To do");`+"\n")
|
||||
f.declareConfiguration(`{"identities": {"mail": {"account": "mail"}}}`, `{"tags": {}}`)
|
||||
if err := f.Prepare(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
final := f.read(profileDir + "/prefs.js")
|
||||
for _, gone := range []string{"mesh_work", "ident_mesh_mailalias", "mailnews.tags.todo."} {
|
||||
if strings.Contains(final, gone) {
|
||||
t.Errorf("what the module created and the setting left is removed: %s is still there", gone)
|
||||
}
|
||||
}
|
||||
for _, kept := range []string{`calendar.registry.c1.uri`, `ldap_2.servers.ab1.carddav.url`, `mailnews.tags.$label1.tag`,
|
||||
`calendar.registry.c4.uri`} {
|
||||
if !strings.Contains(final, kept) {
|
||||
t.Errorf("released or undeclared, %s stays the person's", kept)
|
||||
}
|
||||
}
|
||||
if f.read(profileDir+"/calendar-data/cache.sqlite") != "events" {
|
||||
t.Error("a calendar's events are never touched")
|
||||
}
|
||||
if placed := f.read(profileDir + "/user.js"); strings.Contains(placed, "calendar.registry.c1.") || strings.Contains(placed, "ab1") {
|
||||
t.Errorf("a released entry's lines leave the user.js:\n%s", placed)
|
||||
}
|
||||
}
|
||||
|
||||
const declaredFilters = `{"filters": {"mail": [
|
||||
{"name": "Receipts", "enabled": true, "when": ["new-mail", "manual"], "match": "all",
|
||||
"conditions": [{"field": "subject", "op": "contains", "value": "receipt"}],
|
||||
"actions": [{"action": "Move to folder", "folder": "INBOX/Receipts"}, {"action": "Forward", "value": "archive@mail.example.org"}]},
|
||||
{"name": "Everything", "enabled": false, "when": ["manual"], "match": "every-message", "actions": [{"action": "Mark read"}]}
|
||||
]}}`
|
||||
|
||||
const renderedFilters = "version=\"9\"\nlogging=\"no\"\n" +
|
||||
"name=\"Receipts\"\nenabled=\"yes\"\ntype=\"17\"\naction=\"Move to folder\"\n" +
|
||||
"actionValue=\"imap://other2@imap.example.org/INBOX/Receipts\"\naction=\"Forward\"\nactionValue=\"archive@mail.example.org\"\n" +
|
||||
"condition=\"AND (subject,contains,receipt)\"\n" +
|
||||
"name=\"Everything\"\nenabled=\"no\"\ntype=\"16\"\naction=\"Mark read\"\ncondition=\"ALL\"\n"
|
||||
|
||||
// §8: an account named in filters has its msgFilterRules.dat written whole from the list, the replaced file kept;
|
||||
// an account never named keeps its file byte for byte; one taken out keeps the file it last had.
|
||||
func TestFiltersAreOwnedWholePerDeclaredAccount(t *testing.T) {
|
||||
f := newFake(t)
|
||||
f.profileWithAccounts()
|
||||
f.profileWithConfiguration()
|
||||
f.noPackage()
|
||||
links(t, nil)
|
||||
f.declare("// none")
|
||||
f.declareAccounts(adoptable)
|
||||
f.declareConfiguration(declaredFilters, `{"tags": {}}`)
|
||||
filters := profileDir + "/ImapMail/imap.example.org/" + filterFileName
|
||||
local := profileDir + "/Mail/Local Folders/" + filterFileName
|
||||
old, oldLocal := f.read(filters), f.read(local)
|
||||
|
||||
u, err := f.userJS(true)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if u.Waiting[kindFilters] != 1 || f.read(filters) != old {
|
||||
t.Fatalf("while Betterbird runs the filter file waits: %v", u.Waiting)
|
||||
}
|
||||
stopBetterbird(t, f)
|
||||
if err := f.Prepare(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := f.read(filters); got != renderedFilters {
|
||||
t.Fatalf("the file is the declared list, its folder this node's address:\n%s", got)
|
||||
}
|
||||
keptPath := testHome + "/.config/betterbird/kept/60016jsp.default-default/mail." + filterFileName
|
||||
if f.read(keptPath) != old {
|
||||
t.Fatal("the replaced file is kept in the module's own directory")
|
||||
}
|
||||
if f.read(local) != oldLocal {
|
||||
t.Fatal("an account never named keeps its file byte for byte")
|
||||
}
|
||||
if parsed := parseFilters(renderedFilters, "imap://other2@imap.example.org"); len(parsed.NotDeclarable) != 0 ||
|
||||
mustJSON(parsed.Filters) != mustJSON(mustConfig(t, declaredFilters).Filters["mail"]) {
|
||||
t.Fatalf("the file reads back as the declared list: %+v", parsed)
|
||||
}
|
||||
|
||||
// A filter the person adds in Betterbird is drift first, and gone at the next apply; the kept file stays the first.
|
||||
f.write(filters, renderedFilters+"name=\"Mine\"\nenabled=\"yes\"\ntype=\"16\"\naction=\"Delete\"\ncondition=\"ALL\"\n")
|
||||
d, _ := f.Drift(kindFilters)
|
||||
if len(d.Kinds) != 1 || len(d.Kinds[0].Declared) != 1 || d.Kinds[0].Declared[0].State != "differs" ||
|
||||
!strings.Contains(mustJSON(d.Kinds[0].Declared[0].Differs), "Mine") {
|
||||
t.Fatalf("drift shows the added filter: %+v", d)
|
||||
}
|
||||
if err := f.Prepare(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if f.read(filters) != renderedFilters || f.read(keptPath) != old {
|
||||
t.Fatal("the added filter is gone, and the kept file is still the one first replaced")
|
||||
}
|
||||
|
||||
// Taken out of filters: the module stops writing, the last file stays the person's, the kept copy is named.
|
||||
f.declareConfiguration(`{"filters": {}}`, `{"tags": {}}`)
|
||||
f.write(filters, renderedFilters+"name=\"Later\"\nenabled=\"yes\"\ntype=\"16\"\naction=\"Delete\"\ncondition=\"ALL\"\n")
|
||||
lastHad := f.read(filters)
|
||||
if err := f.Prepare(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if f.read(filters) != lastHad || f.read(keptPath) != old {
|
||||
t.Fatal("an account taken out keeps its file byte for byte, and the kept copy stays")
|
||||
}
|
||||
l, _ := f.List(kindFilters)
|
||||
if l.Kept["mail"] != "~/.config/betterbird/kept/60016jsp.default-default/mail."+filterFileName {
|
||||
t.Fatalf("betterbird_filters says where the kept copy is: %+v", l.Kept)
|
||||
}
|
||||
d, _ = f.Drift(kindFilters)
|
||||
if len(d.Kinds[0].Declared) != 0 || len(d.Kinds[0].Undeclared) != 0 {
|
||||
t.Fatalf("drift no longer counts that account's filters: %+v", d.Kinds[0])
|
||||
}
|
||||
}
|
||||
|
||||
func mustConfig(t *testing.T, body string) Configuration {
|
||||
t.Helper()
|
||||
var c Configuration
|
||||
if err := decodeInto([]byte(body), configurationFile, &c); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return c
|
||||
}
|
||||
|
||||
// §11: per kind and entry, the field that differs with both values; an undeclared entry is listed; once the module's
|
||||
// user.js is placed, what it declares holds.
|
||||
func TestDriftNamesEachDifferingFieldAndTheUndeclaredEntries(t *testing.T) {
|
||||
f := newFake(t)
|
||||
f.profileWithAccounts()
|
||||
f.profileWithConfiguration()
|
||||
f.noPackage()
|
||||
links(t, nil)
|
||||
f.declare("// none")
|
||||
f.declareAccounts(adoptable)
|
||||
f.declareConfiguration(strings.Replace(declaredConfiguration, `"work": {"name": "Team", "url": "https://dav.example.org/cal/team/"},`, "", 1), declaredTags)
|
||||
d, err := f.Drift("")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
by := map[string]KindDrift{}
|
||||
for _, k := range d.Kinds {
|
||||
by[k.Kind] = k
|
||||
}
|
||||
field := func(kind, name, f string) *fieldDiff {
|
||||
for _, e := range by[kind].Declared {
|
||||
if e.Name == name {
|
||||
for _, x := range e.Differs {
|
||||
if x.Field == f {
|
||||
return &x
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if x := field(kindIdentities, "mail", "signature"); x == nil || mustJSON(x.Profile) != `["Regards"]` {
|
||||
t.Errorf("the signature differs, with both values: %+v", by[kindIdentities])
|
||||
}
|
||||
if x := field(kindCalendars, "home", "color"); x == nil || x.Declared != "#3366cc" || x.Profile != "#3366ff" {
|
||||
t.Errorf("the colour differs: %+v", by[kindCalendars])
|
||||
}
|
||||
if x := field(kindAddressBooks, "contacts", "user"); x == nil || x.Profile != "me" {
|
||||
t.Errorf("the user differs: %+v", by[kindAddressBooks])
|
||||
}
|
||||
states := map[string]string{}
|
||||
for _, k := range d.Kinds {
|
||||
for _, e := range k.Declared {
|
||||
states[k.Kind+"/"+e.Name] = e.State
|
||||
}
|
||||
}
|
||||
if states["tags/todo"] != "lacks" || states["calendars/shared"] != refused || states["tags/$label1"] != "holds" ||
|
||||
states["address-books/team"] != "lacks" {
|
||||
t.Errorf("lacks, refused and holds: %v", states)
|
||||
}
|
||||
undeclared := mustJSON(d)
|
||||
for _, want := range []string{`"key":"personal"`, `"key":"c2"`, `"key":"c3"`} {
|
||||
if !strings.Contains(undeclared, want) {
|
||||
t.Errorf("the undeclared entries are listed: lacks %s", want)
|
||||
}
|
||||
}
|
||||
if _, err := f.userJS(true); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
d, _ = f.Drift("")
|
||||
for _, k := range d.Kinds {
|
||||
for _, e := range k.Declared {
|
||||
if e.State == "differs" {
|
||||
t.Errorf("once the user.js is placed, %s %s holds: %+v", k.Kind, e.Name, e)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// §1, §2, §13: the listing tools answer the operator's own values, Owl's marked, an ICS calendar by its name only,
|
||||
// and never a credential: through the tools, a password in an address and a field named for one are withheld.
|
||||
func TestTheListingsShowTheValuesOwlsMarkedAnICSByNameAndNoCredential(t *testing.T) {
|
||||
f := newFake(t)
|
||||
f.profileWithAccounts()
|
||||
f.profileWithConfiguration()
|
||||
f.write(profileDir+"/prefs.js", f.read(profileDir+"/prefs.js")+
|
||||
`user_pref("calendar.registry.c6.type", "caldav");`+"\n"+
|
||||
`user_pref("calendar.registry.c6.uri", "https://other:hunter2@dav.example.org/cal/x/");`+"\n"+
|
||||
`user_pref("calendar.registry.c6.name", "Old");`+"\n")
|
||||
f.write(profileDir+"/logins.json", `{"logins":[{"hostname":"https://dav.example.org","encryptedPassword":"hunter3"}]}`)
|
||||
links(t, nil)
|
||||
f.declare("// none")
|
||||
f.declareAccounts(adoptable)
|
||||
f.declareConfiguration(declaredConfiguration, declaredTags)
|
||||
old := machine
|
||||
machine = f.Machine
|
||||
t.Cleanup(func() { machine = old })
|
||||
answers := map[string]string{}
|
||||
for _, tool := range tools() {
|
||||
if strings.HasPrefix(tool.Name, "betterbird_") && contains([]string{"betterbird_identities", "betterbird_calendars",
|
||||
"betterbird_address_books", "betterbird_filters", "betterbird_tags", "betterbird_accounts", "betterbird_drift"}, tool.Name) {
|
||||
out, err := tool.Run(map[string]any{})
|
||||
if err != nil {
|
||||
t.Fatalf("%s: %v", tool.Name, err)
|
||||
}
|
||||
answers[tool.Name] = mustJSON(out)
|
||||
}
|
||||
}
|
||||
cals := answers["betterbird_calendars"]
|
||||
for _, shown := range []string{"https://dav.example.org/cal/home/", `"name":"Work","url":"owl://work@corp.example.com/calendar"`,
|
||||
`"owl":true`, `"name":"Holidays"`} {
|
||||
if !strings.Contains(cals, shown) {
|
||||
t.Errorf("betterbird_calendars shows %s: %s", shown, cals)
|
||||
}
|
||||
}
|
||||
if strings.Contains(cals, "secret-token") {
|
||||
t.Error("an ICS calendar is listed by its name only")
|
||||
}
|
||||
for name, a := range answers {
|
||||
if strings.Contains(a, "hunter2") || strings.Contains(a, "hunter3") {
|
||||
t.Errorf("%s carries a credential: %s", name, a)
|
||||
}
|
||||
}
|
||||
if !strings.Contains(cals, "https://other:‹withheld›@dav.example.org/cal/x/") {
|
||||
t.Errorf("an address's password is shown as ‹withheld›: %s", cals)
|
||||
}
|
||||
for _, shown := range []string{`"signature":["Regards"]`, "other@mail.example.org"} {
|
||||
if !strings.Contains(answers["betterbird_identities"], shown) {
|
||||
t.Errorf("betterbird_identities shows %s", shown)
|
||||
}
|
||||
}
|
||||
if got := mustJSON(credentialChecked(map[string]any{"name": "x", "password": "p", "nested": []any{map[string]any{"oauthToken": "t"}}})); got !=
|
||||
`{"name":"x","nested":[{"oauthToken":"‹withheld›"}],"password":"‹withheld›"}` {
|
||||
t.Errorf("a field named for a credential is withheld: %s", got)
|
||||
}
|
||||
// The start's words reach the journal: a refusal names kinds, names and keys, never an address.
|
||||
stopBetterbird(t, f)
|
||||
err := f.Prepare()
|
||||
if err == nil || !strings.Contains(err.Error(), "calendars shared is refused") || strings.Contains(err.Error(), "example.org") {
|
||||
t.Fatalf("%v", err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,700 @@
|
||||
package main
|
||||
|
||||
// The five settings derived from the desktop nodes' profiles and proposed for the operator's Approve, one layer
|
||||
// at a time (novox/hq ADR 0329 §10, §12).
|
||||
//
|
||||
// betterbird_derive asks every desktop node that holds Betterbird for what its profile holds (its own derive with
|
||||
// here), and lays the answers out per layer: an entry found on every such node goes into the mesh-wide layer, with
|
||||
// the value of the node the derive was called on, and the answer lists every field another node holds differently,
|
||||
// which then shows as drift there; an entry found on some nodes only goes into each of those nodes' layers, so
|
||||
// nothing is added to another node until the operator moves it. With propose it proposes the next layer through
|
||||
// the controller's settings verb: first the mesh-wide one, then each node's that needs one, each only once no
|
||||
// proposal of the module's is open, against the layer as it then stands, which it carries whole beside the derived
|
||||
// kinds (a layer is replaced whole). The ask shows the values in their shape (ADR 0277 §6); the operator reads them
|
||||
// whole with `settings proposals <id>` at their desk.
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// derivedEntry is one entry a node's profile holds, in the setting's shape. Match is what makes two nodes' entries
|
||||
// the same one: an identity's account and address, a calendar's or an address book's address, a tag's key, an
|
||||
// account's name for its filters.
|
||||
type derivedEntry struct {
|
||||
Match string `json:"match"`
|
||||
Name string `json:"name"`
|
||||
Value json.RawMessage `json:"value"`
|
||||
}
|
||||
|
||||
// nodeDerivation is what one node's profile holds of the five kinds.
|
||||
type nodeDerivation struct {
|
||||
Node string `json:"node"`
|
||||
Kinds map[string][]derivedEntry `json:"kinds"`
|
||||
// NotDerived names what a setting does not hold, by name and why: Owl's, an ICS calendar, a local address
|
||||
// book, an address carrying a password, a filter a setting cannot say.
|
||||
NotDerived []string `json:"not_derived,omitempty"`
|
||||
}
|
||||
|
||||
var nameChars = regexp.MustCompile(`[^a-z0-9]`)
|
||||
|
||||
// entryNameFor is a name of the operator's kind for an entry: lower-case letters and digits from its words,
|
||||
// unique among those used.
|
||||
func entryNameFor(words string, fallback string, used map[string]bool) string {
|
||||
base := nameChars.ReplaceAllString(strings.ToLower(words), "")
|
||||
if base == "" || base[0] < 'a' || base[0] > 'z' {
|
||||
base = fallback + base
|
||||
}
|
||||
if len(base) > 28 {
|
||||
base = base[:28]
|
||||
}
|
||||
name := base
|
||||
for i := 2; used[name]; i++ {
|
||||
name = fmt.Sprintf("%s%d", base, i)
|
||||
}
|
||||
used[name] = true
|
||||
return name
|
||||
}
|
||||
|
||||
func raw(v any) json.RawMessage {
|
||||
b, _ := json.Marshal(v)
|
||||
return b
|
||||
}
|
||||
|
||||
// deriveHere reads this node's profile into the five kinds.
|
||||
func (m *Machine) deriveHere(kinds []string) (nodeDerivation, error) {
|
||||
d := nodeDerivation{Node: nodeName(), Kinds: map[string][]derivedEntry{}}
|
||||
p, err := m.profile()
|
||||
if err != nil {
|
||||
return d, err
|
||||
}
|
||||
prefs := mergedPrefs(p.path)
|
||||
accounts, _, err := m.readAccounts()
|
||||
if err != nil {
|
||||
return d, err
|
||||
}
|
||||
nameOf := map[string]string{}
|
||||
serverOf := map[string]string{}
|
||||
for _, r := range resolveAccounts(accounts, prefs, m.readRecord(p.Dir)) {
|
||||
if r.How != refused {
|
||||
nameOf[r.Keys.Account], serverOf[r.name] = r.name, r.Keys.Server
|
||||
}
|
||||
}
|
||||
want := map[string]bool{}
|
||||
for _, k := range kinds {
|
||||
want[k] = true
|
||||
}
|
||||
// Identities, by their account's name; the default carries no name or address (the setting accounts' own).
|
||||
identityName := map[string]string{}
|
||||
used := map[string]bool{}
|
||||
byAccount := map[string]bool{}
|
||||
for _, pi := range m.profileIdentities(prefs) {
|
||||
account, declared := nameOf[pi.AccountKey]
|
||||
switch {
|
||||
case pi.Owl:
|
||||
if pi.Default {
|
||||
d.NotDerived = append(d.NotDerived, fmt.Sprintf("identities: %s <%s> is Owl's (novox/hq ADR 0316 §13)",
|
||||
pi.Value.Name, pi.Value.Address))
|
||||
}
|
||||
continue
|
||||
case !declared:
|
||||
if !byAccount[pi.AccountKey] {
|
||||
byAccount[pi.AccountKey] = true
|
||||
d.NotDerived = append(d.NotDerived, fmt.Sprintf("identities: those of the profile's account %s, which the "+
|
||||
"setting accounts does not hold", pi.AccountKey))
|
||||
}
|
||||
continue
|
||||
}
|
||||
v := pi.Value
|
||||
v.Account = account
|
||||
e := derivedEntry{}
|
||||
if pi.Default {
|
||||
v.Address, v.Name = "", ""
|
||||
e.Match, e.Name = account+"|", entryNameFor(account, "id", used)
|
||||
} else {
|
||||
local, _, _ := strings.Cut(v.Address, "@")
|
||||
e.Match, e.Name = account+"|"+strings.ToLower(v.Address), entryNameFor(account+local, "id", used)
|
||||
}
|
||||
identityName[pi.Key] = e.Name
|
||||
e.Value = raw(v)
|
||||
if want[kindIdentities] {
|
||||
d.Kinds[kindIdentities] = append(d.Kinds[kindIdentities], e)
|
||||
}
|
||||
}
|
||||
// Calendars: CalDAV only; Owl's are listed, an ICS one by its name only, a local one by its name (§13).
|
||||
used = map[string]bool{}
|
||||
for _, pc := range profileCalendars(prefs) {
|
||||
if !want[kindCalendars] {
|
||||
break
|
||||
}
|
||||
switch {
|
||||
case pc.Kind == "owl" || strings.HasPrefix(pc.Kind, "owl"):
|
||||
d.NotDerived = append(d.NotDerived, fmt.Sprintf("calendars: %s (%s) is Owl's", pc.Value.Name, pc.Value.URL))
|
||||
continue
|
||||
case pc.Kind == "ics":
|
||||
d.NotDerived = append(d.NotDerived, fmt.Sprintf("calendars: %s is subscribed by an ICS address, which is "+
|
||||
"often itself the secret: listed by its name only, never declared (novox/hq ADR 0329 §13)", pc.Value.Name))
|
||||
continue
|
||||
case pc.Kind != "caldav":
|
||||
d.NotDerived = append(d.NotDerived, fmt.Sprintf("calendars: %s is %s, kept in the profile: its events are never declared",
|
||||
pc.Value.Name, pc.Kind))
|
||||
continue
|
||||
case addressPassword(pc.Value.URL):
|
||||
d.NotDerived = append(d.NotDerived, fmt.Sprintf("calendars: %s at %s carries a password in its address: refused "+
|
||||
"(novox/hq ADR 0329 §2)", pc.Value.Name, withoutPassword(pc.Value.URL)))
|
||||
continue
|
||||
}
|
||||
v := pc.Value
|
||||
v.Invitations = identityName[pc.IdentityKey]
|
||||
d.Kinds[kindCalendars] = append(d.Kinds[kindCalendars], derivedEntry{Match: strings.ToLower(strings.TrimRight(v.URL, "/")),
|
||||
Name: entryNameFor(v.Name, "calendar", used), Value: raw(v)})
|
||||
}
|
||||
// Address books: CardDAV only.
|
||||
used = map[string]bool{}
|
||||
for _, b := range profileAddressBooks(prefs) {
|
||||
if !want[kindAddressBooks] {
|
||||
break
|
||||
}
|
||||
switch {
|
||||
case !b.CardDAV:
|
||||
d.NotDerived = append(d.NotDerived, fmt.Sprintf("address-books: %s is %s: its contacts are never declared", b.Value.Name, b.Kind))
|
||||
continue
|
||||
case addressPassword(b.Value.URL):
|
||||
d.NotDerived = append(d.NotDerived, fmt.Sprintf("address-books: %s at %s carries a password in its address: "+
|
||||
"refused (novox/hq ADR 0329 §2)", b.Value.Name, withoutPassword(b.Value.URL)))
|
||||
continue
|
||||
}
|
||||
d.Kinds[kindAddressBooks] = append(d.Kinds[kindAddressBooks], derivedEntry{
|
||||
Match: strings.ToLower(strings.TrimRight(b.Value.URL, "/")), Name: entryNameFor(b.Value.Name, "book", used), Value: raw(b.Value)})
|
||||
}
|
||||
// Tags, by their own key.
|
||||
if want[kindTags] {
|
||||
for _, t := range profileTags(prefs) {
|
||||
d.Kinds[kindTags] = append(d.Kinds[kindTags], derivedEntry{Match: t.Key, Name: t.Key, Value: raw(t.Value)})
|
||||
}
|
||||
}
|
||||
// Filters of each declared account whose file holds one or more: an account with none keeps its file as the
|
||||
// person's until a setting names it.
|
||||
if want[kindFilters] {
|
||||
for _, name := range sortedKeys(serverOf) {
|
||||
dir := m.accountDirectory(p.path, prefs, serverOf[name])
|
||||
if dir == "" {
|
||||
continue
|
||||
}
|
||||
text, err := readBounded(dir + "/" + filterFileName)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
parsed := parseFilters(string(text), folderBase(prefs, serverOf[name]))
|
||||
for _, why := range parsed.NotDeclarable {
|
||||
d.NotDerived = append(d.NotDerived, "filters of "+name+": "+why)
|
||||
}
|
||||
if len(parsed.Filters) > 0 {
|
||||
d.Kinds[kindFilters] = append(d.Kinds[kindFilters], derivedEntry{Match: name, Name: name, Value: raw(parsed.Filters)})
|
||||
}
|
||||
}
|
||||
}
|
||||
return d, nil
|
||||
}
|
||||
|
||||
// difference is one field another node holds differently from the value the mesh-wide layer gets.
|
||||
type difference struct {
|
||||
Kind string `json:"kind"`
|
||||
Entry string `json:"entry"`
|
||||
Node string `json:"node"`
|
||||
Field string `json:"field"`
|
||||
MeshWide any `json:"mesh_wide"`
|
||||
There any `json:"there"`
|
||||
}
|
||||
|
||||
// DeriveAnswer is what betterbird_derive answers its caller: the values it derived, which ADR 0329 §10 lets it show.
|
||||
type DeriveAnswer struct {
|
||||
// Here is this node's own derivation, when asked with here.
|
||||
Here *nodeDerivation `json:"here,omitempty"`
|
||||
// From are the desktop nodes derived from, and NotHeard those holding Betterbird that did not answer.
|
||||
From []string `json:"derived_from,omitempty"`
|
||||
NotHeard []string `json:"not_heard,omitempty"`
|
||||
// Layers are the derived values per layer: "mesh-wide", then each node's own.
|
||||
Layers map[string]map[string]any `json:"layers,omitempty"`
|
||||
Differences []difference `json:"differences,omitempty"`
|
||||
NotDerived map[string][]string `json:"not_derived,omitempty"`
|
||||
// Next is the layer a propose proposes now, or why none.
|
||||
Next string `json:"next,omitempty"`
|
||||
Proposed string `json:"proposed,omitempty"`
|
||||
Note string `json:"note,omitempty"`
|
||||
}
|
||||
|
||||
const meshWide = "mesh-wide"
|
||||
|
||||
// Derive derives the kinds asked for (all five when none) from the desktop nodes, and with propose proposes the
|
||||
// next layer. prefs are values for the setting prefs to carry in the mesh-wide layer (issue 586: the operator's
|
||||
// setup moved out of the catalogue); layer names the layer to propose instead of the next in order.
|
||||
func (m *Machine) Derive(kinds []string, here, propose bool, prefs map[string]any, layer string) (DeriveAnswer, error) {
|
||||
var a DeriveAnswer
|
||||
if len(kinds) == 0 {
|
||||
kinds = allKinds
|
||||
}
|
||||
for _, k := range kinds {
|
||||
if !contains(allKinds, k) {
|
||||
return a, fmt.Errorf("kinds are %s; %q is not one", strings.Join(allKinds, ", "), k)
|
||||
}
|
||||
}
|
||||
if here {
|
||||
d, err := m.deriveHere(kinds)
|
||||
if err != nil {
|
||||
return a, err
|
||||
}
|
||||
a.Here = &d
|
||||
return a, nil
|
||||
}
|
||||
self := nodeName()
|
||||
if self == "" {
|
||||
return a, errors.New("this runtime does not say which machine it serves (MESH_NODE), so the node derived on is unknown")
|
||||
}
|
||||
holders, err := holdersOfBetterbird()
|
||||
if err != nil {
|
||||
return a, err
|
||||
}
|
||||
derived := map[string]nodeDerivation{}
|
||||
for _, n := range holders {
|
||||
if n == self {
|
||||
d, err := m.deriveHere(kinds)
|
||||
if err != nil {
|
||||
return a, fmt.Errorf("this node's profile: %v", err)
|
||||
}
|
||||
derived[n] = d
|
||||
continue
|
||||
}
|
||||
d, err := deriveOn(n, kinds)
|
||||
if err != nil {
|
||||
a.NotHeard = append(a.NotHeard, n+": "+err.Error())
|
||||
continue
|
||||
}
|
||||
derived[n] = d
|
||||
}
|
||||
if _, ok := derived[self]; !ok {
|
||||
return a, fmt.Errorf("%s does not hold Betterbird as the controller says: derive on a desktop node", self)
|
||||
}
|
||||
a.From = sortedKeys(derived)
|
||||
a.Layers, a.Differences = layOut(self, derived, kinds)
|
||||
a.NotDerived = map[string][]string{}
|
||||
for n, d := range derived {
|
||||
if len(d.NotDerived) > 0 {
|
||||
a.NotDerived[n] = d.NotDerived
|
||||
}
|
||||
}
|
||||
if len(a.NotDerived) == 0 {
|
||||
a.NotDerived = nil
|
||||
}
|
||||
if len(prefs) > 0 {
|
||||
a.Layers[meshWide]["prefs"] = prefs
|
||||
}
|
||||
a.Note = "the values each layer would get; an entry on every desktop node is mesh-wide with this node's value, one " +
|
||||
"on some nodes only is in those nodes' own layers. With propose, the next layer is proposed for the operator's " +
|
||||
"Approve, carrying the layer as it stands beside these kinds; nothing changes until then (novox/hq ADR 0329 §10)"
|
||||
if !propose {
|
||||
return a, nil
|
||||
}
|
||||
if len(a.NotHeard) > 0 {
|
||||
return a, fmt.Errorf("nothing was proposed: %d desktop node(s) holding Betterbird did not answer (%s), and a "+
|
||||
"layer derived without them would put on every node what only some hold; derive again when they are awake",
|
||||
len(a.NotHeard), strings.Join(a.NotHeard, "; "))
|
||||
}
|
||||
if open, err := openProposals(); err != nil {
|
||||
return a, fmt.Errorf("nothing was proposed: whether a proposal of the module's is open cannot be read: %v", err)
|
||||
} else if len(open) > 0 {
|
||||
return a, fmt.Errorf("nothing was proposed: the module's proposal %s waits for the operator; the next layer is "+
|
||||
"proposed once it is set or declined (novox/hq ADR 0329 §10, ADR 0277 §10)", strings.Join(open, ", "))
|
||||
}
|
||||
order := []string{meshWide}
|
||||
for _, n := range sortedKeys(a.Layers) {
|
||||
if n != meshWide {
|
||||
order = append(order, n)
|
||||
}
|
||||
}
|
||||
if layer != "" {
|
||||
if _, ok := a.Layers[layer]; !ok {
|
||||
return a, fmt.Errorf("nothing was proposed: layer is %s", strings.Join(order, " or "))
|
||||
}
|
||||
order = []string{layer}
|
||||
}
|
||||
for _, l := range order {
|
||||
node := l
|
||||
if l == meshWide {
|
||||
node = ""
|
||||
}
|
||||
current, err := readLayer(node)
|
||||
if err != nil {
|
||||
return a, fmt.Errorf("nothing was proposed: %v", err)
|
||||
}
|
||||
next := withKinds(current, a.Layers[l])
|
||||
if l == meshWide {
|
||||
if prefs, ok := next["prefs"]; ok {
|
||||
body, _ := json.Marshal(map[string]any{"prefs": prefs})
|
||||
if _, err := parseDeclaredPrefs(body); err != nil {
|
||||
return a, fmt.Errorf("nothing was proposed: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
if err := proposable(next, l == meshWide); err != nil {
|
||||
return a, fmt.Errorf("nothing was proposed for %s: %v", l, err)
|
||||
}
|
||||
if mustJSON(next) == mustJSON(current) {
|
||||
continue
|
||||
}
|
||||
a.Next = l
|
||||
said, err := proposeLayer(node, next, true)
|
||||
a.Proposed = m.maskValue(said)
|
||||
if err != nil {
|
||||
return a, err
|
||||
}
|
||||
a.Note = "proposed the " + l + " layer for the operator's Approve; the next layer is proposed by calling again " +
|
||||
"once this one is set or declined"
|
||||
return a, nil
|
||||
}
|
||||
a.Next = "none: every layer already holds what was derived"
|
||||
return a, nil
|
||||
}
|
||||
|
||||
// proposable checks a layer's derived kinds as the module would read them, so the operator is never asked to set
|
||||
// a value the module then refuses. A key to adopt names one node's profile, so the mesh-wide layer holds none (§4).
|
||||
func proposable(layer map[string]any, mesh bool) error {
|
||||
conf := map[string]any{}
|
||||
for _, k := range []string{kindIdentities, kindCalendars, kindAddressBooks, kindFilters} {
|
||||
if v, ok := layer[k]; ok {
|
||||
conf[k] = v
|
||||
}
|
||||
}
|
||||
var c Configuration
|
||||
if err := decodeInto([]byte(mustJSON(conf)), configurationFile, &c); err != nil {
|
||||
return err
|
||||
}
|
||||
if v, ok := layer[kindTags]; ok {
|
||||
if err := decodeInto([]byte(mustJSON(map[string]any{kindTags: v})), tagsFile, &c); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
// A node's layer may name an identity the mesh-wide layer holds for a calendar's invitations: that is judged
|
||||
// where the layers meet, on the node.
|
||||
var problems []string
|
||||
if mesh {
|
||||
for name, id := range c.Identities {
|
||||
if id != nil && id.Key != "" {
|
||||
problems = append(problems, "identities "+name+": a key is given in a node's own layer only")
|
||||
}
|
||||
}
|
||||
for name, cal := range c.Calendars {
|
||||
if cal != nil && cal.Key != "" {
|
||||
problems = append(problems, "calendars "+name+": a key is given in a node's own layer only")
|
||||
}
|
||||
}
|
||||
for name, ab := range c.AddressBooks {
|
||||
if ab != nil && ab.Key != "" {
|
||||
problems = append(problems, "address-books "+name+": a key is given in a node's own layer only")
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, p := range configurationProblems(c) {
|
||||
if !strings.Contains(p, "invitations names") {
|
||||
problems = append(problems, p)
|
||||
}
|
||||
}
|
||||
if len(problems) > 0 {
|
||||
return errors.New(strings.Join(problems, "; "))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// layOut puts each kind's entries into the layers (§10).
|
||||
func layOut(self string, derived map[string]nodeDerivation, kinds []string) (map[string]map[string]any, []difference) {
|
||||
layers := map[string]map[string]any{meshWide: {}}
|
||||
var diffs []difference
|
||||
nodes := sortedKeys(derived)
|
||||
for _, kind := range kinds {
|
||||
type seen struct {
|
||||
by map[string]derivedEntry
|
||||
}
|
||||
groups := map[string]*seen{}
|
||||
var order []string
|
||||
for _, n := range append([]string{self}, nodes...) {
|
||||
for _, e := range derived[n].Kinds[kind] {
|
||||
g := groups[e.Match]
|
||||
if g == nil {
|
||||
g = &seen{by: map[string]derivedEntry{}}
|
||||
groups[e.Match] = g
|
||||
order = append(order, e.Match)
|
||||
}
|
||||
if _, already := g.by[n]; !already {
|
||||
g.by[n] = e
|
||||
}
|
||||
}
|
||||
}
|
||||
mesh := map[string]any{}
|
||||
used := map[string]bool{}
|
||||
for _, match := range order {
|
||||
g := groups[match]
|
||||
if len(g.by) == len(nodes) {
|
||||
mine := g.by[self]
|
||||
name := uniqueName(mine.Name, used)
|
||||
var v any
|
||||
_ = json.Unmarshal(mine.Value, &v)
|
||||
mesh[name] = v
|
||||
for _, n := range nodes {
|
||||
if n == self {
|
||||
continue
|
||||
}
|
||||
diffs = append(diffs, fieldDifferences(kind, name, n, mine.Value, g.by[n].Value)...)
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, match := range order {
|
||||
g := groups[match]
|
||||
if len(g.by) == len(nodes) {
|
||||
continue
|
||||
}
|
||||
for _, n := range sortedKeys(g.by) {
|
||||
e := g.by[n]
|
||||
if layers[n] == nil {
|
||||
layers[n] = map[string]any{}
|
||||
}
|
||||
nodeKind, _ := layers[n][kind].(map[string]any)
|
||||
if nodeKind == nil {
|
||||
nodeKind = map[string]any{}
|
||||
layers[n][kind] = nodeKind
|
||||
}
|
||||
var v any
|
||||
_ = json.Unmarshal(e.Value, &v)
|
||||
nodeKind[uniqueName(e.Name, used)] = v
|
||||
}
|
||||
}
|
||||
if len(mesh) > 0 {
|
||||
layers[meshWide][kind] = mesh
|
||||
}
|
||||
}
|
||||
return layers, diffs
|
||||
}
|
||||
|
||||
func uniqueName(name string, used map[string]bool) string {
|
||||
n := name
|
||||
for i := 2; used[n]; i++ {
|
||||
n = fmt.Sprintf("%s%d", name, i)
|
||||
}
|
||||
used[n] = true
|
||||
return n
|
||||
}
|
||||
|
||||
// fieldDifferences are the fields a node holds differently from the mesh-wide value of one entry.
|
||||
func fieldDifferences(kind, entry, node string, mesh, there json.RawMessage) []difference {
|
||||
var a, b map[string]any
|
||||
if json.Unmarshal(mesh, &a) != nil || json.Unmarshal(there, &b) != nil {
|
||||
if mustJSON(json.RawMessage(mesh)) != mustJSON(json.RawMessage(there)) {
|
||||
var x, y any
|
||||
_ = json.Unmarshal(mesh, &x)
|
||||
_ = json.Unmarshal(there, &y)
|
||||
return []difference{{Kind: kind, Entry: entry, Node: node, Field: "the whole list", MeshWide: x, There: y}}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
var out []difference
|
||||
for _, f := range sortedKeys(mergeKeys(a, b)) {
|
||||
if mustJSON(a[f]) != mustJSON(b[f]) {
|
||||
out = append(out, difference{Kind: kind, Entry: entry, Node: node, Field: f, MeshWide: a[f], There: b[f]})
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func mergeKeys(a, b map[string]any) map[string]bool {
|
||||
out := map[string]bool{}
|
||||
for k := range a {
|
||||
out[k] = true
|
||||
}
|
||||
for k := range b {
|
||||
out[k] = true
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// withKinds is a layer with the derived kinds in place of its own, every other key kept.
|
||||
func withKinds(layer map[string]any, kinds map[string]any) map[string]any {
|
||||
out := map[string]any{}
|
||||
for k, v := range layer {
|
||||
out[k] = v
|
||||
}
|
||||
for k, v := range kinds {
|
||||
if k == "prefs" {
|
||||
merged := map[string]any{}
|
||||
if was, ok := out["prefs"].(map[string]any); ok {
|
||||
for n, x := range was {
|
||||
merged[n] = x
|
||||
}
|
||||
}
|
||||
for n, x := range v.(map[string]any) {
|
||||
merged[n] = x
|
||||
}
|
||||
out[k] = merged
|
||||
continue
|
||||
}
|
||||
out[k] = v
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func contains(list []string, s string) bool {
|
||||
for _, x := range list {
|
||||
if x == s {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// readLayer is the module's layer as the controller holds it: the mesh-wide one for node "", else that node's.
|
||||
func readLayer(node string) (map[string]any, error) {
|
||||
args := map[string]any{"module": "betterbird"}
|
||||
if node != "" {
|
||||
args["node"] = node
|
||||
}
|
||||
raw, err := asker("seat:mesh-controller.settings", args)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the controller's settings verb did not answer: %v", err)
|
||||
}
|
||||
said, ok := verbSaid(raw)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("the controller did not answer the layer: %s", said)
|
||||
}
|
||||
// A missing layer is said in exactly these words; anything else must read as the layer's JSON. A proposal
|
||||
// replaces the layer whole, so an answer read wrongly as empty would drop what the layer holds: refused, loudly.
|
||||
text := strings.TrimSpace(said)
|
||||
where := "the whole mesh"
|
||||
if node != "" {
|
||||
where = node
|
||||
}
|
||||
if strings.HasPrefix(text, "betterbird on "+where+": no layer") {
|
||||
return map[string]any{}, nil
|
||||
}
|
||||
out := map[string]any{}
|
||||
if err := json.NewDecoder(strings.NewReader(text)).Decode(&out); err != nil || !strings.HasPrefix(text, "{") {
|
||||
first, _, _ := strings.Cut(text, "\n")
|
||||
return nil, fmt.Errorf("the controller's answer for the layer of betterbird on %s cannot be read as the layer "+
|
||||
"(it begins %q), so nothing is proposed: a layer is replaced whole and would lose what it holds", where, first)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// proposeLayer proposes a layer whole for the operator's Approve (ADR 0277); replace says a key the layer had
|
||||
// and the proposal does not hold is meant to go, which a re-derive needs when an entry left every profile.
|
||||
func proposeLayer(node string, values map[string]any, replace bool) (string, error) {
|
||||
body, err := json.Marshal(values)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
args := map[string]any{"module": "betterbird", "values": string(body), "propose": "true"}
|
||||
if replace {
|
||||
args["replace"] = "true"
|
||||
}
|
||||
if node != "" {
|
||||
args["node"] = node
|
||||
}
|
||||
raw, err := asker("seat:mesh-controller.settings", args)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("the controller's settings verb did not answer: %v", err)
|
||||
}
|
||||
said, ok := verbSaid(raw)
|
||||
if !ok {
|
||||
return said, fmt.Errorf("the controller did not take the proposal: %s", said)
|
||||
}
|
||||
return said, nil
|
||||
}
|
||||
|
||||
var (
|
||||
proposalHead = regexp.MustCompile(`^(\S+)\s+(?:set|clear) betterbird on (.+?)\s+\(`)
|
||||
waitingLine = regexp.MustCompile(`^\s+waiting for the operator until`)
|
||||
)
|
||||
|
||||
// openProposals are the module's proposals that wait for the operator, by id and layer.
|
||||
func openProposals() ([]string, error) {
|
||||
raw, err := asker("seat:mesh-controller.settings", map[string]any{"proposals": "true"})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
said, ok := verbSaid(raw)
|
||||
if !ok {
|
||||
return nil, errors.New(said)
|
||||
}
|
||||
var out []string
|
||||
current := ""
|
||||
for _, line := range strings.Split(said, "\n") {
|
||||
if mm := proposalHead.FindStringSubmatch(line); mm != nil {
|
||||
current = mm[1] + " (" + mm[2] + ")"
|
||||
continue
|
||||
}
|
||||
if !strings.HasPrefix(line, " ") {
|
||||
current = ""
|
||||
}
|
||||
if current != "" && waitingLine.MatchString(line) {
|
||||
out = append(out, current)
|
||||
current = ""
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// holdersOfBetterbird are the nodes the controller has assigned the module to: the desktop nodes.
|
||||
func holdersOfBetterbird() ([]string, error) {
|
||||
raw, err := asker("seat:mesh-controller.nodes", map[string]any{})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the controller's nodes verb did not answer: %v", err)
|
||||
}
|
||||
said, _ := verbSaid(raw)
|
||||
var nodes []struct {
|
||||
Name string `json:"name"`
|
||||
}
|
||||
if err := json.Unmarshal([]byte(said), &nodes); err != nil {
|
||||
return nil, fmt.Errorf("the controller's nodes cannot be read: %v", err)
|
||||
}
|
||||
var out []string
|
||||
for _, n := range nodes {
|
||||
raw, err := asker("seat:mesh-controller.node", map[string]any{"node": n.Name})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the controller's node verb did not answer for %s: %v", n.Name, err)
|
||||
}
|
||||
said, _ := verbSaid(raw)
|
||||
for _, line := range strings.Split(said, "\n") {
|
||||
if rest, ok := strings.CutPrefix(strings.TrimSpace(line), "assigned:"); ok {
|
||||
for _, mod := range strings.Split(rest, ",") {
|
||||
if strings.TrimSpace(mod) == "betterbird" {
|
||||
out = append(out, n.Name)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
sort.Strings(out)
|
||||
if len(out) == 0 {
|
||||
return nil, errors.New("the controller names no node the module is assigned to")
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// deriveOn asks another node's betterbird for what its profile holds.
|
||||
func deriveOn(node string, kinds []string) (nodeDerivation, error) {
|
||||
raw, err := asker("betterbird.betterbird_derive@"+node, map[string]any{"here": true, "kinds": kinds})
|
||||
if err != nil {
|
||||
return nodeDerivation{}, err
|
||||
}
|
||||
said, ok := verbSaid(raw)
|
||||
if !ok {
|
||||
return nodeDerivation{}, errors.New(said)
|
||||
}
|
||||
var a DeriveAnswer
|
||||
if err := json.Unmarshal([]byte(said), &a); err != nil || a.Here == nil {
|
||||
return nodeDerivation{}, fmt.Errorf("its answer cannot be read")
|
||||
}
|
||||
a.Here.Node = node
|
||||
return *a.Here, nil
|
||||
}
|
||||
@@ -0,0 +1,263 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// ADR 0329 §10, §12: derived from every desktop node's profile, laid out per layer, and proposed one layer at a
|
||||
// time, carrying the layer as it stands.
|
||||
|
||||
// derivingMesh is two desktop nodes, g14 (this one, f) and shanks (f2), and a server that holds no Betterbird, as the
|
||||
// controller answers them; layers are the controller's layers, open the proposals that wait.
|
||||
type derivingMesh struct {
|
||||
t *testing.T
|
||||
shanks *fake
|
||||
layers map[string]map[string]any
|
||||
open bool
|
||||
proposed []map[string]any
|
||||
asked []string
|
||||
asleep bool
|
||||
// garbled answers the layer in words that are not its JSON.
|
||||
garbled bool
|
||||
}
|
||||
|
||||
func (d *derivingMesh) ask(key string, body any) (json.RawMessage, error) {
|
||||
d.asked = append(d.asked, key)
|
||||
args, _ := body.(map[string]any)
|
||||
answer := func(ok bool, text string) (json.RawMessage, error) {
|
||||
b, _ := json.Marshal(map[string]any{"ok": ok, "output": text})
|
||||
return b, nil
|
||||
}
|
||||
switch {
|
||||
case key == "seat:mesh-controller.nodes":
|
||||
return answer(true, `[{"name": "g14"}, {"name": "novox"}, {"name": "shanks"}]`)
|
||||
case key == "seat:mesh-controller.node":
|
||||
if args["node"] == "novox" {
|
||||
return answer(true, "novox\n assigned: postgres, nats\n")
|
||||
}
|
||||
return answer(true, fmt.Sprintf("%s\n last heard from here\n assigned: adwaita, betterbird, dunst\n", args["node"]))
|
||||
case key == "betterbird.betterbird_derive@shanks":
|
||||
if d.asleep {
|
||||
return nil, errors.New("nats: no responders available for request")
|
||||
}
|
||||
here, err := d.shanks.deriveHere(stringsOf(args["kinds"]))
|
||||
if err != nil {
|
||||
d.t.Fatal(err)
|
||||
}
|
||||
text, _ := json.Marshal(credentialChecked(DeriveAnswer{Here: &here}))
|
||||
return json.Marshal(map[string]any{"content": []any{map[string]any{"type": "text", "text": string(text)}}})
|
||||
case key == "seat:mesh-controller.settings" && args["proposals"] == "true":
|
||||
if d.open {
|
||||
return answer(true, "sab12 set betterbird on the whole mesh (accounts, identities)\n by g14.node-tools at 2026-10-13 01:00; fingerprint ab12\n waiting for the operator until 2026-10-14 01:00\n"+
|
||||
"scd34 set mounts on shanks (sources)\n by g14.node-tools at 2026-10-11 00:05; fingerprint dd0b\n chosen: done\n")
|
||||
}
|
||||
return answer(true, "scd34 set mounts on shanks (sources)\n by g14.node-tools; fingerprint dd0b\n chosen: done\n")
|
||||
case key == "seat:mesh-controller.settings" && args["propose"] == "true":
|
||||
d.proposed = append(d.proposed, args)
|
||||
return answer(true, "proposal sab12: Set the settings of betterbird?\n fingerprint ab12")
|
||||
case key == "seat:mesh-controller.settings":
|
||||
node, _ := args["node"].(string)
|
||||
if d.garbled {
|
||||
return answer(true, "Settings of betterbird on "+node+":\n accounts = {…}\n")
|
||||
}
|
||||
layer := d.layers[node]
|
||||
if layer == nil {
|
||||
where := node
|
||||
if where == "" {
|
||||
where = "the whole mesh"
|
||||
}
|
||||
return answer(true, "betterbird on "+where+": no layer — the module's definition says\nbetterbird on "+where+", every value")
|
||||
}
|
||||
b, _ := json.MarshalIndent(layer, "", " ")
|
||||
return answer(true, string(b)+"\n approved by the operator\nbetterbird on the whole mesh, every value and where it comes from:\n")
|
||||
}
|
||||
d.t.Fatalf("asked %s %v", key, body)
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func stringsOf(v any) []string {
|
||||
switch x := v.(type) {
|
||||
case []string:
|
||||
return x
|
||||
case []any:
|
||||
var out []string
|
||||
for _, s := range x {
|
||||
out = append(out, s.(string))
|
||||
}
|
||||
return out
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func TestDeriveLaysEachEntryOutPerLayerAndProposesOneLayerAtATime(t *testing.T) {
|
||||
f := newFake(t)
|
||||
f.profileWithAccounts()
|
||||
f.profileWithConfiguration()
|
||||
f.write(profileDir+"/prefs.js", f.read(profileDir+"/prefs.js")+
|
||||
`user_pref("calendar.registry.c6.type", "caldav");`+"\n"+
|
||||
`user_pref("calendar.registry.c6.uri", "https://other:hunter2@dav.example.org/cal/x/");`+"\n"+
|
||||
`user_pref("calendar.registry.c6.name", "Old");`+"\n")
|
||||
f.declareAccounts(adoptable)
|
||||
links(t, nil)
|
||||
f2 := newFake(t)
|
||||
f2.profileWithAccounts()
|
||||
f2.profileWithConfiguration()
|
||||
// shanks holds the same accounts and calendars, another signature, another tag colour and a calendar of its own.
|
||||
f2.write(profileDir+"/prefs.js", strings.NewReplacer(`"Regards"`, `"Cheers"`, `"#FF0000"`, `"#CC0000"`).Replace(f2.read(profileDir+"/prefs.js"))+
|
||||
`user_pref("calendar.registry.s1.type", "caldav");`+"\n"+
|
||||
`user_pref("calendar.registry.s1.uri", "https://dav.example.org/cal/standup/");`+"\n"+
|
||||
`user_pref("calendar.registry.s1.name", "Standup");`+"\n")
|
||||
f2.declareAccounts(adoptable)
|
||||
mesh := &derivingMesh{t: t, shanks: f2, layers: map[string]map[string]any{"": {"accounts": map[string]any{"mail": "as approved"}}}}
|
||||
oldNode, oldAsk := nodeName, asker
|
||||
t.Cleanup(func() { nodeName, asker = oldNode, oldAsk })
|
||||
nodeName = func() string { return "g14" }
|
||||
asker = mesh.ask
|
||||
|
||||
a, err := f.Derive(nil, false, false, nil, "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !reflect.DeepEqual(a.From, []string{"g14", "shanks"}) || len(a.NotHeard) != 0 {
|
||||
t.Fatalf("derived from the nodes holding Betterbird: %v %v", a.From, a.NotHeard)
|
||||
}
|
||||
whole := mustJSON(a)
|
||||
mw := a.Layers[meshWide]
|
||||
ids, _ := mw[kindIdentities].(map[string]any)
|
||||
if mail, _ := ids["mail"].(map[string]any); mail == nil || mustJSON(mail["signature"]) != `["Regards"]` || mail["address"] != nil {
|
||||
t.Errorf("an identity on both nodes is mesh-wide with this node's value, the default without its address: %v", ids)
|
||||
}
|
||||
cals, _ := mw[kindCalendars].(map[string]any)
|
||||
if cals["home"] == nil || cals["standup"] != nil {
|
||||
t.Errorf("a calendar both hold is mesh-wide, one only shanks holds is not: %v", cals)
|
||||
}
|
||||
if sh := mustJSON(a.Layers["shanks"]); !strings.Contains(sh, "https://dav.example.org/cal/standup/") {
|
||||
t.Errorf("an entry on one node only goes into that node's layer: %s", sh)
|
||||
}
|
||||
tags, _ := mw[kindTags].(map[string]any)
|
||||
if l1, _ := tags["$label1"].(map[string]any); l1 == nil || l1["color"] != "#FF0000" {
|
||||
t.Errorf("the tag is mesh-wide with this node's colour: %v", tags)
|
||||
}
|
||||
diffs := map[string]difference{}
|
||||
for _, d := range a.Differences {
|
||||
diffs[d.Kind+"/"+d.Entry+"/"+d.Field] = d
|
||||
}
|
||||
if d := diffs["identities/mail/signature"]; d.Node != "shanks" || mustJSON(d.There) != `["Cheers"]` {
|
||||
t.Errorf("the answer lists the field shanks holds differently: %+v", a.Differences)
|
||||
}
|
||||
if d := diffs["tags/$label1/color"]; d.There != "#CC0000" || d.MeshWide != "#FF0000" {
|
||||
t.Errorf("%+v", a.Differences)
|
||||
}
|
||||
for _, never := range []string{"secret-token", "hunter2", "owl://", `"Work"`} {
|
||||
if strings.Contains(mustJSON(a.Layers), never) {
|
||||
t.Errorf("no ICS address, credential or Owl calendar is derived: %s in %s", never, mustJSON(a.Layers))
|
||||
}
|
||||
}
|
||||
if !strings.Contains(whole, "Holidays") || !strings.Contains(whole, "‹withheld›") || strings.Contains(mustJSON(credentialChecked(a)), "hunter2") {
|
||||
t.Errorf("what is left out is named, an ICS calendar by its name and a password as ‹withheld›: %v", a.NotDerived)
|
||||
}
|
||||
if len(mesh.proposed) != 0 {
|
||||
t.Fatal("without propose nothing is proposed")
|
||||
}
|
||||
|
||||
// Propose: the mesh-wide layer first, as it stands with the derived kinds, and the operator's setup of prefs.
|
||||
prefs := map[string]any{}
|
||||
for n, v := range shippedDefaultsWant {
|
||||
prefs[n], _ = prefValue(v)
|
||||
}
|
||||
a, err = f.Derive(nil, false, true, prefs, "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(mesh.proposed) != 1 || mesh.proposed[0]["node"] != nil || mesh.proposed[0]["replace"] != "true" || a.Next != meshWide {
|
||||
t.Fatalf("the mesh-wide layer is proposed first: %v %s", mesh.proposed, a.Next)
|
||||
}
|
||||
var values map[string]any
|
||||
_ = json.Unmarshal([]byte(mesh.proposed[0]["values"].(string)), &values)
|
||||
if values["accounts"] == nil || values[kindIdentities] == nil || values[kindTags] == nil || values["prefs"] == nil {
|
||||
t.Fatalf("the layer as it stands, with every derived kind and prefs: %v", values)
|
||||
}
|
||||
body, _ := json.Marshal(map[string]any{"prefs": values["prefs"]})
|
||||
if got, err := parseDeclaredPrefs(body); err != nil || !reflect.DeepEqual(got, shippedDefaultsWant) {
|
||||
t.Fatalf("the operator's 21 preferences, one entry each: %v %v", got, err)
|
||||
}
|
||||
if !strings.Contains(a.Proposed, "sab12") {
|
||||
t.Fatalf("%s", a.Proposed)
|
||||
}
|
||||
|
||||
// While it waits nothing more is proposed.
|
||||
mesh.open = true
|
||||
if _, err := f.Derive(nil, false, true, nil, ""); err == nil || !strings.Contains(err.Error(), "sab12 (the whole mesh)") || len(mesh.proposed) != 1 {
|
||||
t.Fatalf("one open proposal of the module's at a time: %v", err)
|
||||
}
|
||||
// Approved: the mesh-wide layer holds it, so the next layer is shanks's own.
|
||||
mesh.open = false
|
||||
mesh.layers[""] = values
|
||||
a, err = f.Derive(nil, false, true, prefs, "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(mesh.proposed) != 2 || mesh.proposed[1]["node"] != "shanks" || a.Next != "shanks" ||
|
||||
!strings.Contains(mesh.proposed[1]["values"].(string), "standup") {
|
||||
t.Fatalf("then the node's own layer: %v", mesh.proposed)
|
||||
}
|
||||
var shanks map[string]any
|
||||
if err := json.Unmarshal([]byte(mesh.proposed[1]["values"].(string)), &shanks); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
mesh.layers["shanks"] = shanks
|
||||
if a, err = f.Derive(nil, false, true, prefs, ""); err != nil || len(mesh.proposed) != 2 || !strings.HasPrefix(a.Next, "none") {
|
||||
t.Fatalf("nothing left to propose: %v %s", err, a.Next)
|
||||
}
|
||||
|
||||
// A desktop node that does not answer: derived without it, never proposed.
|
||||
mesh.asleep = true
|
||||
a, err = f.Derive(nil, false, true, nil, meshWide)
|
||||
if err == nil || len(a.NotHeard) != 1 || !strings.Contains(err.Error(), "did not answer") || len(mesh.proposed) != 2 {
|
||||
t.Fatalf("a layer is never proposed without a desktop node: %v %v", err, a.NotHeard)
|
||||
}
|
||||
|
||||
// A key to adopt is a node's: the mesh-wide layer's proposal refuses one.
|
||||
if err := proposable(map[string]any{kindCalendars: map[string]any{"c": map[string]any{"name": "C",
|
||||
"url": "https://dav.example.org/c/", "key": "c3"}}}, true); err == nil || !strings.Contains(err.Error(), "node's own layer") {
|
||||
t.Fatalf("%v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Review 121: a layer the controller answers in any other words than its JSON or "no layer" refuses the proposal,
|
||||
// because a proposal replaces the layer whole and would drop accounts.
|
||||
func TestALayerThatCannotBeReadRefusesTheProposal(t *testing.T) {
|
||||
f := newFake(t)
|
||||
f.profileWithAccounts()
|
||||
f.profileWithConfiguration()
|
||||
f.declareAccounts(adoptable)
|
||||
links(t, nil)
|
||||
f2 := newFake(t)
|
||||
f2.profileWithAccounts()
|
||||
f2.declareAccounts(adoptable)
|
||||
mesh := &derivingMesh{t: t, shanks: f2, garbled: true}
|
||||
oldNode, oldAsk := nodeName, asker
|
||||
t.Cleanup(func() { nodeName, asker = oldNode, oldAsk })
|
||||
nodeName = func() string { return "g14" }
|
||||
asker = mesh.ask
|
||||
if _, err := f.Derive(nil, false, true, nil, ""); err == nil || !strings.Contains(err.Error(), "cannot be read as the layer") ||
|
||||
len(mesh.proposed) != 0 {
|
||||
t.Fatalf("the derive refuses: %v %v", err, mesh.proposed)
|
||||
}
|
||||
if _, err := f.ProfileAccounts(true); err == nil || !strings.Contains(err.Error(), "cannot be read as the layer") ||
|
||||
len(mesh.proposed) != 0 {
|
||||
t.Fatalf("betterbird_accounts refuses: %v %v", err, mesh.proposed)
|
||||
}
|
||||
if _, err := readLayer(""); err == nil {
|
||||
t.Fatal("an unreadable layer is an error, never an empty layer")
|
||||
}
|
||||
mesh.garbled = false
|
||||
if got, err := readLayer("shanks"); err != nil || len(got) != 0 {
|
||||
t.Fatalf("no layer, in the controller's words, is an empty layer: %v %v", got, err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,271 @@
|
||||
package main
|
||||
|
||||
// An account's filters, owned whole per declared account (novox/hq ADR 0329 §8).
|
||||
//
|
||||
// Betterbird keeps an account's filters in msgFilterRules.dat in the account's own directory: a header, then per
|
||||
// filter its name, whether it is on, when it runs (a bit mask), its actions and one condition line. For an account
|
||||
// the setting filters names, the module writes that file from the declared list, naming each folder by its path
|
||||
// within the account and turning it into the folder's address on this node; a filter in the file the setting does
|
||||
// not hold is gone after the apply. Before its first write on a node the module keeps the file it replaces, in its
|
||||
// own directory in the home (ADR 0182). It writes only while Betterbird does not run, because Betterbird rewrites
|
||||
// the file itself (§9).
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// filterFileName is the file an account's filters are in.
|
||||
const filterFileName = "msgFilterRules.dat"
|
||||
|
||||
// accountDirectory is an account's directory in the profile, as its server's preferences name it, or "".
|
||||
func accountDirectory(profile string, prefs map[string]string, server string) string {
|
||||
p := "mail.server." + server + "."
|
||||
if rel := unquote(prefs[p+"directory-rel"]); strings.HasPrefix(rel, "[ProfD]") {
|
||||
dir := filepath.Join(profile, filepath.FromSlash(strings.TrimPrefix(rel, "[ProfD]")))
|
||||
if strings.HasPrefix(filepath.Clean(dir), filepath.Clean(profile)+string(filepath.Separator)) {
|
||||
return dir
|
||||
}
|
||||
return ""
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// folderBase is the address of an account's root folder on this node: imap://user@host for IMAP, mailbox://
|
||||
// for POP3 and local folders, the user's @ written %40 as Betterbird writes it.
|
||||
func folderBase(prefs map[string]string, server string) string {
|
||||
p := "mail.server." + server + "."
|
||||
kind := unquote(prefs[p+"type"])
|
||||
host := unquote(prefs[p+"hostname"])
|
||||
user := unquote(prefs[p+"userName"])
|
||||
scheme := "mailbox"
|
||||
switch kind {
|
||||
case "imap":
|
||||
scheme = "imap"
|
||||
case "none":
|
||||
host, user = "Local Folders", "nobody"
|
||||
}
|
||||
return scheme + "://" + strings.ReplaceAll(url.PathEscape(user), "@", "%40") + "@" + url.PathEscape(host)
|
||||
}
|
||||
|
||||
// folderAddress is a folder's address on this node from its path within the account.
|
||||
func folderAddress(base, path string) string {
|
||||
parts := strings.Split(strings.Trim(path, "/"), "/")
|
||||
for i, p := range parts {
|
||||
parts[i] = url.PathEscape(p)
|
||||
}
|
||||
return base + "/" + strings.Join(parts, "/")
|
||||
}
|
||||
|
||||
// folderPath is a folder's path within the account from its address, when the address is the account's.
|
||||
func folderPath(base, address string) (string, bool) {
|
||||
rest, ok := strings.CutPrefix(address, base+"/")
|
||||
if !ok {
|
||||
return "", false
|
||||
}
|
||||
parts := strings.Split(rest, "/")
|
||||
for i, p := range parts {
|
||||
u, err := url.PathUnescape(p)
|
||||
if err != nil {
|
||||
return "", false
|
||||
}
|
||||
parts[i] = u
|
||||
}
|
||||
return strings.Join(parts, "/"), true
|
||||
}
|
||||
|
||||
// whenOf is a filter's type mask from its words; whenWords the reverse, or false when the mask holds a bit no
|
||||
// word says (a news filter's).
|
||||
func whenOf(words []string) int {
|
||||
n := 0
|
||||
for _, w := range words {
|
||||
n |= filterWhen[w]
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
func whenWords(mask int) ([]string, bool) {
|
||||
var out []string
|
||||
left := mask
|
||||
for _, w := range []string{"new-mail", "manual", "after-junk", "sending", "archiving", "periodic"} {
|
||||
if mask&filterWhen[w] != 0 {
|
||||
out = append(out, w)
|
||||
left &^= filterWhen[w]
|
||||
}
|
||||
}
|
||||
return out, left == 0
|
||||
}
|
||||
|
||||
// renderFilters is the file Betterbird reads for a declared list of filters on this node.
|
||||
func renderFilters(filters []Filter, base string) string {
|
||||
var b strings.Builder
|
||||
b.WriteString("version=\"9\"\nlogging=\"no\"\n")
|
||||
for _, f := range filters {
|
||||
fmt.Fprintf(&b, "name=\"%s\"\n", f.Name)
|
||||
fmt.Fprintf(&b, "enabled=\"%s\"\n", map[bool]string{true: "yes", false: "no"}[f.Enabled])
|
||||
fmt.Fprintf(&b, "type=\"%d\"\n", whenOf(f.When))
|
||||
for _, a := range f.Actions {
|
||||
fmt.Fprintf(&b, "action=\"%s\"\n", a.Action)
|
||||
switch {
|
||||
case a.Folder != "":
|
||||
fmt.Fprintf(&b, "actionValue=\"%s\"\n", folderAddress(base, a.Folder))
|
||||
case a.Value != "":
|
||||
fmt.Fprintf(&b, "actionValue=\"%s\"\n", a.Value)
|
||||
}
|
||||
}
|
||||
fmt.Fprintf(&b, "condition=\"%s\"\n", conditionText(f))
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
func conditionText(f Filter) string {
|
||||
if f.Match == "every-message" {
|
||||
return "ALL"
|
||||
}
|
||||
join := "AND"
|
||||
if f.Match == "any" {
|
||||
join = "OR"
|
||||
}
|
||||
var parts []string
|
||||
for _, c := range f.Conditions {
|
||||
parts = append(parts, fmt.Sprintf("%s (%s,%s,%s)", join, c.Field, c.Op, c.Value))
|
||||
}
|
||||
return strings.Join(parts, " ")
|
||||
}
|
||||
|
||||
var (
|
||||
filterLine = regexp.MustCompile(`^([A-Za-z]+)="((?:[^"\\]|\\.)*)"$`)
|
||||
clause = regexp.MustCompile(`(AND|OR) \(([^,()]+),([^,()]+),([^()]*)\)`)
|
||||
)
|
||||
|
||||
// parsedFilters is a filter file read into the setting's shape. NotDeclarable names, by the filter's name, the
|
||||
// filters a setting cannot hold as they are (a folder of another account, an action of an add-on, a news filter).
|
||||
type parsedFilters struct {
|
||||
Filters []Filter
|
||||
NotDeclarable []string
|
||||
}
|
||||
|
||||
// parseFilters reads a filter file for an account whose folders are under base.
|
||||
func parseFilters(text, base string) parsedFilters {
|
||||
var out parsedFilters
|
||||
var cur *Filter
|
||||
bad := ""
|
||||
finish := func() {
|
||||
if cur == nil {
|
||||
return
|
||||
}
|
||||
if bad != "" {
|
||||
out.NotDeclarable = append(out.NotDeclarable, cur.Name+": "+bad)
|
||||
} else if p := filterProblems(*cur); len(p) > 0 {
|
||||
out.NotDeclarable = append(out.NotDeclarable, cur.Name+": "+p[0])
|
||||
} else {
|
||||
out.Filters = append(out.Filters, *cur)
|
||||
}
|
||||
cur, bad = nil, ""
|
||||
}
|
||||
s := bufio.NewScanner(strings.NewReader(text))
|
||||
s.Buffer(make([]byte, 64<<10), 4<<20)
|
||||
for s.Scan() {
|
||||
mm := filterLine.FindStringSubmatch(strings.TrimSpace(s.Text()))
|
||||
if mm == nil {
|
||||
continue
|
||||
}
|
||||
field, value := mm[1], strings.NewReplacer(`\"`, `"`, `\\`, `\`).Replace(mm[2])
|
||||
if field == "name" {
|
||||
finish()
|
||||
cur = &Filter{Name: value}
|
||||
continue
|
||||
}
|
||||
if cur == nil {
|
||||
continue
|
||||
}
|
||||
switch field {
|
||||
case "enabled":
|
||||
cur.Enabled = value == "yes"
|
||||
case "type":
|
||||
n, _ := strconv.Atoi(value)
|
||||
words, ok := whenWords(n)
|
||||
if !ok && bad == "" {
|
||||
bad = "it runs on news or at a time no setting word says"
|
||||
}
|
||||
cur.When = words
|
||||
case "action":
|
||||
cur.Actions = append(cur.Actions, Action{Action: value})
|
||||
case "actionValue":
|
||||
if len(cur.Actions) == 0 {
|
||||
continue
|
||||
}
|
||||
a := &cur.Actions[len(cur.Actions)-1]
|
||||
if filterActions[a.Action] == "folder" {
|
||||
path, ok := folderPath(base, value)
|
||||
if !ok && bad == "" {
|
||||
bad = "it moves or copies to a folder of another account"
|
||||
}
|
||||
a.Folder = path
|
||||
} else {
|
||||
a.Value = value
|
||||
}
|
||||
case "condition":
|
||||
switch {
|
||||
case value == "ALL":
|
||||
cur.Match = "every-message"
|
||||
default:
|
||||
all := clause.FindAllStringSubmatch(value, -1)
|
||||
if len(all) == 0 && bad == "" {
|
||||
bad = "its condition cannot be read"
|
||||
}
|
||||
for _, c := range all {
|
||||
if c[1] == "OR" {
|
||||
cur.Match = "any"
|
||||
} else if cur.Match == "" {
|
||||
cur.Match = "all"
|
||||
}
|
||||
cur.Conditions = append(cur.Conditions, Condition{Field: c[2], Op: c[3], Value: c[4]})
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
finish()
|
||||
return out
|
||||
}
|
||||
|
||||
// filterDifferences says how a file's filters differ from the declared list, by place and name: missing,
|
||||
// not declared, or differing.
|
||||
func filterDifferences(declared, have []Filter) []string {
|
||||
var out []string
|
||||
for i := 0; i < len(declared) || i < len(have); i++ {
|
||||
switch {
|
||||
case i >= len(have):
|
||||
out = append(out, fmt.Sprintf("filter %d (%s) is declared and not in the file", i+1, declared[i].Name))
|
||||
case i >= len(declared):
|
||||
out = append(out, fmt.Sprintf("filter %d (%s) is in the file and not declared: removed at the next apply", i+1, have[i].Name))
|
||||
default:
|
||||
d, h := mustJSON(declared[i]), mustJSON(have[i])
|
||||
if d != h {
|
||||
out = append(out, fmt.Sprintf("filter %d: declared %s, the file holds %s", i+1, d, h))
|
||||
}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// keptFilterFile is where the module keeps the file it replaced at its first write of an account's filters.
|
||||
func (m *Machine) keptFilterFile(profileDir, account string) string {
|
||||
base := filepath.Base(profileDir)
|
||||
return m.home(".config/betterbird/kept", base, account+"."+filterFileName)
|
||||
}
|
||||
|
||||
func sortedAccountsOf(fs map[string][]Filter) []string {
|
||||
out := make([]string, 0, len(fs))
|
||||
for k := range fs {
|
||||
out = append(out, k)
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,307 @@
|
||||
package main
|
||||
|
||||
// What the mesh installs (novox/hq ADR 0316 §1–§9, to-be 56 §1–§3): Betterbird from its pinned upstream release
|
||||
// in /opt/betterbird, where the community package betterbird-bin put it, so Betterbird keeps choosing the same
|
||||
// profile; the enterprise policy in its distribution directory, which turns Betterbird's own updater off and
|
||||
// installs Owl into every profile from the xpi the mesh placed beside it; and a defaults file naming the
|
||||
// system's spelling dictionaries. The tools read these files and say what they find: the version against the
|
||||
// pin and the newest published build, whether the policy is placed, Owl's version against its pin.
|
||||
//
|
||||
// **The pin on the machine is what the mesh placed.** The manifest pins the release and the xpi by sha256; the
|
||||
// node-engine places exactly those bytes, so the version in /opt/betterbird/application.ini (when the mesh
|
||||
// placed it) and the version in the placed xpi are the pins. The community package's own copy is said as
|
||||
// such, and the switch from it waits for the operator (ADR 0316 §4).
|
||||
|
||||
import (
|
||||
"archive/zip"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
const (
|
||||
installDir = "/opt/betterbird"
|
||||
// policyFile is the enterprise policy the module places (ADR 0316 §7).
|
||||
policyFile = installDir + "/distribution/policies.json"
|
||||
// owlFile is Owl's xpi as the mesh placed it, and the URL the policy installs it from.
|
||||
owlFile = installDir + "/owl/owl.xpi"
|
||||
owlID = "owl@beonex.com"
|
||||
// defaultClientPref is Betterbird's ask at its start to be the default mail, news, feed and calendar
|
||||
// client (Thunderbird's all-thunderbird.js: pref("mail.shell.checkDefaultClient", true); messenger.js opens
|
||||
// systemIntegrationDialog.xhtml while it is true). The module's policy locks it off (novox/hq issue 345).
|
||||
defaultClientPref = "mail.shell.checkDefaultClient"
|
||||
// preparation is the tools' program as the module places it for its start (betterbird-tools prepare).
|
||||
preparation = "/usr/local/lib/mesh-betterbird/betterbird-tools"
|
||||
// versionsURL is where Betterbird publishes its newest build per major version: what its own updater
|
||||
// reads (MailGlue's BetterbirdUpdateCheck), and what the status compares the installed build with.
|
||||
versionsURL = "https://www.betterbird.eu/start/versions.txt"
|
||||
)
|
||||
|
||||
// publishedBuilds fetches Betterbird's list of newest builds: a variable so a test answers.
|
||||
var publishedBuilds = func() (string, error) {
|
||||
client := &http.Client{Timeout: 5 * time.Second}
|
||||
resp, err := client.Get(versionsURL)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return "", fmt.Errorf("%s answered %s", versionsURL, resp.Status)
|
||||
}
|
||||
body, err := io.ReadAll(io.LimitReader(resp.Body, 1<<20))
|
||||
return string(body), err
|
||||
}
|
||||
|
||||
// Installation is what is installed, and from where.
|
||||
type Installation struct {
|
||||
// From is the mesh, the community package (betterbird-bin, before the switch), or nothing.
|
||||
From string `json:"from"`
|
||||
Version string `json:"version,omitempty"`
|
||||
Build string `json:"build,omitempty"`
|
||||
// Newest is the newest build Betterbird published for this major version, and whether this one is behind.
|
||||
Newest string `json:"newest_published_build,omitempty"`
|
||||
Behind bool `json:"behind,omitempty"`
|
||||
// Policy says whether the module's enterprise policy is placed, and what it does.
|
||||
Policy string `json:"policy"`
|
||||
// Owl is the add-on's version as the mesh placed it (the pin), and as the profile runs it.
|
||||
Owl *OwlState `json:"owl,omitempty"`
|
||||
}
|
||||
|
||||
// OwlState is Owl against its pin.
|
||||
type OwlState struct {
|
||||
Pinned string `json:"pinned,omitempty"`
|
||||
InProfile string `json:"in_profile,omitempty"`
|
||||
Active bool `json:"active"`
|
||||
AtThePin bool `json:"at_the_pin"`
|
||||
// Note says, as a fact, a profile running an Owl newer than the pin: the policy's install keeps a newer
|
||||
// version, and Owl installed by policy cannot be removed in Betterbird.
|
||||
Note string `json:"note,omitempty"`
|
||||
}
|
||||
|
||||
// newerVersion says whether version a is newer than b, comparing dotted numbers.
|
||||
func newerVersion(a, b string) bool {
|
||||
if a == "" || b == "" {
|
||||
return false
|
||||
}
|
||||
pa, pb := strings.Split(a, "."), strings.Split(b, ".")
|
||||
for i := 0; i < len(pa) || i < len(pb); i++ {
|
||||
var x, y int
|
||||
if i < len(pa) {
|
||||
fmt.Sscanf(pa[i], "%d", &x)
|
||||
}
|
||||
if i < len(pb) {
|
||||
fmt.Sscanf(pb[i], "%d", &y)
|
||||
}
|
||||
if x != y {
|
||||
return x > y
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// installation reads what is installed. profile may be empty when none is found yet.
|
||||
func (m *Machine) installation(profile string) Installation {
|
||||
var in Installation
|
||||
community, _ := m.installed(packageFor)
|
||||
raw, err := readBounded(m.path(appIni))
|
||||
switch {
|
||||
case community != "":
|
||||
in.From = "the community package " + packageFor + " " + community + ": the switch to the mesh's release waits for the operator's push, with Betterbird quit (novox/hq ADR 0316 §4)"
|
||||
case err == nil:
|
||||
in.From = "the mesh"
|
||||
default:
|
||||
in.From = "not installed"
|
||||
}
|
||||
if err == nil {
|
||||
_, sec := iniSections(raw)
|
||||
in.Version, in.Build = sec["App"]["Version"], sec["App"]["BuildID"]
|
||||
in.Newest, in.Behind = newestBuild(in.Version, in.Build)
|
||||
}
|
||||
in.Policy = m.policyState()
|
||||
pinned := xpiVersion(m.path(owlFile))
|
||||
owl := &OwlState{Pinned: pinned}
|
||||
if profile != "" {
|
||||
for _, a := range addOns(profile) {
|
||||
if a.ID == owlID {
|
||||
owl.InProfile, owl.Active = a.Version, a.Active
|
||||
}
|
||||
}
|
||||
}
|
||||
owl.AtThePin = owl.Pinned != "" && owl.Pinned == owl.InProfile
|
||||
if newerVersion(owl.InProfile, owl.Pinned) {
|
||||
owl.Note = "the profile runs Owl " + owl.InProfile + ", newer than the mesh's pin " + owl.Pinned + ": the policy " +
|
||||
"keeps a newer version and installs the pin only where Owl is missing. To run the same Owl everywhere, " +
|
||||
"move the module's pin in a reviewed change"
|
||||
}
|
||||
if owl.Pinned != "" || owl.InProfile != "" {
|
||||
in.Owl = owl
|
||||
}
|
||||
return in
|
||||
}
|
||||
|
||||
// newestBuild is the newest build Betterbird published for a version's major, and whether build is older.
|
||||
func newestBuild(version, build string) (string, bool) {
|
||||
major, _, _ := strings.Cut(version, ".")
|
||||
if major == "" || build == "" {
|
||||
return "", false
|
||||
}
|
||||
list, err := publishedBuilds()
|
||||
if err != nil {
|
||||
return "unknown: " + err.Error(), false
|
||||
}
|
||||
for _, l := range strings.Split(list, "\n") {
|
||||
f := strings.Split(strings.TrimSpace(l), ",")
|
||||
if len(f) == 3 && f[0] == "Linux" && f[1] == major {
|
||||
return f[2], f[2] > build
|
||||
}
|
||||
}
|
||||
return "unknown: " + versionsURL + " names no build for " + major, false
|
||||
}
|
||||
|
||||
// policyState says whether the module's policy is placed and does what ADR 0316 §5 and §7 and issue 345 ask.
|
||||
func (m *Machine) policyState() string {
|
||||
raw, err := readBounded(m.path(policyFile))
|
||||
if err != nil {
|
||||
return "not placed: " + policyFile + " is missing"
|
||||
}
|
||||
if why := policyProblem(raw); why != "" {
|
||||
return "placed, and " + why
|
||||
}
|
||||
return "placed: Betterbird's updater, its telemetry and add-on updates off, Owl installed into every profile from " +
|
||||
"the mesh's xpi with its updates off, and the ask to be the default mail client, the vendor's in-app notices " +
|
||||
"and the start page locked off"
|
||||
}
|
||||
|
||||
// lockedOff are the preferences the module's policy locks to false (novox/hq issue 345), each with what it is.
|
||||
var lockedOff = []struct{ Name, What string }{
|
||||
{defaultClientPref, "the ask to be the default mail client"},
|
||||
{"mail.inappnotifications.enabled", "the vendor's in-app notices"},
|
||||
{"mailnews.start_page.enabled", "the start page, loaded from the vendor at every start"},
|
||||
}
|
||||
|
||||
// policyProblem is everything the policy fails to say of what ADR 0316 and issue 345 ask, or "".
|
||||
func policyProblem(raw []byte) string {
|
||||
var p struct {
|
||||
Policies struct {
|
||||
DisableAppUpdate bool `json:"DisableAppUpdate"`
|
||||
DisableTelemetry bool `json:"DisableTelemetry"`
|
||||
ExtensionUpdate *bool `json:"ExtensionUpdate"`
|
||||
ExtensionSettings map[string]struct {
|
||||
Mode string `json:"installation_mode"`
|
||||
URL string `json:"install_url"`
|
||||
Disabled bool `json:"updates_disabled"`
|
||||
} `json:"ExtensionSettings"`
|
||||
} `json:"policies"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &p); err != nil {
|
||||
return "it is not JSON: " + err.Error()
|
||||
}
|
||||
var problems []string
|
||||
if !p.Policies.DisableAppUpdate {
|
||||
problems = append(problems, "it does not turn Betterbird's updater off (DisableAppUpdate)")
|
||||
}
|
||||
if !p.Policies.DisableTelemetry {
|
||||
problems = append(problems, "it does not turn telemetry off (DisableTelemetry)")
|
||||
}
|
||||
if p.Policies.ExtensionUpdate == nil || *p.Policies.ExtensionUpdate {
|
||||
problems = append(problems, "it leaves add-ons updating themselves (ExtensionUpdate)")
|
||||
}
|
||||
owl, ok := p.Policies.ExtensionSettings[owlID]
|
||||
switch {
|
||||
case !ok || owl.Mode != "force_installed":
|
||||
problems = append(problems, "it does not install Owl into every profile (force_installed)")
|
||||
case owl.URL != "file://"+owlFile:
|
||||
problems = append(problems, "it installs Owl from "+owl.URL+", not from the mesh's xpi")
|
||||
case !owl.Disabled:
|
||||
problems = append(problems, "it leaves Owl's updates on")
|
||||
}
|
||||
prefs := policyPrefsOf(raw)
|
||||
for _, l := range lockedOff {
|
||||
if pr, ok := prefs[l.Name]; !ok || pr.Value != "false" || !pr.Locked {
|
||||
problems = append(problems, "it does not lock "+l.What+" off (Preferences "+l.Name+": false, locked)")
|
||||
}
|
||||
}
|
||||
return strings.Join(problems, "; ")
|
||||
}
|
||||
|
||||
// defaultClientAsk says why Betterbird asks at its start to be the default mail client, or "" when it does
|
||||
// not: the placed policy's own Preferences entry locks the ask off, else the profile's user.js (which wins at
|
||||
// the start) or prefs.js turns it off. Betterbird's own default is to ask. Decided from the entry itself, never
|
||||
// from what else the policy lacks.
|
||||
func (m *Machine) defaultClientAsk(profile string) string {
|
||||
if v, ok := m.locks()[defaultClientPref]; ok && v == "false" {
|
||||
return ""
|
||||
}
|
||||
if profile != "" {
|
||||
for _, f := range []string{"user.js", "prefs.js"} {
|
||||
b, err := readBounded(filepath.Join(profile, f))
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if v, ok := statementsOf(b)[defaultClientPref]; ok {
|
||||
if v == "false" {
|
||||
return ""
|
||||
}
|
||||
return "the profile's " + f + " sets " + defaultClientPref + " to " + v + " and no policy locks it off"
|
||||
}
|
||||
}
|
||||
}
|
||||
return "no policy locks " + defaultClientPref + " off and Betterbird's default is to ask"
|
||||
}
|
||||
|
||||
// xpiVersion is the version an add-on's manifest.json says, or "" when it cannot be read.
|
||||
func xpiVersion(path string) string {
|
||||
r, err := zip.OpenReader(path)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
defer r.Close()
|
||||
for _, f := range r.File {
|
||||
if f.Name != "manifest.json" {
|
||||
continue
|
||||
}
|
||||
rc, err := f.Open()
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
defer rc.Close()
|
||||
var mf struct {
|
||||
Version string `json:"version"`
|
||||
}
|
||||
if json.NewDecoder(io.LimitReader(rc, 1<<20)).Decode(&mf) != nil {
|
||||
return ""
|
||||
}
|
||||
return mf.Version
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// addOns reads a profile's extensions.json: each add-on's id, version, whether it is on, and where from.
|
||||
func addOns(profile string) []AddOn {
|
||||
raw, err := readBounded(filepath.Join(profile, "extensions.json"))
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
var ext struct {
|
||||
Addons []struct {
|
||||
ID string `json:"id"`
|
||||
Version string `json:"version"`
|
||||
Active bool `json:"active"`
|
||||
Location string `json:"location"`
|
||||
} `json:"addons"`
|
||||
}
|
||||
if json.Unmarshal(raw, &ext) != nil {
|
||||
return nil
|
||||
}
|
||||
var out []AddOn
|
||||
for _, a := range ext.Addons {
|
||||
if a.Location == "app-profile" {
|
||||
out = append(out, AddOn{ID: a.ID, Version: a.Version, Active: a.Active})
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,231 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"archive/zip"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// What the mesh installs, as the tools say it (novox/hq ADR 0316 §16): the version against the pin and the
|
||||
// newest published build, the policy, Owl against its pin; and the reminder test, visibly a test.
|
||||
|
||||
func (f *fake) xpi(path, version string) {
|
||||
f.t.Helper()
|
||||
full := filepath.Join(f.Root, path)
|
||||
if err := os.MkdirAll(filepath.Dir(full), 0o755); err != nil {
|
||||
f.t.Fatal(err)
|
||||
}
|
||||
out, err := os.Create(full)
|
||||
if err != nil {
|
||||
f.t.Fatal(err)
|
||||
}
|
||||
w := zip.NewWriter(out)
|
||||
m, _ := w.Create("manifest.json")
|
||||
m.Write([]byte(`{"version": "` + version + `", "browser_specific_settings": {"gecko": {"id": "owl@beonex.com"}}}`))
|
||||
w.Close()
|
||||
out.Close()
|
||||
}
|
||||
|
||||
// placedByTheMesh is a node after the switch: the release, the policy and Owl's xpi in /opt/betterbird, and
|
||||
// the profile running Owl at the pin.
|
||||
func (f *fake) placedByTheMesh() {
|
||||
f.noPackage()
|
||||
f.write(appIni, "[App]\nVendor=Betterbird Project\nVersion=153.4.0\nBuildID=20260929222610\n")
|
||||
raw, err := os.ReadFile(filepath.Join("..", "..", "install", "distribution", "policies.json"))
|
||||
if err != nil {
|
||||
f.t.Fatal(err)
|
||||
}
|
||||
f.write(policyFile, string(raw))
|
||||
f.xpi(owlFile, "1.5.6.1")
|
||||
f.write(profileDir+"/extensions.json", `{"addons":[{"id":"owl@beonex.com","version":"1.5.6.1","active":true,"location":"app-profile"}]}`)
|
||||
}
|
||||
|
||||
func TestTheStatusSaysTheInstallationAgainstThePinTheNewestBuildThePolicyAndOwl(t *testing.T) {
|
||||
f := newFake(t)
|
||||
f.betterbird()
|
||||
f.placedByTheMesh()
|
||||
links(t, nil)
|
||||
publishedBuilds = func() (string, error) {
|
||||
return "blocklist,{}\nLinux,153,20261101000000\nLinux,140,20260829111240\n", nil
|
||||
}
|
||||
t.Cleanup(func() { publishedBuilds = func() (string, error) { return "", os.ErrDeadlineExceeded } })
|
||||
s, err := f.Status()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
in := s.Installation
|
||||
if in.From != "the mesh" || in.Version != "153.4.0" || in.Newest != "20261101000000" || !in.Behind ||
|
||||
!strings.HasPrefix(in.Policy, "placed:") || in.Owl == nil || in.Owl.Pinned != "1.5.6.1" || !in.Owl.AtThePin || !in.Owl.Active {
|
||||
t.Fatalf("%+v %+v", in, in.Owl)
|
||||
}
|
||||
c, err := f.Check()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
found := false
|
||||
for _, x := range c.Findings {
|
||||
found = found || strings.Contains(x.What, "behind the newest published build 20261101000000")
|
||||
}
|
||||
if !found {
|
||||
t.Fatalf("a release left behind is said: %+v", c.Findings)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheCommunityPackageAPolicyThatIsNotTheModulesAndAnOwlOffItsPinAreSaid(t *testing.T) {
|
||||
f := newFake(t)
|
||||
f.betterbird()
|
||||
f.placedByTheMesh()
|
||||
links(t, nil)
|
||||
f.write(policyFile, `{"policies": {"ExtensionSettings": {"owl@beonex.com": {"installation_mode": "force_installed",
|
||||
"install_url": "https://vendor.example/owl.xpi", "updates_disabled": true}}}}`)
|
||||
f.write(profileDir+"/extensions.json", `{"addons":[{"id":"owl@beonex.com","version":"1.5.7","active":true,"location":"app-profile"}]}`)
|
||||
in := f.installation(filepath.Join(f.Root, profileDir))
|
||||
if !strings.Contains(in.Policy, "DisableAppUpdate") || in.Owl.AtThePin || in.Owl.InProfile != "1.5.7" {
|
||||
t.Fatalf("%+v %+v", in, in.Owl)
|
||||
}
|
||||
// A newer Owl than the pin is a fact with the real remedy, never a finding to remove it.
|
||||
if !strings.Contains(in.Owl.Note, "move the module's pin") {
|
||||
t.Fatalf("%+v", in.Owl)
|
||||
}
|
||||
f.write(policyFile, `{"policies": {"DisableAppUpdate": true, "ExtensionSettings": {"owl@beonex.com": {"installation_mode": "force_installed",
|
||||
"install_url": "file:///opt/betterbird/owl/owl.xpi", "updates_disabled": true}}}}`)
|
||||
c, _ := f.Check()
|
||||
for _, x := range c.Findings {
|
||||
if strings.Contains(x.What, "Owl") {
|
||||
t.Errorf("a newer Owl is not a finding: %+v", x)
|
||||
}
|
||||
}
|
||||
f.write(profileDir+"/extensions.json", `{"addons":[{"id":"owl@beonex.com","version":"1.5.5","active":true,"location":"app-profile"}]}`)
|
||||
c, _ = f.Check()
|
||||
found := false
|
||||
for _, x := range c.Findings {
|
||||
found = found || (strings.Contains(x.What, "older than the mesh's pin") && strings.Contains(x.Do, "move the module's Owl pin"))
|
||||
}
|
||||
if !found {
|
||||
t.Fatalf("an older Owl is a finding whose remedy is the pin: %+v", c.Findings)
|
||||
}
|
||||
f.write(policyFile, `{"policies": {"DisableAppUpdate": true, "ExtensionSettings": {"owl@beonex.com": {"installation_mode": "force_installed",
|
||||
"install_url": "https://vendor.example/owl.xpi", "updates_disabled": true}}}}`)
|
||||
if in := f.installation(""); !strings.Contains(in.Policy, "not from the mesh's xpi") {
|
||||
t.Fatalf("%s", in.Policy)
|
||||
}
|
||||
f.answer = func(name string, args []string) Output {
|
||||
if name == "pacman" && args[0] == "-Q" {
|
||||
return Output{Stdout: "betterbird-bin 153.1.0esr-1\n"}
|
||||
}
|
||||
return Output{}
|
||||
}
|
||||
in = f.installation("")
|
||||
if !strings.HasPrefix(in.From, "the community package betterbird-bin 153.1.0esr-1") || !strings.Contains(in.From, "operator") {
|
||||
t.Fatalf("%s", in.From)
|
||||
}
|
||||
raw, _ := json.Marshal(in)
|
||||
if strings.Contains(string(raw), "example.org") {
|
||||
t.Fatalf("%s", raw)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheReminderTestIsVisiblyATestAndSaysItOpensAWindow(t *testing.T) {
|
||||
f := newFake(t)
|
||||
f.desktopSession()
|
||||
f.betterbird()
|
||||
links(t, nil)
|
||||
f.declare("// none")
|
||||
f.declareAccounts(personal)
|
||||
a, err := f.ReminderTest(40*time.Second, nil, false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
dir := filepath.Join("/run/user/1000", testUnit)
|
||||
css := f.read(dir + "/profile/chrome/userChrome.css")
|
||||
user := f.read(dir + "/profile/user.js")
|
||||
cal := f.read(dir + "/test.ics")
|
||||
if !strings.Contains(css, "MESH TEST") || !strings.Contains(css, `:root:not([windowtype="Calendar:AlarmWindow"])`) {
|
||||
t.Fatalf("the test's windows but the reminder carry a MESH TEST band: %s", css)
|
||||
}
|
||||
if !strings.Contains(user, `user_pref("toolkit.legacyUserProfileCustomizations.stylesheets", true);`) ||
|
||||
strings.Contains(user, "mesh_personal") || strings.Contains(user, "example.org") {
|
||||
t.Fatalf("the test profile reads its userChrome.css and holds no declared account: %s", user)
|
||||
}
|
||||
if !strings.Contains(cal, "SUMMARY:MESH TEST reminder") || !strings.Contains(a.OnScreen, "operator's screen") {
|
||||
t.Fatalf("the event and the answer say it is a test on the operator's screen: %s %+v", cal, a)
|
||||
}
|
||||
if !f.called("systemd-run --user --collect --quiet --unit="+testUnit) || !strings.Contains(strings.Join(f.calls, "\n"), startBin+" --no-remote") {
|
||||
t.Fatalf("the test starts Betterbird itself, never the module's start: %v", f.calls)
|
||||
}
|
||||
for _, tool := range tools() {
|
||||
if tool.Name == "betterbird_reminder_test" && !strings.HasPrefix(tool.Description, "OPENS WINDOWS ON THE OPERATOR'S SCREEN") {
|
||||
t.Fatalf("the description says first that a window opens on the operator's screen: %s", tool.Description)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheUserJSDescriptionSaysAdoption(t *testing.T) {
|
||||
for _, tool := range tools() {
|
||||
if tool.Name == "betterbird_user_js" && (!strings.Contains(tool.Description, "adopted under its own keys") ||
|
||||
strings.Contains(tool.Description, "refuses the apply")) {
|
||||
t.Fatalf("%s", tool.Description)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Betterbird never asks at its start to be the default mail client: the module's policy locks
|
||||
// mail.shell.checkDefaultClient off, and the check says so when it is on (novox/hq issue 345).
|
||||
func TestTheDefaultClientAskIsLockedOffAndTheCheckSaysWhenItIsOn(t *testing.T) {
|
||||
f := newFake(t)
|
||||
f.betterbird()
|
||||
f.placedByTheMesh()
|
||||
links(t, nil)
|
||||
asks := func() string {
|
||||
t.Helper()
|
||||
c, err := f.Check()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, x := range c.Findings {
|
||||
if strings.Contains(x.What, "asks at its start") {
|
||||
return x.What + " | " + x.Do
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
if got := asks(); got != "" {
|
||||
t.Fatalf("the module's policy locks the ask off, yet: %s", got)
|
||||
}
|
||||
unlocked := `{"policies": {"DisableAppUpdate": true, "ExtensionSettings": {"owl@beonex.com": {"installation_mode": "force_installed",
|
||||
"install_url": "file:///opt/betterbird/owl/owl.xpi", "updates_disabled": true}}}}`
|
||||
if why := policyProblem([]byte(unlocked)); !strings.Contains(why, defaultClientPref) {
|
||||
t.Fatalf("a policy that leaves the ask on is said: %q", why)
|
||||
}
|
||||
f.write(policyFile, unlocked)
|
||||
if got := asks(); !strings.Contains(got, "default mail client") || !strings.Contains(got, "push the module") {
|
||||
t.Fatalf("the ask on is a finding with its remedy: %q", got)
|
||||
}
|
||||
f.write(policyFile, `{"policies": {"DisableAppUpdate": true, "Preferences": {"mail.shell.checkDefaultClient": {"Value": false, "Status": "default"}},
|
||||
"ExtensionSettings": {"owl@beonex.com": {"installation_mode": "force_installed", "install_url": "file:///opt/betterbird/owl/owl.xpi", "updates_disabled": true}}}}`)
|
||||
if why := policyProblem(mustRead(t, f, policyFile)); !strings.Contains(why, "locked") {
|
||||
t.Fatalf("a default the profile can override is not a lock: %q", why)
|
||||
}
|
||||
f.write(profileDir+"/prefs.js", `user_pref("mail.shell.checkDefaultClient", true);`+"\n")
|
||||
if got := asks(); got == "" {
|
||||
t.Fatal("a profile that turned the ask back on over an unlocked default is a finding")
|
||||
}
|
||||
f.write(policyFile, unlocked)
|
||||
f.write(profileDir+"/prefs.js", `user_pref("mail.shell.checkDefaultClient", false);`+"\n")
|
||||
if got := asks(); got != "" {
|
||||
t.Fatalf("a profile with the ask off does not ask: %s", got)
|
||||
}
|
||||
}
|
||||
|
||||
func mustRead(t *testing.T, f *fake, p string) []byte {
|
||||
t.Helper()
|
||||
raw, err := os.ReadFile(filepath.Join(f.Root, p))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return raw
|
||||
}
|
||||
@@ -0,0 +1,480 @@
|
||||
package main
|
||||
|
||||
// What the tools show of the five kinds (novox/hq ADR 0329 §1, §11–§13): the profile's entries beside the declared
|
||||
// ones, with the operator's own values, and how a node differs from what is declared. Never a credential (§2):
|
||||
// every answer passes credentialChecked, which withholds a field whose name says password, token, secret or cookie
|
||||
// and the password part of any address.
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// configView is the profile and the settings, read once for an answer.
|
||||
type configView struct {
|
||||
p Profile
|
||||
prefs map[string]string
|
||||
accounts map[string]Account
|
||||
ra []resolvedAccount
|
||||
conf Configuration
|
||||
problem string
|
||||
plan configPlan
|
||||
rec configRecord
|
||||
}
|
||||
|
||||
func (m *Machine) view() (configView, error) {
|
||||
var v configView
|
||||
p, err := m.profile()
|
||||
if err != nil {
|
||||
return v, err
|
||||
}
|
||||
v.p, v.prefs = p, mergedPrefs(p.path)
|
||||
accounts, _, aerr := m.readAccounts()
|
||||
if aerr != nil {
|
||||
v.problem = aerr.Error()
|
||||
}
|
||||
v.accounts = accounts
|
||||
v.ra = resolveAccounts(accounts, v.prefs, m.readRecord(p.Dir))
|
||||
conf, cerr := m.readConfiguration()
|
||||
if cerr != nil {
|
||||
v.problem = strings.TrimPrefix(v.problem+"; "+cerr.Error(), "; ")
|
||||
}
|
||||
v.conf = conf
|
||||
v.rec = m.readConfigRecord(p.Dir)
|
||||
v.plan = m.planConfiguration(p, conf, accounts, v.ra, v.prefs)
|
||||
return v, nil
|
||||
}
|
||||
|
||||
// listedEntry is one entry of the profile, with its values: Owl's marked, an ICS calendar by its name only (§13).
|
||||
type listedEntry struct {
|
||||
Key string `json:"key"`
|
||||
Kind string `json:"kind,omitempty"`
|
||||
// Account is an identity's account key in the profile, or the account name filters are listed under.
|
||||
Account string `json:"account,omitempty"`
|
||||
Owl bool `json:"owl,omitempty"`
|
||||
// DeclaredAs is the declared entry this one is, or empty: the person's own.
|
||||
DeclaredAs string `json:"declared_as,omitempty"`
|
||||
Value any `json:"value,omitempty"`
|
||||
Note string `json:"note,omitempty"`
|
||||
}
|
||||
|
||||
// declaredEntry is one declared entry, its value and how it resolved on this node.
|
||||
type declaredEntry struct {
|
||||
Name string `json:"name"`
|
||||
Value any `json:"value"`
|
||||
How string `json:"how,omitempty"`
|
||||
Key string `json:"key,omitempty"`
|
||||
Why string `json:"why,omitempty"`
|
||||
}
|
||||
|
||||
// KindListing is what a listing tool answers.
|
||||
type KindListing struct {
|
||||
Profile string `json:"profile"`
|
||||
Kind string `json:"kind"`
|
||||
InProfile []listedEntry `json:"in_the_profile"`
|
||||
Declared []declaredEntry `json:"declared"`
|
||||
// Kept says where the module keeps the filter files it replaced, per account (§8).
|
||||
Kept map[string]string `json:"kept_filter_files,omitempty"`
|
||||
Problem string `json:"problem,omitempty"`
|
||||
}
|
||||
|
||||
func (v configView) resolved(kind string) map[string]resolvedEntry {
|
||||
out := map[string]resolvedEntry{}
|
||||
for _, e := range v.plan.Entries {
|
||||
if e.Kind == kind {
|
||||
out[e.Name] = e
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// declaredBy is, per profile key, the declared entry of a kind adopted or created under it.
|
||||
func (v configView) declaredBy(kind string) map[string]string {
|
||||
out := map[string]string{}
|
||||
for _, e := range v.plan.Entries {
|
||||
if e.Kind == kind && e.How != refused {
|
||||
out[e.Key] = e.Name
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// List answers one kind: the profile's entries beside the declared ones.
|
||||
func (m *Machine) List(kind string) (KindListing, error) {
|
||||
v, err := m.view()
|
||||
a := KindListing{Kind: kind, InProfile: []listedEntry{}, Declared: []declaredEntry{}}
|
||||
if err != nil {
|
||||
return a, err
|
||||
}
|
||||
a.Profile, a.Problem = v.p.Dir, v.problem
|
||||
res := v.resolved(kind)
|
||||
by := v.declaredBy(kind)
|
||||
declare := func(name string, value any) {
|
||||
r := res[name]
|
||||
a.Declared = append(a.Declared, declaredEntry{Name: name, Value: value, How: r.How, Key: r.Key, Why: r.Why})
|
||||
}
|
||||
switch kind {
|
||||
case kindIdentities:
|
||||
for _, pi := range m.profileIdentities(v.prefs) {
|
||||
a.InProfile = append(a.InProfile, listedEntry{Key: pi.Key, Account: pi.AccountKey, Owl: pi.Owl,
|
||||
DeclaredAs: by[pi.Key], Value: pi.Value})
|
||||
}
|
||||
for _, name := range sortedKeys(v.conf.Identities) {
|
||||
declare(name, v.conf.Identities[name])
|
||||
}
|
||||
case kindCalendars:
|
||||
for _, pc := range profileCalendars(v.prefs) {
|
||||
e := listedEntry{Key: pc.Key, Kind: pc.Kind, DeclaredAs: by[pc.Key], Value: pc.Value}
|
||||
switch {
|
||||
case strings.HasPrefix(pc.Kind, "owl"):
|
||||
e.Owl, e.Value = true, map[string]string{"name": pc.Value.Name, "url": pc.Value.URL}
|
||||
e.Note = "Owl's: never declared (novox/hq ADR 0316 §13)"
|
||||
case pc.Kind == "ics":
|
||||
e.Value = map[string]string{"name": pc.Value.Name}
|
||||
e.Note = "subscribed by an ICS address, which is often itself the secret: its name only, never declared (novox/hq ADR 0329 §13)"
|
||||
case pc.Kind != "caldav":
|
||||
e.Note = "kept in the profile: its events are never declared"
|
||||
}
|
||||
a.InProfile = append(a.InProfile, e)
|
||||
}
|
||||
for _, name := range sortedKeys(v.conf.Calendars) {
|
||||
declare(name, v.conf.Calendars[name])
|
||||
}
|
||||
case kindAddressBooks:
|
||||
for _, b := range profileAddressBooks(v.prefs) {
|
||||
e := listedEntry{Key: b.Key, Kind: b.Kind, DeclaredAs: by[b.Key], Value: b.Value}
|
||||
if !b.CardDAV {
|
||||
e.Value, e.Note = map[string]string{"name": b.Value.Name}, "its contacts are kept in the profile and never declared"
|
||||
}
|
||||
a.InProfile = append(a.InProfile, e)
|
||||
}
|
||||
for _, name := range sortedKeys(v.conf.AddressBooks) {
|
||||
declare(name, v.conf.AddressBooks[name])
|
||||
}
|
||||
case kindTags:
|
||||
for _, t := range profileTags(v.prefs) {
|
||||
a.InProfile = append(a.InProfile, listedEntry{Key: t.Key, DeclaredAs: by[t.Key], Value: t.Value})
|
||||
}
|
||||
for _, key := range sortedKeys(v.conf.Tags) {
|
||||
declare(key, v.conf.Tags[key])
|
||||
}
|
||||
case kindFilters:
|
||||
for _, f := range m.profileFilters(v) {
|
||||
a.InProfile = append(a.InProfile, f)
|
||||
}
|
||||
for _, account := range v.conf.sortedFilterAccounts() {
|
||||
d := declaredEntry{Name: account, Value: v.conf.Filters[account], How: "owned whole"}
|
||||
for _, fp := range v.plan.filters {
|
||||
if fp.Account == account {
|
||||
d.Key, d.Why = fp.File, strings.TrimSpace(fp.Refused+" "+fp.Waiting)
|
||||
if fp.Refused != "" {
|
||||
d.How = refused
|
||||
}
|
||||
}
|
||||
}
|
||||
a.Declared = append(a.Declared, d)
|
||||
}
|
||||
a.Kept = map[string]string{}
|
||||
for _, account := range sortedKeys(v.rec.Filters) {
|
||||
if r := v.rec.Filters[account]; r.Kept != "" {
|
||||
a.Kept[account] = r.Kept
|
||||
}
|
||||
}
|
||||
if len(a.Kept) == 0 {
|
||||
a.Kept = nil
|
||||
}
|
||||
default:
|
||||
return a, fmt.Errorf("a kind is one of %s", strings.Join(allKinds, ", "))
|
||||
}
|
||||
return a, nil
|
||||
}
|
||||
|
||||
// profileFilters are the filters of each declared account in the profile, by the account's name.
|
||||
func (m *Machine) profileFilters(v configView) []listedEntry {
|
||||
var out []listedEntry
|
||||
for _, r := range v.ra {
|
||||
if r.How == refused {
|
||||
continue
|
||||
}
|
||||
dir := m.accountDirectory(v.p.path, v.prefs, r.Keys.Server)
|
||||
if dir == "" {
|
||||
continue
|
||||
}
|
||||
text, err := readBounded(dir + "/" + filterFileName)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
parsed := parseFilters(string(text), folderBase(v.prefs, r.Keys.Server))
|
||||
e := listedEntry{Key: m.tilde(strings.TrimPrefix(dir+"/"+filterFileName, m.Root)), Account: r.name, Value: parsed.Filters}
|
||||
if _, declared := v.conf.Filters[r.name]; declared {
|
||||
e.DeclaredAs = r.name
|
||||
}
|
||||
if len(parsed.NotDeclarable) > 0 {
|
||||
e.Note = "not declarable as they are: " + strings.Join(parsed.NotDeclarable, "; ")
|
||||
}
|
||||
out = append(out, e)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// fieldDiff is one field a profile holds differently from the declared value.
|
||||
type fieldDiff struct {
|
||||
Field string `json:"field"`
|
||||
Declared any `json:"declared"`
|
||||
Profile any `json:"profile"`
|
||||
}
|
||||
|
||||
// DriftEntry is one declared entry against the profile: holds, differs, lacks or refused (§11).
|
||||
type DriftEntry struct {
|
||||
Name string `json:"name"`
|
||||
State string `json:"state"`
|
||||
Key string `json:"key,omitempty"`
|
||||
Differs []fieldDiff `json:"differs,omitempty"`
|
||||
Why string `json:"why,omitempty"`
|
||||
}
|
||||
|
||||
// KindDrift is one kind's drift, and the profile's entries nothing declares.
|
||||
type KindDrift struct {
|
||||
Kind string `json:"kind"`
|
||||
Declared []DriftEntry `json:"declared"`
|
||||
Undeclared []listedEntry `json:"undeclared,omitempty"`
|
||||
}
|
||||
|
||||
// DriftAnswer is what betterbird_drift answers.
|
||||
type DriftAnswer struct {
|
||||
Profile string `json:"profile"`
|
||||
Kinds []KindDrift `json:"kinds"`
|
||||
Note string `json:"note"`
|
||||
Problem string `json:"problem,omitempty"`
|
||||
}
|
||||
|
||||
// Drift answers, per kind, how the profile differs from what is declared: the profile as Betterbird reads it at
|
||||
// its next start (prefs.js with user.js over it), so a declared value the module placed and Betterbird has not
|
||||
// started with yet holds; the drift then is only what the module did not place.
|
||||
func (m *Machine) Drift(kind string) (DriftAnswer, error) {
|
||||
v, err := m.view()
|
||||
a := DriftAnswer{Kinds: []KindDrift{}}
|
||||
if err != nil {
|
||||
return a, err
|
||||
}
|
||||
a.Profile, a.Problem = v.p.Dir, v.problem
|
||||
a.Note = "the profile as Betterbird reads it at its next start (prefs.js, the module's user.js over it): differs " +
|
||||
"names each field with the declared value and the profile's; lacks is an entry the profile does not hold yet " +
|
||||
"(betterbird_user_js with apply, betterbird_restart); undeclared entries stay the person's"
|
||||
kinds := allKinds
|
||||
if kind != "" {
|
||||
if !contains(allKinds, kind) {
|
||||
return a, fmt.Errorf("a kind is one of %s", strings.Join(allKinds, ", "))
|
||||
}
|
||||
kinds = []string{kind}
|
||||
}
|
||||
for _, k := range kinds {
|
||||
a.Kinds = append(a.Kinds, m.kindDrift(v, k))
|
||||
}
|
||||
return a, nil
|
||||
}
|
||||
|
||||
func (m *Machine) kindDrift(v configView, kind string) KindDrift {
|
||||
d := KindDrift{Kind: kind, Declared: []DriftEntry{}}
|
||||
res := v.resolved(kind)
|
||||
by := v.declaredBy(kind)
|
||||
entry := func(name string, declared any, profile func(key string) (any, bool)) {
|
||||
r := res[name]
|
||||
e := DriftEntry{Name: name, Key: r.Key}
|
||||
switch {
|
||||
case r.How == refused:
|
||||
e.State, e.Why = refused, r.Why
|
||||
default:
|
||||
have, ok := profile(r.Key)
|
||||
if !ok {
|
||||
e.State = "lacks"
|
||||
break
|
||||
}
|
||||
e.Differs = differing(declared, have)
|
||||
e.State = map[bool]string{true: "holds", false: "differs"}[len(e.Differs) == 0]
|
||||
}
|
||||
d.Declared = append(d.Declared, e)
|
||||
}
|
||||
switch kind {
|
||||
case kindIdentities:
|
||||
ids := map[string]profileIdentity{}
|
||||
for _, pi := range m.profileIdentities(v.prefs) {
|
||||
ids[pi.Key] = pi
|
||||
if by[pi.Key] == "" && !pi.Owl {
|
||||
d.Undeclared = append(d.Undeclared, listedEntry{Key: pi.Key, Account: pi.AccountKey, Value: pi.Value})
|
||||
}
|
||||
}
|
||||
for _, name := range sortedKeys(v.conf.Identities) {
|
||||
want := *v.conf.Identities[name]
|
||||
want.Key = ""
|
||||
entry(name, want, func(key string) (any, bool) {
|
||||
pi, ok := ids[key]
|
||||
if !ok {
|
||||
return nil, false
|
||||
}
|
||||
have := pi.Value
|
||||
have.Account = want.Account
|
||||
if want.Address == "" {
|
||||
have.Address, have.Name = "", ""
|
||||
}
|
||||
return have, true
|
||||
})
|
||||
}
|
||||
case kindCalendars:
|
||||
cals := map[string]profileCalendar{}
|
||||
for _, pc := range profileCalendars(v.prefs) {
|
||||
cals[pc.Key] = pc
|
||||
if by[pc.Key] == "" && pc.Kind == "caldav" {
|
||||
d.Undeclared = append(d.Undeclared, listedEntry{Key: pc.Key, Kind: pc.Kind, Value: pc.Value})
|
||||
}
|
||||
}
|
||||
idName := v.declaredBy(kindIdentities)
|
||||
for _, name := range sortedKeys(v.conf.Calendars) {
|
||||
want := *v.conf.Calendars[name]
|
||||
want.Key = ""
|
||||
entry(name, want, func(key string) (any, bool) {
|
||||
pc, ok := cals[key]
|
||||
if !ok {
|
||||
return nil, false
|
||||
}
|
||||
have := pc.Value
|
||||
have.Invitations = idName[pc.IdentityKey]
|
||||
if have.Reminders == nil && want.Reminders != nil && *want.Reminders {
|
||||
on := true
|
||||
have.Reminders = &on
|
||||
}
|
||||
return have, true
|
||||
})
|
||||
}
|
||||
case kindAddressBooks:
|
||||
books := map[string]profileAddressBook{}
|
||||
for _, b := range profileAddressBooks(v.prefs) {
|
||||
books[b.Key] = b
|
||||
if by[b.Key] == "" && b.CardDAV {
|
||||
d.Undeclared = append(d.Undeclared, listedEntry{Key: b.Key, Kind: b.Kind, Value: b.Value})
|
||||
}
|
||||
}
|
||||
for _, name := range sortedKeys(v.conf.AddressBooks) {
|
||||
want := *v.conf.AddressBooks[name]
|
||||
want.Key = ""
|
||||
entry(name, want, func(key string) (any, bool) {
|
||||
b, ok := books[key]
|
||||
return b.Value, ok
|
||||
})
|
||||
}
|
||||
case kindTags:
|
||||
tags := map[string]profileTag{}
|
||||
for _, t := range profileTags(v.prefs) {
|
||||
tags[t.Key] = t
|
||||
if by[t.Key] == "" {
|
||||
d.Undeclared = append(d.Undeclared, listedEntry{Key: t.Key, Value: t.Value})
|
||||
}
|
||||
}
|
||||
for _, key := range sortedKeys(v.conf.Tags) {
|
||||
want := *v.conf.Tags[key]
|
||||
entry(key, want, func(k string) (any, bool) {
|
||||
t, ok := tags[k]
|
||||
have := t.Value
|
||||
if want.Order == 0 {
|
||||
have.Order = 0
|
||||
}
|
||||
return have, ok
|
||||
})
|
||||
}
|
||||
case kindFilters:
|
||||
files := map[string]listedEntry{}
|
||||
for _, f := range m.profileFilters(v) {
|
||||
files[f.Account] = f
|
||||
released := v.rec.Filters[f.Account].Released
|
||||
if f.DeclaredAs == "" && !released {
|
||||
if list, _ := f.Value.([]Filter); len(list) > 0 {
|
||||
d.Undeclared = append(d.Undeclared, f)
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, fp := range v.plan.filters {
|
||||
e := DriftEntry{Name: fp.Account, Key: fp.File}
|
||||
switch {
|
||||
case fp.Refused != "":
|
||||
e.State, e.Why = refused, fp.Refused
|
||||
case fp.path == "" || !fp.exists:
|
||||
e.State, e.Why = "lacks", strings.TrimSpace(fp.Waiting)
|
||||
default:
|
||||
have, _ := files[fp.Account].Value.([]Filter)
|
||||
for _, line := range filterDifferences(v.conf.Filters[fp.Account], have) {
|
||||
e.Differs = append(e.Differs, fieldDiff{Field: line})
|
||||
}
|
||||
if fp.have != fp.want && len(e.Differs) == 0 {
|
||||
e.Differs = append(e.Differs, fieldDiff{Field: "the file's text, beyond the filters' own words"})
|
||||
}
|
||||
e.State = map[bool]string{true: "holds", false: "differs"}[len(e.Differs) == 0]
|
||||
}
|
||||
d.Declared = append(d.Declared, e)
|
||||
}
|
||||
}
|
||||
return d
|
||||
}
|
||||
|
||||
// differing are the fields of two values of the same shape that differ.
|
||||
func differing(declared, have any) []fieldDiff {
|
||||
var a, b map[string]any
|
||||
_ = json.Unmarshal([]byte(mustJSON(declared)), &a)
|
||||
_ = json.Unmarshal([]byte(mustJSON(have)), &b)
|
||||
var out []fieldDiff
|
||||
for _, f := range sortedKeys(mergeKeys(a, b)) {
|
||||
if mustJSON(a[f]) != mustJSON(b[f]) {
|
||||
out = append(out, fieldDiff{Field: f, Declared: a[f], Profile: b[f]})
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// driftCounts are, per kind, how many declared entries do not hold.
|
||||
func (m *Machine) driftCounts(v configView) map[string]int {
|
||||
out := map[string]int{}
|
||||
for _, k := range allKinds {
|
||||
for _, e := range m.kindDrift(v, k).Declared {
|
||||
if e.State != "holds" {
|
||||
out[k]++
|
||||
}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// credentialChecked is an answer as it may leave the tool (§2): a field whose name says password, token, secret
|
||||
// or cookie withheld, and the password part of any address as ‹withheld›.
|
||||
func credentialChecked(v any) any {
|
||||
b, err := json.Marshal(v)
|
||||
if err != nil {
|
||||
return v
|
||||
}
|
||||
var tree any
|
||||
if json.Unmarshal(b, &tree) != nil {
|
||||
return v
|
||||
}
|
||||
return scrubTree(tree)
|
||||
}
|
||||
|
||||
func scrubTree(n any) any {
|
||||
switch x := n.(type) {
|
||||
case map[string]any:
|
||||
for k, v := range x {
|
||||
if secretName.MatchString(k) {
|
||||
x[k] = "‹withheld›"
|
||||
continue
|
||||
}
|
||||
x[k] = scrubTree(v)
|
||||
}
|
||||
return x
|
||||
case []any:
|
||||
for i, v := range x {
|
||||
x[i] = scrubTree(v)
|
||||
}
|
||||
return x
|
||||
case string:
|
||||
return withoutPassword(x)
|
||||
}
|
||||
return n
|
||||
}
|
||||
@@ -2,20 +2,32 @@
|
||||
// mail and calendar client, in the operator's session, served by the node's runtime as the operator
|
||||
// account. The module holds no seat, so every tool is its own.
|
||||
//
|
||||
// The operator's mail, calendars, accounts and credentials are never read: a tool reads processes,
|
||||
// profiles.ini, and of the profile only prefs.js, user.js and extensions.json, and answers no
|
||||
// preference that can hold a credential or names an account.
|
||||
// The operator's mail, calendar events, contacts and credentials are never read: a tool reads processes,
|
||||
// profiles.ini, and of the profile its prefs.js, user.js, extensions.json, each account's msgFilterRules.dat and
|
||||
// of logins.json only the server each kept sign-in is for. The tools show the operator's own configuration, an
|
||||
// account's, an identity's, a calendar's, an address book's, a filter's and a tag's values, and never a credential
|
||||
// (novox/hq ADR 0329 §1, §2): every answer passes credentialChecked.
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
)
|
||||
|
||||
func main() {
|
||||
// The module's start runs this before Betterbird (novox/hq ADR 0316 §11, §12): the user.js placed and the
|
||||
// declared accounts written into the account lists, while no Betterbird of this account runs.
|
||||
if len(os.Args) == 2 && os.Args[1] == "prepare" {
|
||||
if err := machine.Prepare(); err != nil {
|
||||
fmt.Fprintln(os.Stderr, "betterbird: "+err.Error())
|
||||
os.Exit(1)
|
||||
}
|
||||
return
|
||||
}
|
||||
if err := stdio.Serve("", tools()); err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
os.Exit(1)
|
||||
@@ -25,22 +37,52 @@ func main() {
|
||||
var machine = NewMachine()
|
||||
|
||||
func tools() []stdio.Tool {
|
||||
list := toolList()
|
||||
for i := range list {
|
||||
run := list[i].Run
|
||||
list[i].Run = func(args map[string]any) (any, error) {
|
||||
out, err := run(args)
|
||||
return credentialChecked(out), err
|
||||
}
|
||||
}
|
||||
return list
|
||||
}
|
||||
|
||||
// listTool is the listing tool of one of the five kinds (novox/hq ADR 0329 §12).
|
||||
func listTool(kind, what string) stdio.Tool {
|
||||
return stdio.Tool{
|
||||
Name: "betterbird_" + strings.ReplaceAll(kind, "-", "_"),
|
||||
Description: "The " + what + " of the profile Betterbird uses beside the ones the module's setting " + kind +
|
||||
" declares (novox/hq ADR 0329): each of the profile's with its key, its values and the declared entry it is " +
|
||||
"(none: the person's own), Owl's marked; each declared one with its value and how it resolved on this node " +
|
||||
"(adopted, created or refused with why, and its key). The operator's own values, never a password, token " +
|
||||
"or cookie. betterbird_drift says how they differ. (r)",
|
||||
Run: func(map[string]any) (any, error) { return machine.List(kind) },
|
||||
}
|
||||
}
|
||||
|
||||
func toolList() []stdio.Tool {
|
||||
return []stdio.Tool{
|
||||
{
|
||||
Name: "betterbird_status",
|
||||
Description: "Betterbird: whether it runs (the main process's pid, since, and the scope it runs in; how " +
|
||||
"many child processes), the installed package and version and whether it is from the AUR, where its " +
|
||||
"output goes, the profile it uses and how that was found, the module's user.js (declared, placed, in " +
|
||||
"effect), the add-ons the operator installed, and the preferences that decide how its windows and " +
|
||||
"notifications look (reminder dialog, theme, renderer, system notifications). (r)",
|
||||
"many child processes); what is installed in /opt/betterbird and from where (the mesh's pinned release, " +
|
||||
"or the community package before the switch), its version and build beside the newest published build; " +
|
||||
"whether the enterprise policy is placed; Owl's version against the mesh's pin and whether it is active; " +
|
||||
"where its output goes; the profile it uses and how that was found; the module's user.js (declared, " +
|
||||
"placed, in effect, with every declared identity, calendar, address book and tag as it resolved); the " +
|
||||
"declared accounts by count (in the profile's list, other accounts, not signed in yet); the add-ons in the " +
|
||||
"profile; and the preferences that decide how its windows and notifications look. (r)",
|
||||
Run: func(map[string]any) (any, error) { return machine.Status() },
|
||||
},
|
||||
{
|
||||
Name: "betterbird_prefs",
|
||||
Description: "Named preferences of Betterbird's profile: each one's value and where it comes from (user.js, " +
|
||||
"which wins at the start; prefs.js; or Betterbird's default). A name ending in . or * is a prefix " +
|
||||
"(at most 200 answered). A preference whose name can hold a credential, or that names an account, a " +
|
||||
"server, a calendar or a person, is answered without its value. (r)",
|
||||
Description: "Named preferences of Betterbird's profile: each one's value and where it comes from (the " +
|
||||
"module's policy, which locks it; user.js, which wins at the start; prefs.js; or the default), and the " +
|
||||
"default itself with the file that gives it: Betterbird's own, read from omni.ja and the installation's " +
|
||||
"defaults/pref, or the module's defaults (mesh-defaults.js). A name ending in . or * is a prefix (at most " +
|
||||
"200 answered). A preference whose name can hold a credential, or that names an account, a server, a " +
|
||||
"calendar or a person, is answered without its value. (r)",
|
||||
Input: map[string]any{
|
||||
"names": map[string]any{"type": "array", "items": map[string]any{"type": "string"},
|
||||
"description": "preference names or prefixes, e.g. [\"calendar.alarms.\", \"extensions.activeThemeID\"]"},
|
||||
@@ -53,13 +95,57 @@ func tools() []stdio.Tool {
|
||||
return machine.Prefs(names)
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "betterbird_config",
|
||||
Description: "Betterbird's preferences through the module's layers, preference by preference: the value its " +
|
||||
"policy locks, the setting prefs (the operator's setup, mesh-wide with this node's layer over it, one entry " +
|
||||
"per preference), Betterbird's own default where the module keeps that, the profile's own saved value and " +
|
||||
"the value in effect. It flags drift: a declared value the profile's user.js does not hold yet, or a value " +
|
||||
"the profile saved where the module keeps Betterbird's default. With keep (names), it proposes this node's " +
|
||||
"layer of the setting prefs holding the profile's values of those preferences beside what the layer holds, " +
|
||||
"through the controller's settings verb, for the operator's Approve; it never writes user.js. (r; a with keep)",
|
||||
Input: map[string]any{
|
||||
"keep": map[string]any{"type": "array", "items": map[string]any{"type": "string"},
|
||||
"description": "managed preferences whose profile value this machine keeps: proposed as a node layer, e.g. [\"mail.uifontsize\"]"},
|
||||
},
|
||||
Run: func(args map[string]any) (any, error) {
|
||||
keep, err := texts(args, "keep")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return machine.Config(keep)
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "betterbird_config_apply",
|
||||
Description: "Bring the profile back to the module's configuration: takes the drifting values " +
|
||||
"(betterbird_config) out of the profile's prefs.js, so the module's defaults hold at Betterbird's next " +
|
||||
"start, and says what it cleared, each with the value it had and the one that now holds. Without apply " +
|
||||
"it lists them only. With apply it acts only while Betterbird is not running, because Betterbird " +
|
||||
"rewrites prefs.js when it quits; every other line of prefs.js is kept. (r; a with apply)",
|
||||
Input: map[string]any{
|
||||
"apply": map[string]any{"type": "boolean", "description": "clear the drifting values (default false: a dry run)"},
|
||||
},
|
||||
Run: func(args map[string]any) (any, error) {
|
||||
apply, err := flag(args, "apply", false)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return machine.ConfigApply(apply)
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "betterbird_user_js",
|
||||
Description: "The module's user.js — the preferences the module declares, from its setting prefs, written by " +
|
||||
"the mesh to ~/.config/betterbird/user.js — against the profile's user.js: whether it is placed, and " +
|
||||
"whether the running Betterbird started after it was. With apply, place it in the profile (only where " +
|
||||
"the profile has no user.js or the module's own; an operator's file is never overwritten). Betterbird " +
|
||||
"reads it at its next start: betterbird_restart. (r; a with apply)",
|
||||
Description: "The module's user.js — the setting prefs, then the declared accounts' preferences (setting " +
|
||||
"accounts), then the declared identities, calendars, address books and tags — against the profile's " +
|
||||
"user.js: whether it is placed, and whether the running Betterbird started after it was. With apply, place " +
|
||||
"it in the profile (only where the profile has no user.js or the module's own; an operator's file is never " +
|
||||
"overwritten). A declared entry the profile already holds is adopted under its own keys; one it does not " +
|
||||
"hold is created under the module's keys; an ambiguous one is refused alone, with the remedy (novox/hq ADR " +
|
||||
"0317, ADR 0329 §7). What Betterbird rewrites itself — the account and identity lists, the calendars' " +
|
||||
"order, the removals and each declared account's msgFilterRules.dat — is written only while Betterbird is " +
|
||||
"closed, and the answer counts what waits (§8, §9). Accounts' values are withheld in the user.js shown. " +
|
||||
"Betterbird reads it at its next start: betterbird_restart. (r; a with apply)",
|
||||
Input: map[string]any{
|
||||
"apply": map[string]any{"type": "boolean", "description": "place the declared user.js in the profile (default false)"},
|
||||
},
|
||||
@@ -118,19 +204,115 @@ func tools() []stdio.Tool {
|
||||
},
|
||||
{
|
||||
Name: "betterbird_check",
|
||||
Description: "Check what the module promises: Betterbird is installed; one runs; its profile is found; the " +
|
||||
"module's user.js is on the machine, valid, placed in the profile and in effect; its output is read " +
|
||||
"(a pipe nobody reads loses what it prints). Answers ok and each finding with what to do. (r)",
|
||||
Description: "Check what the module promises: Betterbird installed from the mesh's pinned release and not " +
|
||||
"behind the newest published build; the enterprise policy placed and the module's defaults with it; no " +
|
||||
"managed preference drifting from them; Owl at its pin and active; one " +
|
||||
"Betterbird runs; its profile is found; the module's user.js on the machine, valid, placed and in " +
|
||||
"effect; the declared accounts in the profile's list, none refused, each signed in once; per kind, no " +
|
||||
"declared identity, calendar, address book, filter or tag drifting or refused, and no change waiting for " +
|
||||
"Betterbird to close; its output read. Answers ok and each finding with what to do. (r)",
|
||||
Run: func(map[string]any) (any, error) { return machine.Check() },
|
||||
},
|
||||
{
|
||||
Name: "betterbird_accounts",
|
||||
Description: "The accounts of the profile Betterbird uses with their values (name, address, user, servers' " +
|
||||
"hosts and ports) and the declared account each is, Owl's marked by name and address (never declared), " +
|
||||
"with counts by kind. Never a password, a token or anything of the login store (novox/hq ADR 0329 §1, §2). " +
|
||||
"With propose, it builds the module's setting accounts from them (every account but Owl's) and proposes " +
|
||||
"it in the mesh-wide layer as it stands, through the controller's settings verb, for the operator's " +
|
||||
"Approve on a channel that proves who answers (novox/hq ADR 0317, ADR 0277); the answer shows the value it " +
|
||||
"proposed. (r; a with propose)",
|
||||
Input: map[string]any{
|
||||
"propose": map[string]any{"type": "boolean", "description": "build the setting accounts from the profile and propose it to the operator (default false)"},
|
||||
},
|
||||
Run: func(args map[string]any) (any, error) {
|
||||
propose, err := flag(args, "propose", false)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return machine.ProfileAccounts(propose)
|
||||
},
|
||||
},
|
||||
listTool(kindIdentities, "identities (each account's default and its other identities, with name, address, reply-to, organisation and signature)"),
|
||||
listTool(kindCalendars, "calendars (CalDAV with name, address, colour and options; an ICS one by its name only)"),
|
||||
listTool(kindAddressBooks, "address books (CardDAV with name, address and user; a local one by its name)"),
|
||||
listTool(kindFilters, "filters (per declared account: its msgFilterRules.dat as filters, folders by their path within the account; where the module keeps a file it replaced)"),
|
||||
listTool(kindTags, "tags (key, name, colour, place)"),
|
||||
{
|
||||
Name: "betterbird_derive",
|
||||
Description: "Derive the module's settings identities, calendars, address-books, filters and tags (or the kinds " +
|
||||
"given) from the profiles of every desktop node holding Betterbird, laid out per layer: an entry on every " +
|
||||
"such node goes into the mesh-wide layer with this node's value, and the answer lists each field another " +
|
||||
"node holds differently; one on some nodes only goes into those nodes' own layers. Owl's, ICS calendars, " +
|
||||
"local address books and any address carrying a password are left out and named. prefs puts values of the " +
|
||||
"setting prefs into the mesh-wide layer. With propose, the next layer (the mesh-wide first, then each " +
|
||||
"node's) is proposed for the operator's Approve through the controller's settings verb, carrying the layer " +
|
||||
"as it stands beside the derived kinds; refused while a proposal of the module's waits, or while a desktop " +
|
||||
"node does not answer (novox/hq ADR 0329 §10, ADR 0277). here answers this node's profile only, as the " +
|
||||
"other nodes are asked. The values are the operator's own; never a credential. (r; a with propose)",
|
||||
Input: map[string]any{
|
||||
"kinds": map[string]any{"type": "array", "items": map[string]any{"type": "string", "enum": allKinds},
|
||||
"description": "the kinds to derive (default all five)"},
|
||||
"here": map[string]any{"type": "boolean", "description": "this node's profile only (default false)"},
|
||||
"propose": map[string]any{"type": "boolean", "description": "propose the next layer to the operator (default false)"},
|
||||
"prefs": map[string]any{"type": "object", "description": "values of the setting prefs for the mesh-wide " +
|
||||
"layer, by preference name, e.g. {\"mail.uifontsize\": 14}"},
|
||||
"layer": map[string]any{"type": "string", "description": "the layer to propose instead of the next: mesh-wide or a node's name"},
|
||||
},
|
||||
Run: func(args map[string]any) (any, error) {
|
||||
kinds, err := texts(args, "kinds")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
here, err := flag(args, "here", false)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
propose, err := flag(args, "propose", false)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
layer, err := text(args, "layer", false)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var prefs map[string]any
|
||||
if v, ok := args["prefs"]; ok && v != nil {
|
||||
if prefs, ok = v.(map[string]any); !ok {
|
||||
return nil, fmt.Errorf("prefs is an object of preference names and values, not %T", v)
|
||||
}
|
||||
}
|
||||
return machine.Derive(kinds, here, propose, prefs, layer)
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "betterbird_drift",
|
||||
Description: "How this node's profile differs from the declared identities, calendars, address books, " +
|
||||
"filters and tags (novox/hq ADR 0329 §11): per declared entry whether the profile holds it, holds another " +
|
||||
"value (each differing field with the declared value and the profile's), lacks it, or it is refused; and " +
|
||||
"the profile's entries nothing declares, which stay the person's. The profile as Betterbird reads it at " +
|
||||
"its next start. (r)",
|
||||
Input: map[string]any{
|
||||
"kind": map[string]any{"type": "string", "enum": allKinds, "description": "one kind (default every kind)"},
|
||||
},
|
||||
Run: func(args map[string]any) (any, error) {
|
||||
kind, err := text(args, "kind", false)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return machine.Drift(kind)
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "betterbird_reminder_test",
|
||||
Description: "Show a calendar reminder without touching the operator's calendar: a second Betterbird in a " +
|
||||
"throw-away profile under the runtime directory (--no-remote, unit mesh-betterbird-reminder-test, at most " +
|
||||
"10 minutes), with the module's user.js and one local calendar file holding one event whose reminder is " +
|
||||
"due after `after` seconds. carry copies named preferences of the operator's profile (a theme, a " +
|
||||
"colour switch) into it, never a credential or an account. Then xorg_windows (class Calendar) and " +
|
||||
"xorg_screenshot show the reminder window. stop ends it and removes the profile. (a)",
|
||||
Description: "OPENS WINDOWS ON THE OPERATOR'S SCREEN: tell the operator before calling it. Shows a calendar " +
|
||||
"reminder without touching the operator's calendar: a second Betterbird in a throw-away profile under " +
|
||||
"the runtime directory (--no-remote, unit mesh-betterbird-reminder-test, at most 10 minutes), its main " +
|
||||
"window marked MESH TEST in red, with the module's user.js (no accounts) and one local calendar file " +
|
||||
"holding one event, \"MESH TEST reminder\", whose reminder is due after `after` seconds. carry copies " +
|
||||
"named preferences of the operator's profile (a theme, a colour switch) into it, never a credential or " +
|
||||
"an account. Then xorg_windows (class Calendar) and xorg_screenshot show the reminder window. stop ends " +
|
||||
"it and removes the profile. (a)",
|
||||
Input: map[string]any{
|
||||
"after": map[string]any{"type": "integer", "description": "seconds until the reminder is due (default 40, 15 to 300)"},
|
||||
"carry": map[string]any{"type": "array", "items": map[string]any{"type": "string"},
|
||||
|
||||
@@ -2,25 +2,32 @@ package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// betterbird's shape (novox/hq issue 345, ADR 0208): no package (Betterbird comes from the AUR, and the
|
||||
// node-engine installs from the official repositories only), no seat, no start of its own (the operator
|
||||
// starts it from the launcher); the X display on its own machine; one directory and one file of its own,
|
||||
// the user.js the setting prefs fills; and the Go bundle serving exactly the listed betterbird_ tools.
|
||||
// betterbird's shape (novox/hq issue 345, ADR 0208, ADR 0316): the X display on its own machine; its directory
|
||||
// in the home with the mergeable files the settings fill, each naming the keys it takes (ADR 0326): prefs, accounts
|
||||
// and configuration (identities, calendars, address-books, filters) trusted, tags not (ADR 0329 §5); the community package declared absent before anything is placed in /opt/betterbird; Betterbird from
|
||||
// its pinned upstream release, Owl pinned by its vendor's digest, the policy and the dictionaries' default
|
||||
// unpacked into the installation; the tools' program for the start's preparation, the start and the launcher;
|
||||
// every new build waiting for the operator's push; and the Go bundle serving exactly the listed tools.
|
||||
|
||||
type manifest struct {
|
||||
Module string `json:"module"`
|
||||
Version string `json:"version"`
|
||||
Requires []string `json:"requires"`
|
||||
Settings map[string]any `json:"settings"`
|
||||
Tools []string `json:"tools"`
|
||||
Resources []map[string]any `json:"resources"`
|
||||
Module string `json:"module"`
|
||||
Version string `json:"version"`
|
||||
Upgrade map[string]string `json:"upgrade"`
|
||||
Requires []string `json:"requires"`
|
||||
Invokes []string `json:"invokes"`
|
||||
Settings map[string]any `json:"settings"`
|
||||
Tools []string `json:"tools"`
|
||||
Resources []map[string]any `json:"resources"`
|
||||
Build struct {
|
||||
Artifacts []map[string]any `json:"artifacts"`
|
||||
} `json:"build"`
|
||||
@@ -41,43 +48,245 @@ func readManifest(t *testing.T) (manifest, string) {
|
||||
return m, string(raw)
|
||||
}
|
||||
|
||||
func TestItOwnsTheUserJSAndNothingOfTheProfile(t *testing.T) {
|
||||
func resource(t *testing.T, m manifest, id string) map[string]any {
|
||||
t.Helper()
|
||||
for _, r := range m.Resources {
|
||||
if r["id"] == id {
|
||||
return r
|
||||
}
|
||||
}
|
||||
t.Fatalf("module.json has no resource %s", id)
|
||||
return nil
|
||||
}
|
||||
|
||||
func artifact(t *testing.T, m manifest, name string) map[string]any {
|
||||
t.Helper()
|
||||
for _, a := range m.Build.Artifacts {
|
||||
if a["name"] == name {
|
||||
return a
|
||||
}
|
||||
}
|
||||
t.Fatalf("module.json builds no artifact %s", name)
|
||||
return nil
|
||||
}
|
||||
|
||||
func TestItPlacesBetterbirdWhereTheCommunityPackageDidAndOnlyAfterItIsGone(t *testing.T) {
|
||||
m, raw := readManifest(t)
|
||||
if m.Module != "betterbird" || !reflect.DeepEqual(m.Requires, []string{"x11-display"}) {
|
||||
t.Fatalf("%+v", m)
|
||||
}
|
||||
// Every new build waits for a person's push: the switch removes a package and replaces what a running
|
||||
// Betterbird has open (ADR 0316 §4).
|
||||
if m.Upgrade["policy"] != "record" || strings.TrimSpace(m.Upgrade["why"]) == "" {
|
||||
t.Fatalf("the module is marked for a person's approval, with why: %v", m.Upgrade)
|
||||
}
|
||||
var ids []string
|
||||
for _, r := range m.Resources {
|
||||
ids = append(ids, r["id"].(string))
|
||||
if r["type"] == "package" {
|
||||
t.Errorf("betterbird-bin is the AUR's: a package resource asks the node-engine for what it cannot install")
|
||||
}
|
||||
if p, _ := r["path"].(string); !strings.HasPrefix(p, "${machine:account-home}/.config/betterbird") {
|
||||
t.Errorf("a path outside the module's directory: %s", p)
|
||||
}
|
||||
}
|
||||
if strings.Join(ids, ",") != "configuration-dir,user-js" {
|
||||
t.Fatalf("%v", ids)
|
||||
want := "configuration-dir,prefs,accounts,configuration,tags,community-package,installation,owl,policies,spelling,preparation,start,launcher"
|
||||
if strings.Join(ids, ",") != want {
|
||||
t.Fatalf("the resources, in the order the node-engine applies them: %v, want %s", ids, want)
|
||||
}
|
||||
pkg := resource(t, m, "community-package")
|
||||
if pkg["type"] != "package" || pkg["package"] != packageFor || pkg["absent"] != true {
|
||||
t.Fatalf("the community package is declared absent (ADR 0180): %v", pkg)
|
||||
}
|
||||
places := map[string]string{"installation": installDir, "owl": filepath.Dir(owlFile),
|
||||
"policies": filepath.Dir(policyFile), "spelling": installDir + "/defaults/pref", "preparation": filepath.Dir(preparation)}
|
||||
arts := map[string]string{"installation": "archive", "owl": "owl", "policies": "distribution", "spelling": "defaults", "preparation": "tools"}
|
||||
for id, path := range places {
|
||||
r := resource(t, m, id)
|
||||
if r["type"] != "archive" || r["path"] != path || r["artifact"] != arts[id] || r["owner"] != nil {
|
||||
t.Errorf("%s is the %s archive unpacked at %s, root's: %v", id, arts[id], path, r)
|
||||
}
|
||||
}
|
||||
for _, never := range []string{".thunderbird", "prefs.js", "logins.json", "key4.db"} {
|
||||
if strings.Contains(raw, never) {
|
||||
if regexp.MustCompile(regexp.QuoteMeta(never) + `\b`).MatchString(raw) {
|
||||
t.Errorf("module.json names %q: the profile is Betterbird's", never)
|
||||
}
|
||||
}
|
||||
s := strings.ToLower(raw)
|
||||
for _, never := range []string{"/home/", "jochen", "g14", "shanks", "novox.be", "http://", "mediahuis"} {
|
||||
if strings.Contains(s, never) {
|
||||
t.Errorf("module.json names %q", never)
|
||||
}
|
||||
}
|
||||
file := m.Resources[1]
|
||||
content := file["content"].(string)
|
||||
if !strings.HasPrefix(content, mark) || !strings.Contains(content, "${setting:prefs}") || file["trusted"] != nil {
|
||||
t.Fatalf("the user.js begins with the mark, holds the setting, and stays trusted (only the terminal sets it): %v", file)
|
||||
}
|
||||
|
||||
// ADR 0329 §4–§6, ADR 0326, issue 586: every setting reaches the machine as a mergeable file of the account's that
|
||||
// names the keys it takes; four kinds and accounts and prefs are trusted, tags is not; no setting has a default in the
|
||||
// definition, since each is the operator's own value; and each file's default declares nothing.
|
||||
func TestTheSettingsAreMergeableFilesThatNameTheirKeysAndTagsAloneIsUntrusted(t *testing.T) {
|
||||
m, _ := readManifest(t)
|
||||
if len(m.Settings) != 0 {
|
||||
t.Fatalf("no setting has a default in the definition: the operator's setup is the settings' mesh-wide layer (issue 586, ADR 0262): %v", m.Settings)
|
||||
}
|
||||
// The default is a comment, so a machine without the setting places a valid, empty user.js.
|
||||
def := m.Settings["prefs"].(map[string]any)["default"].(string)
|
||||
if err := validUserJS(strings.ReplaceAll(content, "${setting:prefs}", def)); err != nil {
|
||||
files := map[string]struct {
|
||||
path, mode string
|
||||
takes []any
|
||||
trusted bool
|
||||
}{
|
||||
"prefs": {prefsFile, "0644", []any{"prefs"}, true},
|
||||
"accounts": {accountsFile, "0600", []any{"accounts"}, true},
|
||||
"configuration": {configurationFile, "0600", []any{"identities", "calendars", "address-books", "filters"}, true},
|
||||
"tags": {tagsFile, "0644", []any{"tags"}, false},
|
||||
}
|
||||
merging := 0
|
||||
for _, r := range m.Resources {
|
||||
if r["merge"] != nil {
|
||||
merging++
|
||||
}
|
||||
}
|
||||
if merging != len(files) {
|
||||
t.Fatalf("%d mergeable files, want %d", merging, len(files))
|
||||
}
|
||||
for id, want := range files {
|
||||
r := resource(t, m, id)
|
||||
if r["path"] != "${machine:account-home}/"+want.path || r["merge"] != "json" || r["trusted"] != want.trusted ||
|
||||
r["mode"] != want.mode || r["owner"] != "${machine:account}" || !reflect.DeepEqual(r["takes-settings"], want.takes) {
|
||||
t.Errorf("%s: %v", id, r)
|
||||
}
|
||||
var content map[string]any
|
||||
if err := json.Unmarshal([]byte(r["content"].(string)), &content); err != nil || len(content) != len(want.takes) {
|
||||
t.Errorf("%s's default holds each key it takes, empty: %v %v", id, content, err)
|
||||
}
|
||||
}
|
||||
got, err := parseAccounts([]byte(resource(t, m, "accounts")["content"].(string)))
|
||||
if err != nil || len(got) != 0 {
|
||||
t.Fatalf("its default declares no account: %v %v", got, err)
|
||||
}
|
||||
prefs, err := parseDeclaredPrefs([]byte(resource(t, m, "prefs")["content"].(string)))
|
||||
if err != nil || len(prefs) != 0 {
|
||||
t.Fatalf("its default declares no preference: %v %v", prefs, err)
|
||||
}
|
||||
var c Configuration
|
||||
if err := decodeInto([]byte(resource(t, m, "configuration")["content"].(string)), configurationFile, &c); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := decodeInto([]byte(resource(t, m, "tags")["content"].(string)), tagsFile, &c); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBothPinsAreAURLAVersionAndASha256AndOwlIsNeverInTheRepository(t *testing.T) {
|
||||
m, _ := readManifest(t)
|
||||
sha := regexp.MustCompile(`^[0-9a-f]{64}$`)
|
||||
for _, name := range []string{"archive", "owl"} {
|
||||
a := artifact(t, m, name)
|
||||
u, _ := a["url"].(string)
|
||||
if a["kind"] != "archive" || !strings.HasPrefix(u, "https://") || !sha.MatchString(fmt.Sprint(a["sha256"])) ||
|
||||
strings.TrimSpace(fmt.Sprint(a["version"])) == "" || a["from"] != nil {
|
||||
t.Errorf("%s is pinned by URL, version and sha256, and copied from nowhere in the repository: %v", name, a)
|
||||
}
|
||||
}
|
||||
if a := artifact(t, m, "archive"); a["within"] != "betterbird" {
|
||||
t.Errorf("the release unpacks into betterbird/, whose content is the installation: %v", a)
|
||||
}
|
||||
if a := artifact(t, m, "owl"); a["file"] != filepath.Base(owlFile) {
|
||||
t.Errorf("Owl's xpi is kept whole as %s: %v", filepath.Base(owlFile), a)
|
||||
}
|
||||
// Owl's licence forbids copying it to third parties: no xpi anywhere in the module's tree.
|
||||
filepath.Walk(filepath.Join("..", ".."), func(p string, info os.FileInfo, err error) error {
|
||||
if err == nil && strings.HasSuffix(strings.ToLower(p), ".xpi") {
|
||||
t.Errorf("%s is in the repository", p)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
// owlSupports is the range of Betterbird versions each pinned Owl declares in its manifest.json
|
||||
// (browser_specific_settings.gecko strict_min_version, strict_max_version), read from the vendor's xpi
|
||||
// when its pin was taken. Moving Owl's pin means reading the new xpi's range into here.
|
||||
var owlSupports = map[string][2]string{
|
||||
"1.5.6.1": {"115.0", "156.*"},
|
||||
}
|
||||
|
||||
// owlAccepts says why a Betterbird version is outside the range an Owl version declares, or nil.
|
||||
func owlAccepts(owl, betterbird string) error {
|
||||
r, known := owlSupports[owl]
|
||||
if !known {
|
||||
return fmt.Errorf("Owl %s's supported range is not recorded in owlSupports: read it from the xpi's manifest.json", owl)
|
||||
}
|
||||
major := func(v string) int {
|
||||
head, _, _ := strings.Cut(v, ".")
|
||||
n, err := strconv.Atoi(head)
|
||||
if err != nil {
|
||||
return -1
|
||||
}
|
||||
return n
|
||||
}
|
||||
b, lo, hi := major(betterbird), major(r[0]), major(r[1])
|
||||
if b < 0 || b < lo || b > hi {
|
||||
return fmt.Errorf("Betterbird %s is outside the range Owl %s supports (%s to %s): a Betterbird Owl does not support "+
|
||||
"waits for Owl's pin to move (novox/hq ADR 0316 §8)", betterbird, owl, r[0], r[1])
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func TestABetterbirdOwlDoesNotSupportIsRefused(t *testing.T) {
|
||||
m, _ := readManifest(t)
|
||||
owl, bb := fmt.Sprint(artifact(t, m, "owl")["version"]), fmt.Sprint(artifact(t, m, "archive")["version"])
|
||||
if err := owlAccepts(owl, bb); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, outside := range []string{"157.0esr-bb1", "102.15.1esr-bb4"} {
|
||||
if err := owlAccepts(owl, outside); err == nil {
|
||||
t.Errorf("Betterbird %s was accepted with Owl %s", outside, owl)
|
||||
}
|
||||
}
|
||||
if err := owlAccepts("9.9", bb); err == nil {
|
||||
t.Error("an Owl whose range nobody read was accepted")
|
||||
}
|
||||
}
|
||||
|
||||
func TestThePolicyTurnsTheUpdaterOffAndInstallsTheMeshsOwlEverywhere(t *testing.T) {
|
||||
m, _ := readManifest(t)
|
||||
if a := artifact(t, m, "distribution"); a["kind"] != "archive" || a["from"] != "install/distribution" {
|
||||
t.Fatalf("%v", a)
|
||||
}
|
||||
raw, err := os.ReadFile(filepath.Join("..", "..", "install", "distribution", "policies.json"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if why := policyProblem(raw); why != "" {
|
||||
t.Fatalf("the module's policies.json: %s", why)
|
||||
}
|
||||
entries, _ := os.ReadDir(filepath.Join("..", "..", "install", "distribution"))
|
||||
if len(entries) != 1 {
|
||||
t.Fatalf("the distribution directory holds the policy alone: %v", entries)
|
||||
}
|
||||
if a := artifact(t, m, "defaults"); a["kind"] != "archive" || a["from"] != "install/defaults-pref" {
|
||||
t.Fatalf("%v", a)
|
||||
}
|
||||
prefs, err := os.ReadFile(filepath.Join("..", "..", "install", "defaults-pref", "vendor-prefs.js"))
|
||||
if err != nil || !strings.Contains(string(prefs), `pref("spellchecker.dictionary_path", "/usr/share/hunspell");`) {
|
||||
t.Fatalf("the dictionaries are named as a default, in place of a link: %s %v", prefs, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheStartPreparesThenStartsBetterbirdAndTheLauncherUsesIt(t *testing.T) {
|
||||
m, _ := readManifest(t)
|
||||
start := resource(t, m, "start")
|
||||
content := start["content"].(string)
|
||||
if start["path"] != "/usr/local/bin/betterbird" || start["mode"] != "0755" ||
|
||||
!strings.Contains(content, "\n"+preparation+" prepare ||") || !strings.HasSuffix(content, "\nexec "+startBin+" \"$@\"\n") {
|
||||
t.Fatalf("the start runs the preparation, says when it did not complete, and starts Betterbird either way: %v", start)
|
||||
}
|
||||
launcher := resource(t, m, "launcher")
|
||||
lc := launcher["content"].(string)
|
||||
if launcher["path"] != "/usr/local/share/applications/eu.betterbird.Betterbird.desktop" ||
|
||||
!strings.Contains(lc, "\nExec=/usr/local/bin/betterbird %u\n") || !strings.Contains(lc, "\nIcon="+installDir+"/") ||
|
||||
!strings.Contains(lc, "\nStartupWMClass=eu.betterbird.Betterbird\n") {
|
||||
t.Fatalf("the launcher starts the module's start and names the icon in the installation: %v", launcher)
|
||||
}
|
||||
if b := artifact(t, m, "tools"); filepath.Base(preparation) != b["binary"] {
|
||||
t.Fatalf("the preparation runs the tools' binary: %v", b)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheToolsAgreeWithTheManifest(t *testing.T) {
|
||||
m, raw := readManifest(t)
|
||||
m, _ := readManifest(t)
|
||||
served := map[string]bool{}
|
||||
for _, tool := range tools() {
|
||||
served[tool.Name] = true
|
||||
@@ -94,17 +303,14 @@ func TestTheToolsAgreeWithTheManifest(t *testing.T) {
|
||||
for name := range served {
|
||||
t.Errorf("the bundle serves %s, which module.json does not list", name)
|
||||
}
|
||||
if len(m.Build.Artifacts) != 1 {
|
||||
t.Fatalf("%v", m.Build.Artifacts)
|
||||
}
|
||||
b := m.Build.Artifacts[0]
|
||||
b := artifact(t, m, "tools")
|
||||
if b["kind"] != "bundle" || b["language"] != "go" || b["from"] != "cmd/betterbird-tools" || b["binary"] != "betterbird-tools" {
|
||||
t.Errorf("the Go tools bundle: %v", b)
|
||||
}
|
||||
s := strings.ToLower(raw)
|
||||
for _, never := range []string{"/home/", "jochen", "g14", "shanks", "novox.be", "http"} {
|
||||
if strings.Contains(s, never) {
|
||||
t.Errorf("module.json names %q", never)
|
||||
}
|
||||
// The tools propose through the controller's settings verb (ADR 0317 §9), and betterbird_derive asks the controller
|
||||
// which nodes hold the module and each of them for its profile (ADR 0329 §10).
|
||||
if !reflect.DeepEqual(m.Invokes, []string{"seat:mesh-controller.settings", "seat:mesh-controller.nodes",
|
||||
"seat:mesh-controller.node", "betterbird.betterbird_derive"}) {
|
||||
t.Errorf("the module invokes the controller's settings, nodes and node verbs and its own derive: %v", m.Invokes)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,749 @@
|
||||
package main
|
||||
|
||||
// How the five settings of ADR 0329 reach the profile (§7–§9).
|
||||
//
|
||||
// Identities, calendars, address books and tags are matched as accounts are (ADR 0317): an entry the module
|
||||
// recorded on this node is kept by its recorded key; else an entry under the module's own key is its creation;
|
||||
// else a key a node's layer names is adopted when the profile holds it and it matches; else the profile's
|
||||
// entries that match (an identity by its address within its account, a calendar and an address book by their
|
||||
// address, a tag by its key): one is adopted, none means the entry is created under the module's own key, more
|
||||
// than one is refused with the remedy. Their declared fields are written into the user.js the module places,
|
||||
// where they win at every start. An entry leaving the setting releases what was adopted and removes the
|
||||
// preferences of what was created; never a calendar's events, an address book's contacts or a cache.
|
||||
//
|
||||
// What Betterbird rewrites itself is written only while it does not run (§9): the lists in prefs.js (an adopted
|
||||
// account's identities, the calendars' order) and each account's msgFilterRules.dat. Until then the module's
|
||||
// check counts, per kind, the changes that wait.
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// configRecordFile is where the module records, per profile, what it adopted or created of the five kinds and
|
||||
// which filter files it wrote (ADR 0329 §7, §8, ADR 0317 §3).
|
||||
const configRecordFile = ".config/betterbird/configuration-record.json"
|
||||
|
||||
// resolvedEntry is one declared entry as it resolved on this node.
|
||||
type resolvedEntry struct {
|
||||
Kind string `json:"kind"`
|
||||
Name string `json:"name"`
|
||||
How string `json:"how"`
|
||||
Key string `json:"key,omitempty"`
|
||||
Why string `json:"why,omitempty"`
|
||||
}
|
||||
|
||||
type recordedEntry struct {
|
||||
How string `json:"how"`
|
||||
Key string `json:"key"`
|
||||
}
|
||||
|
||||
// filterRecord is what the module did with one account's filter file on a profile.
|
||||
type filterRecord struct {
|
||||
// File is the account's filter file as the module last wrote it, and Kept the file it replaced at its first write.
|
||||
File string `json:"file"`
|
||||
Kept string `json:"kept,omitempty"`
|
||||
// Released says the account left the setting filters: the module no longer writes its file (§8).
|
||||
Released bool `json:"released,omitempty"`
|
||||
}
|
||||
|
||||
type configRecord struct {
|
||||
Entries map[string]map[string]recordedEntry `json:"entries,omitempty"`
|
||||
Filters map[string]filterRecord `json:"filters,omitempty"`
|
||||
}
|
||||
|
||||
func (m *Machine) readConfigRecord(profileDir string) configRecord {
|
||||
all := map[string]configRecord{}
|
||||
if raw, err := readBounded(m.home(configRecordFile)); err == nil {
|
||||
_ = json.Unmarshal(raw, &all)
|
||||
}
|
||||
r := all[profileDir]
|
||||
if r.Entries == nil {
|
||||
r.Entries = map[string]map[string]recordedEntry{}
|
||||
}
|
||||
if r.Filters == nil {
|
||||
r.Filters = map[string]filterRecord{}
|
||||
}
|
||||
return r
|
||||
}
|
||||
|
||||
func (m *Machine) writeConfigRecord(profileDir string, r configRecord) error {
|
||||
all := map[string]configRecord{}
|
||||
if raw, err := readBounded(m.home(configRecordFile)); err == nil {
|
||||
_ = json.Unmarshal(raw, &all)
|
||||
}
|
||||
all[profileDir] = r
|
||||
body, err := json.MarshalIndent(all, "", " ")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return writeAtomic(m.home(configRecordFile), append(body, '\n'), 0o600)
|
||||
}
|
||||
|
||||
// writeAtomic writes a file beside itself and renames it into place.
|
||||
func writeAtomic(path string, data []byte, mode os.FileMode) error {
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
|
||||
return err
|
||||
}
|
||||
tmp := path + ".mesh-new"
|
||||
if err := writeSynced(tmp, data, mode); err != nil {
|
||||
os.Remove(tmp)
|
||||
return err
|
||||
}
|
||||
if err := os.Rename(tmp, path); err != nil {
|
||||
os.Remove(tmp)
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// The module's own keys for what it creates: each names the entry.
|
||||
func ownIdentityKey(name string) string { return "ident_mesh_" + name }
|
||||
func ownCalendarKey(name string) string { return "mesh_" + name }
|
||||
func ownBookKey(name string) string { return "mesh_" + name }
|
||||
|
||||
var (
|
||||
ownIdentity = regexp.MustCompile(`^ident_mesh_[a-z][a-z0-9]{0,30}$`)
|
||||
ownMeshKey = regexp.MustCompile(`^mesh_[a-z][a-z0-9]{0,30}$`)
|
||||
)
|
||||
|
||||
// configPlan is what an apply does for the five kinds on this profile.
|
||||
type configPlan struct {
|
||||
Entries []resolvedEntry
|
||||
// lines are the user.js statements, a comment before each entry's.
|
||||
lines []string
|
||||
// identityLists are, per adopted account key, the identity keys of the module's own it wants in its list.
|
||||
identityLists map[string][]string
|
||||
// calendarOrder are the calendar keys the module created, for calendar.list.sortOrder.
|
||||
calendarOrder []string
|
||||
// removals are the preference prefixes of what the module created and the setting no longer holds.
|
||||
removals []string
|
||||
filters []filterPlan
|
||||
// released are the accounts that left the setting filters since the module last wrote their file.
|
||||
released []string
|
||||
record configRecord
|
||||
problem string
|
||||
}
|
||||
|
||||
// filterPlan is one declared account's filter file on this node.
|
||||
type filterPlan struct {
|
||||
Account string `json:"account"`
|
||||
File string `json:"file,omitempty"`
|
||||
// Waiting says why the file cannot be written yet, when it cannot.
|
||||
Waiting string `json:"waiting,omitempty"`
|
||||
Refused string `json:"refused,omitempty"`
|
||||
want string
|
||||
have string
|
||||
exists bool
|
||||
path string
|
||||
}
|
||||
|
||||
func (p filterPlan) changes() bool { return p.Refused == "" && p.path != "" && p.want != p.have }
|
||||
|
||||
// planConfiguration resolves every declared entry against the profile and says what an apply writes.
|
||||
func (m *Machine) planConfiguration(profile Profile, c Configuration, accounts map[string]Account,
|
||||
ra []resolvedAccount, prefs map[string]string) configPlan {
|
||||
rec := m.readConfigRecord(profile.Dir)
|
||||
plan := configPlan{identityLists: map[string][]string{},
|
||||
record: configRecord{Entries: map[string]map[string]recordedEntry{}, Filters: map[string]filterRecord{}}}
|
||||
byName := map[string]resolvedAccount{}
|
||||
for _, r := range ra {
|
||||
byName[r.name] = r
|
||||
}
|
||||
have := profileAccounts(prefs)
|
||||
listOfAccount := map[string][]string{}
|
||||
for _, a := range have {
|
||||
listOfAccount[a.Key] = a.Identities
|
||||
}
|
||||
identityKeyOf := map[string]string{}
|
||||
|
||||
// Identities.
|
||||
taken := map[string]bool{}
|
||||
for _, name := range sortedKeys(c.Identities) {
|
||||
id := c.Identities[name]
|
||||
e := resolvedEntry{Kind: kindIdentities, Name: name}
|
||||
acct, declared := byName[id.Account]
|
||||
switch {
|
||||
case !declared:
|
||||
e.How, e.Why = refused, fmt.Sprintf("its account %s is not in the setting accounts", id.Account)
|
||||
case acct.How == refused:
|
||||
e.How, e.Why = refused, fmt.Sprintf("its account %s is refused: betterbird_check says why", id.Account)
|
||||
case id.Address == "":
|
||||
e.How, e.Key = acct.How, acct.Keys.Identity
|
||||
if taken[e.Key] {
|
||||
e.How, e.Key, e.Why = refused, "", fmt.Sprintf("account %s has its default identity declared twice", id.Account)
|
||||
}
|
||||
default:
|
||||
list := listOfAccount[acct.Keys.Account]
|
||||
var others []string
|
||||
if len(list) > 1 {
|
||||
others = list[1:]
|
||||
}
|
||||
present := func(k string) bool {
|
||||
for _, x := range others {
|
||||
if x == k {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
matches := func(k string) bool {
|
||||
return strings.EqualFold(unquote(prefs["mail.identity."+k+".useremail"]), id.Address)
|
||||
}
|
||||
e = resolveEntry(kindIdentities, name, id.Key, ownIdentityKey(name), rec.Entries[kindIdentities][name],
|
||||
others, present, matches, taken, "an identity of its account with its address",
|
||||
"give the one to adopt as key in this node's layer of the setting identities")
|
||||
if e.How == created {
|
||||
plan.identityLists[acct.Keys.Account] = append(plan.identityLists[acct.Keys.Account], e.Key)
|
||||
}
|
||||
}
|
||||
if e.How != refused {
|
||||
taken[e.Key] = true
|
||||
identityKeyOf[name] = e.Key
|
||||
plan.lines = append(plan.lines, fmt.Sprintf("// identity %s: %s under %s", name, e.How, e.Key))
|
||||
plan.lines = append(plan.lines, identityLines(e, id, acct)...)
|
||||
}
|
||||
plan.Entries = append(plan.Entries, e)
|
||||
}
|
||||
// A created account's identity list is the module's whole: its default and the identities it created.
|
||||
for _, name := range sortedKeys(byName) {
|
||||
r := byName[name]
|
||||
if r.How == created && len(plan.identityLists[r.Keys.Account]) > 0 {
|
||||
list := append([]string{r.Keys.Identity}, plan.identityLists[r.Keys.Account]...)
|
||||
plan.lines = append(plan.lines, fmt.Sprintf("user_pref(%s, %s);", quoted("mail.account."+r.Keys.Account+".identities"),
|
||||
quoted(strings.Join(list, ","))))
|
||||
delete(plan.identityLists, r.Keys.Account)
|
||||
}
|
||||
}
|
||||
// Adopted accounts keep their list in prefs.js: the module's own keys are written into it while Betterbird is
|
||||
// closed. An adopted account whose list holds none of them and wants none is left out.
|
||||
for _, a := range have {
|
||||
if _, wanted := plan.identityLists[a.Key]; wanted {
|
||||
continue
|
||||
}
|
||||
for _, k := range a.Identities {
|
||||
if ownIdentity.MatchString(k) {
|
||||
plan.identityLists[a.Key] = []string{}
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Calendars.
|
||||
cals := profileCalendars(prefs)
|
||||
calKeys := map[string]profileCalendar{}
|
||||
var caldav []string
|
||||
for _, pc := range cals {
|
||||
calKeys[pc.Key] = pc
|
||||
if pc.Kind == "caldav" {
|
||||
caldav = append(caldav, pc.Key)
|
||||
}
|
||||
}
|
||||
taken = map[string]bool{}
|
||||
for _, name := range sortedKeys(c.Calendars) {
|
||||
cal := c.Calendars[name]
|
||||
present := func(k string) bool { pc, ok := calKeys[k]; return ok && pc.Kind == "caldav" }
|
||||
matches := func(k string) bool { return sameURL(calKeys[k].Value.URL, cal.URL) }
|
||||
e := resolveEntry(kindCalendars, name, cal.Key, ownCalendarKey(name), rec.Entries[kindCalendars][name],
|
||||
caldav, present, matches, taken, "a CalDAV calendar at its address",
|
||||
"remove the duplicate in Betterbird, or give the one to adopt as key in this node's layer of the setting calendars")
|
||||
if e.How != refused {
|
||||
taken[e.Key] = true
|
||||
if e.How == created {
|
||||
plan.calendarOrder = append(plan.calendarOrder, e.Key)
|
||||
}
|
||||
plan.lines = append(plan.lines, fmt.Sprintf("// calendar %s: %s under %s", name, e.How, e.Key))
|
||||
plan.lines = append(plan.lines, calendarLines(e, cal, identityKeyOf[cal.Invitations])...)
|
||||
}
|
||||
plan.Entries = append(plan.Entries, e)
|
||||
}
|
||||
|
||||
// Address books.
|
||||
books := profileAddressBooks(prefs)
|
||||
bookKeys := map[string]profileAddressBook{}
|
||||
var carddav []string
|
||||
for _, b := range books {
|
||||
bookKeys[b.Key] = b
|
||||
if b.CardDAV {
|
||||
carddav = append(carddav, b.Key)
|
||||
}
|
||||
}
|
||||
taken = map[string]bool{}
|
||||
for _, name := range sortedKeys(c.AddressBooks) {
|
||||
ab := c.AddressBooks[name]
|
||||
present := func(k string) bool { b, ok := bookKeys[k]; return ok && b.CardDAV }
|
||||
matches := func(k string) bool { return sameURL(bookKeys[k].Value.URL, ab.URL) }
|
||||
e := resolveEntry(kindAddressBooks, name, ab.Key, ownBookKey(name), rec.Entries[kindAddressBooks][name],
|
||||
carddav, present, matches, taken, "a CardDAV address book at its address",
|
||||
"remove the duplicate in Betterbird, or give the one to adopt as key in this node's layer of the setting address-books")
|
||||
if e.How != refused {
|
||||
taken[e.Key] = true
|
||||
plan.lines = append(plan.lines, fmt.Sprintf("// address book %s: %s under %s", name, e.How, e.Key))
|
||||
plan.lines = append(plan.lines, addressBookLines(e, ab, name)...)
|
||||
}
|
||||
plan.Entries = append(plan.Entries, e)
|
||||
}
|
||||
|
||||
// Tags: a tag's key is its own, unique in a profile, so a match is never ambiguous.
|
||||
tagKeys := map[string]bool{}
|
||||
for _, t := range profileTags(prefs) {
|
||||
tagKeys[t.Key] = true
|
||||
}
|
||||
for _, key := range sortedKeys(c.Tags) {
|
||||
t := c.Tags[key]
|
||||
e := resolvedEntry{Kind: kindTags, Name: key, Key: key, How: adopted}
|
||||
if was, ok := rec.Entries[kindTags][key]; ok && was.How == created {
|
||||
e.How = created
|
||||
} else if !tagKeys[key] {
|
||||
e.How = created
|
||||
}
|
||||
plan.lines = append(plan.lines, fmt.Sprintf("// tag %s: %s", key, e.How))
|
||||
plan.lines = append(plan.lines, tagLines(key, t)...)
|
||||
plan.Entries = append(plan.Entries, e)
|
||||
}
|
||||
|
||||
// What the setting no longer holds: an adopted entry is released (its lines go), a created one's preferences
|
||||
// are removed from prefs.js while Betterbird is closed.
|
||||
declaredNow := map[string]map[string]bool{}
|
||||
for _, e := range plan.Entries {
|
||||
if declaredNow[e.Kind] == nil {
|
||||
declaredNow[e.Kind] = map[string]bool{}
|
||||
}
|
||||
declaredNow[e.Kind][e.Name] = true
|
||||
if e.How != refused {
|
||||
if plan.record.Entries[e.Kind] == nil {
|
||||
plan.record.Entries[e.Kind] = map[string]recordedEntry{}
|
||||
}
|
||||
plan.record.Entries[e.Kind][e.Name] = recordedEntry{How: e.How, Key: e.Key}
|
||||
}
|
||||
}
|
||||
for _, kind := range []string{kindIdentities, kindCalendars, kindAddressBooks, kindTags} {
|
||||
for _, name := range sortedKeys(rec.Entries[kind]) {
|
||||
was := rec.Entries[kind][name]
|
||||
if declaredNow[kind][name] || was.How != created {
|
||||
continue
|
||||
}
|
||||
switch kind {
|
||||
case kindIdentities:
|
||||
plan.removals = append(plan.removals, "mail.identity."+was.Key+".")
|
||||
case kindCalendars:
|
||||
plan.removals = append(plan.removals, "calendar.registry."+was.Key+".")
|
||||
case kindAddressBooks:
|
||||
plan.removals = append(plan.removals, "ldap_2.servers."+was.Key+".")
|
||||
case kindTags:
|
||||
plan.removals = append(plan.removals, "mailnews.tags."+was.Key+".")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Filters, owned whole per declared account (§8).
|
||||
for _, account := range sortedAccountsOf(c.Filters) {
|
||||
fp := filterPlan{Account: account}
|
||||
r, ok := byName[account]
|
||||
switch {
|
||||
case !ok:
|
||||
fp.Refused = fmt.Sprintf("its account %s is not in the setting accounts", account)
|
||||
case r.How == refused:
|
||||
fp.Refused = fmt.Sprintf("its account %s is refused: betterbird_check says why", account)
|
||||
default:
|
||||
dir := m.accountDirectory(profile.path, prefs, r.Keys.Server)
|
||||
if dir == "" {
|
||||
fp.Waiting = "Betterbird has not made the account's directory yet: it does at its first start with the account"
|
||||
break
|
||||
}
|
||||
fp.path = filepath.Join(dir, filterFileName)
|
||||
fp.File = m.tilde(strings.TrimPrefix(fp.path, m.Root))
|
||||
fp.want = renderFilters(c.Filters[account], folderBase(prefs, r.Keys.Server))
|
||||
raw, err := readBounded(fp.path)
|
||||
fp.exists = err == nil
|
||||
fp.have = string(raw)
|
||||
}
|
||||
was := rec.Filters[account]
|
||||
was.Released = false
|
||||
if fp.path != "" {
|
||||
was.File = fp.File
|
||||
}
|
||||
if was.File != "" || was.Kept != "" {
|
||||
plan.record.Filters[account] = was
|
||||
}
|
||||
plan.filters = append(plan.filters, fp)
|
||||
}
|
||||
for _, account := range sortedKeys(rec.Filters) {
|
||||
if _, still := c.Filters[account]; still {
|
||||
continue
|
||||
}
|
||||
was := rec.Filters[account]
|
||||
if !was.Released {
|
||||
plan.released = append(plan.released, account)
|
||||
}
|
||||
was.Released = true
|
||||
plan.record.Filters[account] = was
|
||||
}
|
||||
return plan
|
||||
}
|
||||
|
||||
// resolveEntry resolves an identity, a calendar or an address book as ADR 0317 resolves an account (§7).
|
||||
func resolveEntry(kind, name, layerKey, own string, was recordedEntry, candidates []string, present, matches func(string) bool,
|
||||
taken map[string]bool, what, remedy string) resolvedEntry {
|
||||
e := resolvedEntry{Kind: kind, Name: name}
|
||||
refuse := func(format string, args ...any) resolvedEntry {
|
||||
e.How, e.Key, e.Why = refused, "", fmt.Sprintf(format, args...)
|
||||
return e
|
||||
}
|
||||
// 1. Recorded, while the profile still holds it.
|
||||
if was.Key != "" && !taken[was.Key] && (present(was.Key) || was.How == created) && (layerKey == "" || layerKey == was.Key) {
|
||||
e.How, e.Key = was.How, was.Key
|
||||
return e
|
||||
}
|
||||
// 2. Under the module's own key: its creation.
|
||||
if present(own) {
|
||||
e.How, e.Key = created, own
|
||||
return e
|
||||
}
|
||||
// 3. The key a node's layer names.
|
||||
if layerKey != "" {
|
||||
switch {
|
||||
case !present(layerKey):
|
||||
return refuse("the key %s this node's layer names is not in the profile", layerKey)
|
||||
case taken[layerKey]:
|
||||
return refuse("the key %s is already another declared entry's", layerKey)
|
||||
case !matches(layerKey):
|
||||
return refuse("the key %s this node's layer names is not %s", layerKey, what)
|
||||
}
|
||||
e.How, e.Key = adopted, layerKey
|
||||
return e
|
||||
}
|
||||
// 4. By the match.
|
||||
var found []string
|
||||
for _, k := range candidates {
|
||||
if !taken[k] && !ownMeshKey.MatchString(k) && !ownIdentity.MatchString(k) && matches(k) {
|
||||
found = append(found, k)
|
||||
}
|
||||
}
|
||||
switch len(found) {
|
||||
case 0:
|
||||
e.How, e.Key = created, own
|
||||
case 1:
|
||||
e.How, e.Key = adopted, found[0]
|
||||
default:
|
||||
return refuse("%d entries in the profile are %s (%s): %s (novox/hq ADR 0329 §7)", len(found), what,
|
||||
strings.Join(found, ", "), remedy)
|
||||
}
|
||||
return e
|
||||
}
|
||||
|
||||
// sameURL compares two addresses without a trailing slash's difference.
|
||||
func sameURL(a, b string) bool {
|
||||
return a != "" && strings.EqualFold(strings.TrimRight(a, "/"), strings.TrimRight(b, "/"))
|
||||
}
|
||||
|
||||
func pref(name, value string) string { return fmt.Sprintf("user_pref(%s, %s);", quoted(name), value) }
|
||||
|
||||
func boolWord(b bool) string { return strconv.FormatBool(b) }
|
||||
|
||||
// identityLines are an identity's declared fields under its key. The default identity's name and address are
|
||||
// the setting accounts' (§4), so only another identity carries them here.
|
||||
func identityLines(e resolvedEntry, id *Identity, acct resolvedAccount) []string {
|
||||
p := "mail.identity." + e.Key + "."
|
||||
var out []string
|
||||
if id.Address != "" {
|
||||
out = append(out, pref(p+"fullName", jsQuoted(id.Name)), pref(p+"useremail", jsQuoted(id.Address)))
|
||||
if e.How == created {
|
||||
out = append(out, pref(p+"valid", "true"), pref(p+"smtpServer", quoted(acct.Keys.SMTP)))
|
||||
}
|
||||
}
|
||||
above := 0
|
||||
if id.SignatureAboveQuote {
|
||||
above = 1
|
||||
}
|
||||
out = append(out,
|
||||
pref(p+"reply_to", jsQuoted(id.ReplyTo)),
|
||||
pref(p+"organization", jsQuoted(id.Organization)),
|
||||
pref(p+"htmlSigText", jsQuoted(strings.Join(id.Signature, "\n"))),
|
||||
pref(p+"htmlSigFormat", boolWord(id.SignatureHTML)),
|
||||
pref(p+"attach_signature", "false"),
|
||||
pref(p+"reply_on_top", strconv.Itoa(above)),
|
||||
pref(p+"sig_bottom", boolWord(!id.SignatureAboveQuote)))
|
||||
return out
|
||||
}
|
||||
|
||||
func calendarLines(e resolvedEntry, cal *Calendar, identityKey string) []string {
|
||||
p := "calendar.registry." + e.Key + "."
|
||||
var out []string
|
||||
if e.How == created {
|
||||
out = append(out, pref(p+"type", quoted("caldav")), pref(p+"cache.enabled", "true"),
|
||||
pref(p+"calendar-main-in-composite", "true"))
|
||||
}
|
||||
reminders := cal.Reminders == nil || *cal.Reminders
|
||||
out = append(out, pref(p+"uri", jsQuoted(cal.URL)), pref(p+"name", jsQuoted(cal.Name)))
|
||||
if cal.Color != "" {
|
||||
out = append(out, pref(p+"color", jsQuoted(strings.ToLower(cal.Color))))
|
||||
}
|
||||
out = append(out, pref(p+"readOnly", boolWord(cal.ReadOnly)), pref(p+"suppressAlarms", boolWord(!reminders)),
|
||||
pref(p+"username", jsQuoted(cal.User)), pref(p+"imip.identity.key", jsQuoted(identityKey)))
|
||||
return out
|
||||
}
|
||||
|
||||
func addressBookLines(e resolvedEntry, ab *AddressBook, name string) []string {
|
||||
p := "ldap_2.servers." + e.Key + "."
|
||||
var out []string
|
||||
if e.How == created {
|
||||
out = append(out, pref(p+"dirType", "102"), pref(p+"filename", quoted("abook-mesh_"+name+".sqlite")))
|
||||
}
|
||||
return append(out, pref(p+"description", jsQuoted(ab.Name)), pref(p+"carddav.url", jsQuoted(ab.URL)),
|
||||
pref(p+"carddav.username", jsQuoted(ab.User)))
|
||||
}
|
||||
|
||||
func tagLines(key string, t *Tag) []string {
|
||||
p := "mailnews.tags." + key + "."
|
||||
out := []string{pref(p+"tag", jsQuoted(t.Name))}
|
||||
if t.Color != "" {
|
||||
out = append(out, pref(p+"color", jsQuoted(strings.ToUpper(t.Color))))
|
||||
}
|
||||
if t.Order > 0 {
|
||||
out = append(out, pref(p+"ordinal", quoted(tagOrdinal(t.Order))))
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// configBlock is the user.js block of the five kinds.
|
||||
func (p configPlan) configBlock() string {
|
||||
if len(p.lines) == 0 {
|
||||
return ""
|
||||
}
|
||||
return "\n// The identities, calendars, address books and tags the settings declare (novox/hq ADR 0329), under the\n" +
|
||||
"// keys each was adopted or created under. No password, token or cookie.\n" + strings.Join(p.lines, "\n") + "\n"
|
||||
}
|
||||
|
||||
// closedFor says why the profile's own files may not be written now, or "": Betterbird runs, or the profile's
|
||||
// lock says one holds it (§9).
|
||||
func (m *Machine) closedFor(profile string) string {
|
||||
if ps := m.mainProcs(); len(ps) > 0 {
|
||||
return fmt.Sprintf("Betterbird is running (pid %d), and it rewrites these files when it quits", ps[0].PID)
|
||||
}
|
||||
if _, err := os.Lstat(filepath.Join(profile, "lock")); err == nil {
|
||||
return "the profile is locked (its lock file is there): a Betterbird holds it, or one ended without quitting"
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// prefsChanges is prefs.js as the plan wants it, and how many changes each kind has in it.
|
||||
func (p configPlan) prefsChanges(text string) (string, map[string]int) {
|
||||
counts := map[string]int{}
|
||||
var lines []string
|
||||
done := map[string]bool{}
|
||||
s := bufio.NewScanner(strings.NewReader(text))
|
||||
s.Buffer(make([]byte, 64<<10), 4<<20)
|
||||
for s.Scan() {
|
||||
line := s.Text()
|
||||
mm := prefLine.FindStringSubmatch(line)
|
||||
if mm == nil {
|
||||
lines = append(lines, line)
|
||||
continue
|
||||
}
|
||||
name := mm[1]
|
||||
removed := false
|
||||
for _, prefix := range p.removals {
|
||||
if strings.HasPrefix(name, prefix) {
|
||||
removed = true
|
||||
counts[kindOfPrefix(prefix)]++
|
||||
break
|
||||
}
|
||||
}
|
||||
if removed {
|
||||
continue
|
||||
}
|
||||
if acct, ok := strings.CutPrefix(name, "mail.account."); ok && strings.HasSuffix(acct, ".identities") {
|
||||
key := strings.TrimSuffix(acct, ".identities")
|
||||
if want, ok := p.identityLists[key]; ok {
|
||||
done[name] = true
|
||||
next := listedWithOwn(listOf(mm[2]), want, ownIdentity.MatchString, ",")
|
||||
if next != strings.Join(listOf(mm[2]), ",") {
|
||||
line = pref(name, quoted(next))
|
||||
counts[kindIdentities]++
|
||||
}
|
||||
}
|
||||
}
|
||||
if name == "calendar.list.sortOrder" {
|
||||
done[name] = true
|
||||
have := strings.Fields(unquote(mm[2]))
|
||||
next := listedWithOwn(have, p.calendarOrder, ownMeshKey.MatchString, " ")
|
||||
if next != strings.Join(have, " ") {
|
||||
line = pref(name, quoted(next))
|
||||
counts[kindCalendars]++
|
||||
}
|
||||
}
|
||||
lines = append(lines, line)
|
||||
}
|
||||
if !done["calendar.list.sortOrder"] && len(p.calendarOrder) > 0 {
|
||||
lines = append(lines, pref("calendar.list.sortOrder", quoted(strings.Join(p.calendarOrder, " "))))
|
||||
counts[kindCalendars]++
|
||||
}
|
||||
out := strings.Join(lines, "\n")
|
||||
if text == "" || strings.HasSuffix(text, "\n") {
|
||||
out += "\n"
|
||||
}
|
||||
return out, counts
|
||||
}
|
||||
|
||||
func kindOfPrefix(prefix string) string {
|
||||
switch {
|
||||
case strings.HasPrefix(prefix, "mail.identity."):
|
||||
return kindIdentities
|
||||
case strings.HasPrefix(prefix, "calendar."):
|
||||
return kindCalendars
|
||||
case strings.HasPrefix(prefix, "ldap_2."):
|
||||
return kindAddressBooks
|
||||
}
|
||||
return kindTags
|
||||
}
|
||||
|
||||
// listedWithOwn is a list with the module's own keys made the wanted ones, every other key in its place.
|
||||
func listedWithOwn(have, wanted []string, own func(string) bool, sep string) string {
|
||||
want := map[string]bool{}
|
||||
for _, k := range wanted {
|
||||
want[k] = true
|
||||
}
|
||||
var out []string
|
||||
there := map[string]bool{}
|
||||
for _, k := range have {
|
||||
if own(k) && !want[k] {
|
||||
continue
|
||||
}
|
||||
out = append(out, k)
|
||||
there[k] = true
|
||||
}
|
||||
for _, k := range wanted {
|
||||
if !there[k] {
|
||||
out = append(out, k)
|
||||
}
|
||||
}
|
||||
return strings.Join(out, sep)
|
||||
}
|
||||
|
||||
// Waiting counts, per kind, the changes of what Betterbird rewrites that wait for it to close (§9).
|
||||
func (m *Machine) waiting(profile string, p configPlan, ra []resolvedAccount) map[string]int {
|
||||
raw, _ := readBounded(filepath.Join(profile, "prefs.js"))
|
||||
_, counts := p.prefsChanges(string(raw))
|
||||
var wantAccounts, wantSMTP []string
|
||||
for _, r := range ra {
|
||||
if r.How == created {
|
||||
wantAccounts, wantSMTP = append(wantAccounts, r.Keys.Account), append(wantSMTP, r.Keys.SMTP)
|
||||
}
|
||||
}
|
||||
if _, changed := withListed(string(raw), map[string][]string{
|
||||
"mail.accountmanager.accounts": wantAccounts, "mail.smtpservers": wantSMTP}); changed {
|
||||
counts["accounts"]++
|
||||
}
|
||||
for _, f := range p.filters {
|
||||
if f.changes() {
|
||||
counts[kindFilters]++
|
||||
}
|
||||
}
|
||||
for k, v := range counts {
|
||||
if v == 0 {
|
||||
delete(counts, k)
|
||||
}
|
||||
}
|
||||
return counts
|
||||
}
|
||||
|
||||
// writeClosed writes what Betterbird rewrites itself, only while it is closed: prefs.js's lists and the
|
||||
// removals, and each declared account's filter file, keeping the file it replaces at its first write. It
|
||||
// answers what it wrote, or why it waits.
|
||||
func (m *Machine) writeClosed(profile Profile, p *configPlan) (string, error) {
|
||||
var said []string
|
||||
path := filepath.Join(profile.path, "prefs.js")
|
||||
raw, err := readBounded(path)
|
||||
if err != nil && !errors.Is(err, os.ErrNotExist) {
|
||||
return "", err
|
||||
}
|
||||
next, counts := p.prefsChanges(string(raw))
|
||||
pending := next != string(raw)
|
||||
for _, f := range p.filters {
|
||||
pending = pending || f.changes()
|
||||
}
|
||||
if !pending {
|
||||
return "unchanged", nil
|
||||
}
|
||||
if why := m.closedFor(profile.path); why != "" {
|
||||
return "", errors.New(why + ": the identities' and calendars' lists, the removals and the filter files are " +
|
||||
"written at its next start from the launcher, or by betterbird_restart")
|
||||
}
|
||||
if next != string(raw) {
|
||||
mode := os.FileMode(0o600)
|
||||
if st, err := os.Stat(path); err == nil {
|
||||
mode = st.Mode().Perm()
|
||||
}
|
||||
if err := writeAtomic(path, []byte(next), mode); err != nil {
|
||||
return "", err
|
||||
}
|
||||
said = append(said, fmt.Sprintf("prefs.js: %d change(s) to the lists and removals", sum(counts)))
|
||||
}
|
||||
for i, f := range p.filters {
|
||||
if !f.changes() {
|
||||
continue
|
||||
}
|
||||
was := p.record.Filters[f.Account]
|
||||
if was.Kept == "" && f.exists {
|
||||
kept := m.keptFilterFile(profile.Dir, f.Account)
|
||||
if err := writeAtomic(kept, []byte(f.have), 0o600); err != nil {
|
||||
return strings.Join(said, "; "), fmt.Errorf("the filter file of %s was not written: the file it replaces could not be kept: %v", f.Account, err)
|
||||
}
|
||||
was.Kept = m.tilde(strings.TrimPrefix(kept, m.Root))
|
||||
}
|
||||
mode := os.FileMode(0o644)
|
||||
if st, err := os.Stat(f.path); err == nil {
|
||||
mode = st.Mode().Perm()
|
||||
}
|
||||
if err := writeAtomic(f.path, []byte(f.want), mode); err != nil {
|
||||
return strings.Join(said, "; "), err
|
||||
}
|
||||
was.File = f.File
|
||||
p.record.Filters[f.Account] = was
|
||||
p.filters[i].have, p.filters[i].exists = f.want, true
|
||||
said = append(said, "the filters of "+f.Account)
|
||||
}
|
||||
if len(said) == 0 {
|
||||
return "unchanged", nil
|
||||
}
|
||||
return "written: " + strings.Join(said, "; "), nil
|
||||
}
|
||||
|
||||
func sum(m map[string]int) int {
|
||||
n := 0
|
||||
for _, v := range m {
|
||||
n += v
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
// accountDirectory is an account's directory in the profile, as its server's preferences name it, or "" when
|
||||
// they name none within the profile.
|
||||
func (m *Machine) accountDirectory(profile string, prefs map[string]string, server string) string {
|
||||
if dir := accountDirectory(profile, prefs, server); dir != "" {
|
||||
return dir
|
||||
}
|
||||
abs := unquote(prefs["mail.server."+server+".directory"])
|
||||
if abs == "" {
|
||||
return ""
|
||||
}
|
||||
dir := m.path(abs)
|
||||
if strings.HasPrefix(filepath.Clean(dir), filepath.Clean(profile)+string(filepath.Separator)) {
|
||||
return dir
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func mustJSON(v any) string {
|
||||
b, _ := json.Marshal(v)
|
||||
return string(bytes.TrimSpace(b))
|
||||
}
|
||||
@@ -0,0 +1,228 @@
|
||||
package main
|
||||
|
||||
// The setting prefs, one entry per preference (novox/hq issue 586, ADR 0329; ADR 0316 §11 for the user.js).
|
||||
//
|
||||
// Until issue 586 the setting was one line of user_pref(…) statements, so a node's layer replaced it whole and
|
||||
// the operator's own choices were shipped in the catalogue as the module's defaults file. Now the setting is a
|
||||
// map from a preference's name to its value, and reaches the machine as ~/.config/betterbird/prefs.json, a
|
||||
// file the node-engine merges from the module's layers key by key: the mesh-wide layer holds the operator's
|
||||
// setup, and a node's layer overrides one preference, or takes one out with null. The module itself ships only
|
||||
// its policy's locks and Betterbird's own defaults (mesh-defaults.js names the preferences it manages there).
|
||||
//
|
||||
// The tools turn the setting into the head of the user.js they place in the profile, where Betterbird reads it
|
||||
// at its start and lets it win over what it saved itself. A preference that can hold a credential, or that
|
||||
// another setting owns (an account's, an identity's, a calendar's, an address book's, a tag's), is refused, so
|
||||
// each value has one owner (ADR 0316 §11, ADR 0329 §6).
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"math"
|
||||
"os"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// prefsFile is the setting prefs on the machine, written by the node-engine.
|
||||
const prefsFile = ".config/betterbird/prefs.json"
|
||||
|
||||
// configurationOwned are the preferences the settings of ADR 0329 own: the setting prefs refuses them (§6).
|
||||
var configurationOwned = []string{"calendar.registry.", "calendar.list.", "ldap_2.servers.", "mailnews.tags."}
|
||||
|
||||
// prefName is a preference's name as the setting may give it.
|
||||
var prefName = regexp.MustCompile(`^[A-Za-z0-9_][A-Za-z0-9_.\-\[\]@{}]{0,199}$`)
|
||||
|
||||
// prefOwner says why the setting prefs may not set a preference, or "" when it may.
|
||||
func prefOwner(name string) string {
|
||||
if passwordSecret.MatchString(name) {
|
||||
return "a preference that can hold a credential: not through the mesh"
|
||||
}
|
||||
for _, p := range accountOwned {
|
||||
if strings.HasPrefix(name, p) {
|
||||
return "an account's preference: an account is declared in the setting accounts, never in prefs (novox/hq ADR 0316 §11)"
|
||||
}
|
||||
}
|
||||
for _, p := range configurationOwned {
|
||||
if strings.HasPrefix(name, p) {
|
||||
return "owned by the setting calendars, address-books or tags, never by prefs (novox/hq ADR 0329 §6)"
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// readDeclaredPrefs reads the setting prefs on the machine: each preference's value as user.js writes it. A
|
||||
// missing file says the module is not pushed here in this form; a file that names a preference another setting
|
||||
// owns, or holds a value user.js cannot carry, is refused whole, naming the preference.
|
||||
func (m *Machine) readDeclaredPrefs() (map[string]string, bool, error) {
|
||||
raw, err := readBounded(m.home(prefsFile))
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
return map[string]string{}, false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
prefs, err := parseDeclaredPrefs(raw)
|
||||
return prefs, true, err
|
||||
}
|
||||
|
||||
func parseDeclaredPrefs(raw []byte) (map[string]string, error) {
|
||||
var top map[string]json.RawMessage
|
||||
if err := json.Unmarshal(raw, &top); err != nil {
|
||||
return nil, fmt.Errorf("~/%s is not a JSON object: %v", prefsFile, err)
|
||||
}
|
||||
out := map[string]string{}
|
||||
body, ok := top["prefs"]
|
||||
if !ok || string(body) == "null" {
|
||||
return out, nil
|
||||
}
|
||||
var values map[string]any
|
||||
dec := json.NewDecoder(bytes.NewReader(body))
|
||||
dec.UseNumber()
|
||||
if err := dec.Decode(&values); err != nil {
|
||||
return nil, fmt.Errorf("the setting prefs is a map from a preference's name to its value: %v", err)
|
||||
}
|
||||
var problems []string
|
||||
for _, name := range sortedKeys(values) {
|
||||
if !prefName.MatchString(name) {
|
||||
problems = append(problems, fmt.Sprintf("%q is not a preference's name", name))
|
||||
continue
|
||||
}
|
||||
if why := prefOwner(name); why != "" {
|
||||
problems = append(problems, name+": "+why)
|
||||
continue
|
||||
}
|
||||
lit, err := prefLiteral(values[name])
|
||||
if err != nil {
|
||||
problems = append(problems, name+": "+err.Error())
|
||||
continue
|
||||
}
|
||||
if lit != "" {
|
||||
out[name] = lit
|
||||
}
|
||||
}
|
||||
if len(problems) > 0 {
|
||||
return nil, fmt.Errorf("the setting prefs is refused: %s", strings.Join(problems, "; "))
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// prefLiteral is a setting's value as user.js writes it: true or false, a whole number, or a string. null is
|
||||
// no value: a node's layer takes the mesh-wide one out with it, and Betterbird's own default holds there.
|
||||
func prefLiteral(v any) (string, error) {
|
||||
switch x := v.(type) {
|
||||
case nil:
|
||||
return "", nil
|
||||
case bool:
|
||||
return strconv.FormatBool(x), nil
|
||||
case json.Number:
|
||||
n, err := x.Int64()
|
||||
if err != nil || n > math.MaxInt32 || n < math.MinInt32 {
|
||||
return "", fmt.Errorf("%s is not a whole number Betterbird keeps", x)
|
||||
}
|
||||
return strconv.FormatInt(n, 10), nil
|
||||
case float64:
|
||||
if x != math.Trunc(x) || x > math.MaxInt32 || x < math.MinInt32 {
|
||||
return "", fmt.Errorf("%v is not a whole number Betterbird keeps", x)
|
||||
}
|
||||
return strconv.FormatInt(int64(x), 10), nil
|
||||
case string:
|
||||
if strings.ContainsAny(x, lineEnds) {
|
||||
return "", errors.New("a string preference is one line")
|
||||
}
|
||||
return jsQuoted(x), nil
|
||||
}
|
||||
return "", fmt.Errorf("a value is true, false, a whole number or a string, not %T", v)
|
||||
}
|
||||
|
||||
// prefValue is a preference's literal as the setting holds it: the reverse of prefLiteral.
|
||||
func prefValue(lit string) (any, bool) {
|
||||
switch {
|
||||
case lit == "true":
|
||||
return true, true
|
||||
case lit == "false":
|
||||
return false, true
|
||||
}
|
||||
if n, err := strconv.ParseInt(lit, 10, 32); err == nil {
|
||||
return n, true
|
||||
}
|
||||
if strings.HasPrefix(lit, `"`) {
|
||||
if s, err := strconv.Unquote(lit); err == nil {
|
||||
return s, true
|
||||
}
|
||||
}
|
||||
return nil, false
|
||||
}
|
||||
|
||||
// lineEnds are the characters a reader may take as a line's end: a setting holds none (novox/hq issue 339).
|
||||
const lineEnds = "\n\r\v\f\x00\u0085
"
|
||||
|
||||
// jsQuoted is a string as a preference file writes it: quote, backslash and the control characters escaped the
|
||||
// way Gecko's preference parser reads them.
|
||||
func jsQuoted(s string) string {
|
||||
var b strings.Builder
|
||||
b.WriteByte('"')
|
||||
for _, r := range s {
|
||||
switch {
|
||||
case r == '"':
|
||||
b.WriteString(`\"`)
|
||||
case r == '\\':
|
||||
b.WriteString(`\\`)
|
||||
case r == '\n':
|
||||
b.WriteString(`\n`)
|
||||
case r == '\r':
|
||||
b.WriteString(`\r`)
|
||||
case r == '\t':
|
||||
b.WriteString(`\t`)
|
||||
case r < 0x20 || r == 0x7f || r == 0x2028 || r == 0x2029 || r == 0x85:
|
||||
fmt.Fprintf(&b, `\u%04x`, r)
|
||||
default:
|
||||
b.WriteRune(r)
|
||||
}
|
||||
}
|
||||
b.WriteByte('"')
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// declaredText is the head of the user.js the module places: its mark and the setting prefs, one statement a
|
||||
// line, by name. present says the setting is on this machine; err says why it is refused.
|
||||
func (m *Machine) declaredText() (string, bool, error) {
|
||||
prefs, present, err := m.readDeclaredPrefs()
|
||||
if err != nil || !present {
|
||||
return "", present, err
|
||||
}
|
||||
return composeDeclared(prefs), true, nil
|
||||
}
|
||||
|
||||
// composeDeclared is the user.js head for a set of declared preferences.
|
||||
func composeDeclared(prefs map[string]string) string {
|
||||
var b strings.Builder
|
||||
b.WriteString(mark + ", novox/hq issue 345). Owned by the mesh: the tools write it into the profile from the\n" +
|
||||
"// module's settings at Betterbird's start (betterbird_user_js with apply, betterbird_restart, the launcher), and\n" +
|
||||
"// Betterbird reads it at its start, where it wins over what Betterbird saved. Change the settings, never this file.\n" +
|
||||
"//\n// The setting prefs (novox/hq issue 586): the operator's setup, mesh-wide with a node's own layer over it.\n")
|
||||
names := make([]string, 0, len(prefs))
|
||||
for n := range prefs {
|
||||
names = append(names, n)
|
||||
}
|
||||
sort.Strings(names)
|
||||
for _, n := range names {
|
||||
fmt.Fprintf(&b, "user_pref(%s, %s);\n", quoted(n), prefs[n])
|
||||
}
|
||||
if len(names) == 0 {
|
||||
b.WriteString("// none\n")
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
func sortedKeys[V any](m map[string]V) []string {
|
||||
out := make([]string, 0, len(m))
|
||||
for k := range m {
|
||||
out = append(out, k)
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,178 @@
|
||||
package main
|
||||
|
||||
// The profile's identities, calendars, address books and tags, read from its preferences into the setting's
|
||||
// shape (novox/hq ADR 0329). The profile is read as Betterbird starts with it: prefs.js, with user.js over it.
|
||||
// Nothing here reads the login store, a calendar's events or an address book's contacts (§13).
|
||||
|
||||
import (
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// profileIdentity is one identity the profile holds.
|
||||
type profileIdentity struct {
|
||||
Key string
|
||||
AccountKey string
|
||||
// Default says it is its account's first identity.
|
||||
Default bool
|
||||
Owl bool
|
||||
Value Identity
|
||||
}
|
||||
|
||||
// profileIdentities reads every identity of every account in the profile's list, the default first.
|
||||
func (m *Machine) profileIdentities(prefs map[string]string) []profileIdentity {
|
||||
var out []profileIdentity
|
||||
for _, a := range profileAccounts(prefs) {
|
||||
for i, id := range a.Identities {
|
||||
out = append(out, profileIdentity{Key: id, AccountKey: a.Key, Default: i == 0, Owl: a.Owl,
|
||||
Value: m.readIdentity(prefs, id)})
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// readIdentity reads one identity's fields. A signature the profile reads from a file is read as its text, since
|
||||
// a file's path differs per node (§10).
|
||||
func (m *Machine) readIdentity(prefs map[string]string, key string) Identity {
|
||||
p := "mail.identity." + key + "."
|
||||
text := func(name string) string { return unquote(prefs[p+name]) }
|
||||
id := Identity{Address: text("useremail"), Name: text("fullName"), ReplyTo: text("reply_to"),
|
||||
Organization: text("organization"), SignatureHTML: prefs[p+"htmlSigFormat"] == "true",
|
||||
SignatureAboveQuote: prefs[p+"reply_on_top"] == "1" && prefs[p+"sig_bottom"] == "false"}
|
||||
sig := text("htmlSigText")
|
||||
if prefs[p+"attach_signature"] == "true" {
|
||||
if file := text("sig_file"); file != "" {
|
||||
sig = m.signatureFile(file)
|
||||
}
|
||||
}
|
||||
if sig = strings.TrimRight(strings.ReplaceAll(sig, "\r\n", "\n"), "\n"); sig != "" {
|
||||
id.Signature = strings.Split(sig, "\n")
|
||||
}
|
||||
return id
|
||||
}
|
||||
|
||||
// signatureFile is a signature file's text, read only within the home and bounded.
|
||||
func (m *Machine) signatureFile(file string) string {
|
||||
clean := filepath.Clean(file)
|
||||
if !strings.HasPrefix(clean, strings.TrimSuffix(m.Home, "/")+"/") {
|
||||
return ""
|
||||
}
|
||||
raw, err := readBounded(m.path(clean))
|
||||
if err != nil || len(raw) > 64<<10 {
|
||||
return ""
|
||||
}
|
||||
return string(raw)
|
||||
}
|
||||
|
||||
// keysUnder are the distinct second-level keys of the preferences under a prefix: calendar.registry.<key>.….
|
||||
func keysUnder(prefs map[string]string, prefix string) []string {
|
||||
seen := map[string]bool{}
|
||||
for name := range prefs {
|
||||
if rest, ok := strings.CutPrefix(name, prefix); ok {
|
||||
if key, _, found := strings.Cut(rest, "."); found && key != "" {
|
||||
seen[key] = true
|
||||
}
|
||||
}
|
||||
}
|
||||
return sortedKeys(seen)
|
||||
}
|
||||
|
||||
// profileCalendar is one calendar of the profile: Kind is Betterbird's (caldav, ics, storage, owl …).
|
||||
type profileCalendar struct {
|
||||
Key string
|
||||
Kind string
|
||||
// IdentityKey is the identity that answers its invitations.
|
||||
IdentityKey string
|
||||
Value Calendar
|
||||
}
|
||||
|
||||
func profileCalendars(prefs map[string]string) []profileCalendar {
|
||||
var out []profileCalendar
|
||||
for _, key := range keysUnder(prefs, "calendar.registry.") {
|
||||
p := "calendar.registry." + key + "."
|
||||
text := func(name string) string { return unquote(prefs[p+name]) }
|
||||
c := profileCalendar{Key: key, Kind: text("type"), IdentityKey: text("imip.identity.key"),
|
||||
Value: Calendar{Name: text("name"), URL: text("uri"), Color: strings.ToLower(text("color")),
|
||||
ReadOnly: prefs[p+"readOnly"] == "true", User: text("username")}}
|
||||
if prefs[p+"suppressAlarms"] == "true" {
|
||||
off := false
|
||||
c.Value.Reminders = &off
|
||||
}
|
||||
if c.Kind == "" && c.Value.URL == "" {
|
||||
continue
|
||||
}
|
||||
out = append(out, c)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// profileAddressBook is one address book of the profile: CardDAV is the kind a setting holds.
|
||||
type profileAddressBook struct {
|
||||
Key string
|
||||
CardDAV bool
|
||||
Kind string
|
||||
Value AddressBook
|
||||
}
|
||||
|
||||
// Betterbird's kinds of address book (dirType).
|
||||
var addressBookKinds = map[string]string{"0": "LDAP", "101": "local", "102": "CardDAV"}
|
||||
|
||||
func profileAddressBooks(prefs map[string]string) []profileAddressBook {
|
||||
var out []profileAddressBook
|
||||
for _, key := range keysUnder(prefs, "ldap_2.servers.") {
|
||||
p := "ldap_2.servers." + key + "."
|
||||
if _, has := prefs[p+"dirType"]; !has && prefs[p+"description"] == "" {
|
||||
continue
|
||||
}
|
||||
kind := addressBookKinds[prefs[p+"dirType"]]
|
||||
if kind == "" {
|
||||
kind = "other"
|
||||
}
|
||||
out = append(out, profileAddressBook{Key: key, CardDAV: kind == "CardDAV", Kind: kind,
|
||||
Value: AddressBook{Name: unquote(prefs[p+"description"]), URL: unquote(prefs[p+"carddav.url"]),
|
||||
User: unquote(prefs[p+"carddav.username"])}})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// profileTag is one tag of the profile.
|
||||
type profileTag struct {
|
||||
Key string
|
||||
ordinal string
|
||||
Value Tag
|
||||
}
|
||||
|
||||
// profileTags reads the tags, each with its place as Betterbird orders them.
|
||||
func profileTags(prefs map[string]string) []profileTag {
|
||||
var out []profileTag
|
||||
for _, key := range keysUnder(prefs, "mailnews.tags.") {
|
||||
p := "mailnews.tags." + key + "."
|
||||
if _, has := prefs[p+"tag"]; !has {
|
||||
continue
|
||||
}
|
||||
out = append(out, profileTag{Key: key, ordinal: unquote(prefs[p+"ordinal"]),
|
||||
Value: Tag{Name: unquote(prefs[p+"tag"]), Color: strings.ToUpper(unquote(prefs[p+"color"]))}})
|
||||
}
|
||||
// Betterbird orders the tags by their ordinal, a tag without one by its key in its place.
|
||||
at := func(t profileTag) string {
|
||||
if t.ordinal != "" {
|
||||
return t.ordinal
|
||||
}
|
||||
return t.Key
|
||||
}
|
||||
sort.SliceStable(out, func(i, j int) bool {
|
||||
if at(out[i]) != at(out[j]) {
|
||||
return at(out[i]) < at(out[j])
|
||||
}
|
||||
return out[i].Key < out[j].Key
|
||||
})
|
||||
for i := range out {
|
||||
out[i].Value.Order = i + 1
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// tagOrdinal is the ordinal a tag's place is written as, so Betterbird's order by text is the declared one.
|
||||
func tagOrdinal(order int) string { return strconv.Itoa(1000 + order) }
|
||||
@@ -10,6 +10,13 @@ package main
|
||||
// module declares. A theme or another preference of the operator's profile can be carried over by
|
||||
// name (carry), within the same limits as betterbird_prefs: never a credential, never an account.
|
||||
//
|
||||
// **It is visibly a test** (the operator, 2026-10-12): on 2026-10-12 at 12:05 a test Betterbird with an empty
|
||||
// profile carrying the operator's theme looked like the operator's configuration lost. So the test profile
|
||||
// carries a userChrome.css that marks its main window with a red band reading MESH TEST, and the event is
|
||||
// called "MESH TEST reminder". The reminder window itself is left as Betterbird draws it, because how it is
|
||||
// drawn is what the test looks at. The tool's description says a window opens on the operator's screen, so
|
||||
// a caller tells the operator first.
|
||||
//
|
||||
// It lives for at most ten minutes (the unit's RuntimeMaxSec), and stop ends it early and removes the
|
||||
// profile. The reminder window is then found with xorg_windows (class Calendar, role AlarmWindow) and
|
||||
// looked at with xorg_screenshot.
|
||||
@@ -41,6 +48,7 @@ type ReminderAnswer struct {
|
||||
Carried []string `json:"carried,omitempty"`
|
||||
Withheld []string `json:"withheld,omitempty"`
|
||||
Look string `json:"look"`
|
||||
OnScreen string `json:"on_screen,omitempty"`
|
||||
Stopped bool `json:"stopped,omitempty"`
|
||||
}
|
||||
|
||||
@@ -50,11 +58,38 @@ func ics(now time.Time, after time.Duration, stamp string) string {
|
||||
start := now.Add(after + 10*time.Minute)
|
||||
return strings.Join([]string{"BEGIN:VCALENDAR", "VERSION:2.0", "PRODID:-//novox mesh//betterbird reminder test//EN",
|
||||
"BEGIN:VEVENT", "UID:mesh-reminder-test-" + stamp + "@mesh", "DTSTAMP:" + f(now), "DTSTART:" + f(start),
|
||||
"DTEND:" + f(start.Add(30*time.Minute)), "SUMMARY:Mesh reminder test", "BEGIN:VALARM", "ACTION:DISPLAY",
|
||||
"DESCRIPTION:Mesh reminder test", "TRIGGER:-PT10M", "END:VALARM", "END:VEVENT", "END:VCALENDAR", ""}, "\r\n")
|
||||
"DTEND:" + f(start.Add(30*time.Minute)), "SUMMARY:" + testTitle, "BEGIN:VALARM", "ACTION:DISPLAY",
|
||||
"DESCRIPTION:" + testTitle, "TRIGGER:-PT10M", "END:VALARM", "END:VEVENT", "END:VCALENDAR", ""}, "\r\n")
|
||||
}
|
||||
|
||||
// testTitle is the test event's title: what the reminder window shows.
|
||||
const testTitle = "MESH TEST reminder (a throw-away Betterbird of the mesh, not your calendar)"
|
||||
|
||||
// testChrome is the test profile's userChrome.css: a red band reading MESH TEST across the top of every
|
||||
// Betterbird window but the reminder (windowtype Calendar:AlarmWindow), and a red frame around it, so the test
|
||||
// Betterbird is never taken for the operator's.
|
||||
const testChrome = `/* The mesh's reminder test (module betterbird, novox/hq issue 345): a throw-away Betterbird, not yours. */
|
||||
:root:not([windowtype="Calendar:AlarmWindow"]) {
|
||||
outline: 6px solid #d00000 !important;
|
||||
outline-offset: -6px !important;
|
||||
}
|
||||
:root:not([windowtype="Calendar:AlarmWindow"]) > body::before {
|
||||
content: "MESH TEST \2014 a throw-away Betterbird started by the mesh for a reminder test; your mail and calendar are not here. It ends by itself within 10 minutes.";
|
||||
display: block !important;
|
||||
background: #d00000 !important;
|
||||
color: #ffffff !important;
|
||||
font: bold 15px sans-serif !important;
|
||||
padding: 6px 12px !important;
|
||||
text-align: center !important;
|
||||
}
|
||||
`
|
||||
|
||||
// testPrefs are the test profile's own preferences: its one calendar, and no first-run questions.
|
||||
//
|
||||
// betterbird.update.checkIntervalDays 0 is Betterbird's own switch for its update check (MailGlue): a
|
||||
// fresh profile has never checked, so Betterbird checks at its start and, when betterbird.eu publishes a
|
||||
// newer build than the installed one, opens the About window on the operator's screen and gives it the
|
||||
// focus (seen on the workstation 2026-10-12, novox/hq issue 345). The test is about the reminder only.
|
||||
func testPrefs(calendar string) string {
|
||||
return strings.Join([]string{
|
||||
`user_pref("calendar.registry.meshtest.type", "ics");`,
|
||||
@@ -68,7 +103,9 @@ func testPrefs(calendar string) string {
|
||||
`user_pref("datareporting.policy.dataSubmissionEnabled", false);`,
|
||||
`user_pref("toolkit.telemetry.reportingpolicy.firstRun", false);`,
|
||||
`user_pref("app.update.auto", false);`,
|
||||
`user_pref("betterbird.update.checkIntervalDays", 0);`,
|
||||
`user_pref("mail.rights.version", 1);`,
|
||||
`user_pref("toolkit.legacyUserProfileCustomizations.stylesheets", true);`,
|
||||
"",
|
||||
}, "\n")
|
||||
}
|
||||
@@ -125,14 +162,22 @@ func (m *Machine) ReminderTest(after time.Duration, carry []string, stop bool) (
|
||||
if err := os.WriteFile(calendar, []byte(ics(now, after, strconv.FormatInt(now.Unix(), 10))), 0o600); err != nil {
|
||||
return a, err
|
||||
}
|
||||
user := testPrefs(strings.TrimPrefix(calendar, m.Root))
|
||||
if mine, err := readBounded(m.home(declared)); err == nil && validUserJS(string(mine)) == nil {
|
||||
user = string(mine) + "\n" + user
|
||||
// The module's user.js first, then what is carried, then the test's own preferences: the last
|
||||
// statement wins, so neither can take the test's calendar or its quiet start away.
|
||||
user := ""
|
||||
if mine, present, err := m.declaredText(); err == nil && present && validUserJS(mine) == nil {
|
||||
user = mine + "\n"
|
||||
}
|
||||
user += strings.Join(carried, "\n") + "\n"
|
||||
user += strings.Join(carried, "\n") + "\n" + testPrefs(strings.TrimPrefix(calendar, m.Root))
|
||||
if err := os.WriteFile(filepath.Join(dir, "profile", "user.js"), []byte(user), 0o600); err != nil {
|
||||
return a, err
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Join(dir, "profile", "chrome"), 0o700); err != nil {
|
||||
return a, err
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(dir, "profile", "chrome", "userChrome.css"), []byte(testChrome), 0o600); err != nil {
|
||||
return a, err
|
||||
}
|
||||
// The unit removes the profile when it ends, however it ends: at its 10 minutes, on stop, or when the
|
||||
// test Betterbird is closed.
|
||||
call := []string{"--user", "--collect", "--quiet", "--unit=" + testUnit, "-p", "RuntimeMaxSec=600",
|
||||
@@ -146,7 +191,9 @@ func (m *Machine) ReminderTest(after time.Duration, carry []string, stop bool) (
|
||||
if err := failed(m.cmd(8*time.Second, s.Env(), "systemd-run", call...), "systemd-run", call...); err != nil {
|
||||
return a, err
|
||||
}
|
||||
a.Event = "Mesh reminder test, in a calendar file of the test profile only"
|
||||
a.Event = testTitle + ", in a calendar file of the test profile only"
|
||||
a.OnScreen = "a second Betterbird's window is now on the operator's screen, marked MESH TEST in red; its reminder " +
|
||||
"window follows when the reminder is due. Both go when the test ends (stop, or 10 minutes)"
|
||||
a.Due = now.Add(after).UTC().Format(time.RFC3339)
|
||||
return a, nil
|
||||
}
|
||||
|
||||
@@ -0,0 +1,16 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"os"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// No test reaches the network or the mesh: Betterbird's list of published builds and the controller's
|
||||
// settings verb answer from here, and a test that needs them says what they answer.
|
||||
func TestMain(m *testing.M) {
|
||||
publishedBuilds = func() (string, error) { return "", errors.New("no network in tests") }
|
||||
asker = func(string, any) (json.RawMessage, error) { return nil, errors.New("no mesh in tests") }
|
||||
os.Exit(m.Run())
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
// Written by the mesh (module betterbird, novox/hq issue 345, issue 586): the preferences the module manages at
|
||||
// Betterbird's own default. It holds no value of the operator's: the operator's setup is the module's setting
|
||||
// prefs, mesh-wide with a node's own layer over it, which the tools place in the profile's user.js, where it wins
|
||||
// at every start; the policy's locks (distribution/policies.json) win over everything. betterbird_config shows the
|
||||
// layers and the drift. Change it in the catalogue, never here.
|
||||
//
|
||||
// A line "keeps Betterbird's default" names a preference the module manages without writing a value: Betterbird's
|
||||
// own default holds there, and a value the profile saved is drift, which betterbird_config_apply clears.
|
||||
|
||||
// keeps Betterbird's default: extensions.activeThemeID
|
||||
// The default theme (default-theme@mozilla.org) follows the desktop's dark mode. Betterbird keeps the active theme
|
||||
// in its add-on database, so another theme is changed back in the add-on manager, never by clearing prefs.js.
|
||||
// keeps Betterbird's default: browser.display.use_system_colors
|
||||
// Betterbird's own colours, which the default theme is drawn for.
|
||||
// keeps Betterbird's default: layout.css.always_underline_links
|
||||
// Links are underlined only where the message's own style says so.
|
||||
// keeps Betterbird's default: mail.spam.manualMark
|
||||
// Marking a message as junk by hand does nothing more than mark it.
|
||||
// keeps Betterbird's default: mail.compose.show_attachment_pane
|
||||
// The composer shows its attachment pane only when a message has attachments.
|
||||
@@ -0,0 +1,3 @@
|
||||
// Written by the mesh (module betterbird, novox/hq ADR 0316 §3): the system's spelling dictionaries, named
|
||||
// as a default preference in place of the link the community package made.
|
||||
pref("spellchecker.dictionary_path", "/usr/share/hunspell");
|
||||
@@ -0,0 +1,28 @@
|
||||
{
|
||||
"policies": {
|
||||
"DisableAppUpdate": true,
|
||||
"DisableTelemetry": true,
|
||||
"ExtensionUpdate": false,
|
||||
"ExtensionSettings": {
|
||||
"owl@beonex.com": {
|
||||
"installation_mode": "force_installed",
|
||||
"install_url": "file:///opt/betterbird/owl/owl.xpi",
|
||||
"updates_disabled": true
|
||||
}
|
||||
},
|
||||
"Preferences": {
|
||||
"mail.shell.checkDefaultClient": {
|
||||
"Value": false,
|
||||
"Status": "locked"
|
||||
},
|
||||
"mail.inappnotifications.enabled": {
|
||||
"Value": false,
|
||||
"Status": "locked"
|
||||
},
|
||||
"mailnews.start_page.enabled": {
|
||||
"Value": false,
|
||||
"Status": "locked"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
+152
-10
@@ -1,23 +1,36 @@
|
||||
{
|
||||
"module": "betterbird",
|
||||
"version": "1",
|
||||
"upgrade": {
|
||||
"policy": "record",
|
||||
"why": "a new build may remove the community package betterbird-bin and replaces the installation in /opt/betterbird, which a running Betterbird has open: the switch on each node and every upgrade wait for the operator, who quits Betterbird first and then pushes (novox/hq ADR 0316 §4, §5)"
|
||||
},
|
||||
"requires": [
|
||||
"x11-display"
|
||||
],
|
||||
"settings": {
|
||||
"prefs": {
|
||||
"kind": "preference",
|
||||
"default": "// none",
|
||||
"why": "Betterbird runs with its own defaults until a machine's assignment names a preference: one line of user_pref(\"name\", value); statements, which the module's user.js holds and Betterbird reads at its start (novox/hq issue 345)"
|
||||
}
|
||||
},
|
||||
"invokes": [
|
||||
"seat:mesh-controller.settings",
|
||||
"seat:mesh-controller.nodes",
|
||||
"seat:mesh-controller.node",
|
||||
"betterbird.betterbird_derive"
|
||||
],
|
||||
"tools": [
|
||||
"betterbird_status",
|
||||
"betterbird_prefs",
|
||||
"betterbird_config",
|
||||
"betterbird_config_apply",
|
||||
"betterbird_user_js",
|
||||
"betterbird_log",
|
||||
"betterbird_restart",
|
||||
"betterbird_check",
|
||||
"betterbird_accounts",
|
||||
"betterbird_identities",
|
||||
"betterbird_calendars",
|
||||
"betterbird_address_books",
|
||||
"betterbird_filters",
|
||||
"betterbird_tags",
|
||||
"betterbird_derive",
|
||||
"betterbird_drift",
|
||||
"betterbird_reminder_test"
|
||||
],
|
||||
"resources": [
|
||||
@@ -29,12 +42,109 @@
|
||||
"mode": "0755"
|
||||
},
|
||||
{
|
||||
"id": "user-js",
|
||||
"id": "prefs",
|
||||
"type": "file",
|
||||
"path": "${machine:account-home}/.config/betterbird/user.js",
|
||||
"path": "${machine:account-home}/.config/betterbird/prefs.json",
|
||||
"owner": "${machine:account}",
|
||||
"mode": "0644",
|
||||
"content": "// Betterbird's user.js (module betterbird, novox/hq issue 345). Owned by the mesh: the node-engine\n// writes it here at every push, and the module's tools copy it whole into Betterbird's profile\n// (betterbird_user_js with apply, betterbird_restart), where Betterbird reads it at its start and lets it\n// win over the preferences Betterbird saves itself. Change the module's setting prefs, never this file or the profile's copy.\n//\n// Only comments and user_pref(\"name\", value); statements; the tools refuse anything else, and any\n// preference whose name can hold a credential.\n${setting:prefs}\n"
|
||||
"merge": "json",
|
||||
"takes-settings": [
|
||||
"prefs"
|
||||
],
|
||||
"content": "{\n \"prefs\": {}\n}\n",
|
||||
"trusted": true
|
||||
},
|
||||
{
|
||||
"id": "accounts",
|
||||
"type": "file",
|
||||
"path": "${machine:account-home}/.config/betterbird/accounts.json",
|
||||
"owner": "${machine:account}",
|
||||
"mode": "0600",
|
||||
"merge": "json",
|
||||
"takes-settings": [
|
||||
"accounts"
|
||||
],
|
||||
"content": "{\n \"accounts\": {}\n}\n",
|
||||
"trusted": true
|
||||
},
|
||||
{
|
||||
"id": "configuration",
|
||||
"type": "file",
|
||||
"path": "${machine:account-home}/.config/betterbird/configuration.json",
|
||||
"owner": "${machine:account}",
|
||||
"mode": "0600",
|
||||
"merge": "json",
|
||||
"takes-settings": [
|
||||
"identities",
|
||||
"calendars",
|
||||
"address-books",
|
||||
"filters"
|
||||
],
|
||||
"content": "{\n \"identities\": {},\n \"calendars\": {},\n \"address-books\": {},\n \"filters\": {}\n}\n",
|
||||
"trusted": true
|
||||
},
|
||||
{
|
||||
"id": "tags",
|
||||
"type": "file",
|
||||
"path": "${machine:account-home}/.config/betterbird/tags.json",
|
||||
"owner": "${machine:account}",
|
||||
"mode": "0644",
|
||||
"merge": "json",
|
||||
"takes-settings": [
|
||||
"tags"
|
||||
],
|
||||
"content": "{\n \"tags\": {}\n}\n",
|
||||
"trusted": false
|
||||
},
|
||||
{
|
||||
"id": "community-package",
|
||||
"type": "package",
|
||||
"package": "betterbird-bin",
|
||||
"absent": true
|
||||
},
|
||||
{
|
||||
"id": "installation",
|
||||
"type": "archive",
|
||||
"path": "/opt/betterbird",
|
||||
"artifact": "archive"
|
||||
},
|
||||
{
|
||||
"id": "owl",
|
||||
"type": "archive",
|
||||
"path": "/opt/betterbird/owl",
|
||||
"artifact": "owl"
|
||||
},
|
||||
{
|
||||
"id": "policies",
|
||||
"type": "archive",
|
||||
"path": "/opt/betterbird/distribution",
|
||||
"artifact": "distribution"
|
||||
},
|
||||
{
|
||||
"id": "spelling",
|
||||
"type": "archive",
|
||||
"path": "/opt/betterbird/defaults/pref",
|
||||
"artifact": "defaults"
|
||||
},
|
||||
{
|
||||
"id": "preparation",
|
||||
"type": "archive",
|
||||
"path": "/usr/local/lib/mesh-betterbird",
|
||||
"artifact": "tools"
|
||||
},
|
||||
{
|
||||
"id": "start",
|
||||
"type": "file",
|
||||
"path": "/usr/local/bin/betterbird",
|
||||
"mode": "0755",
|
||||
"content": "#!/bin/sh\n# Betterbird's start, written by the mesh (module betterbird, novox/hq ADR 0316). Replaced at every push.\n#\n# Before Betterbird starts, the module's tools place its user.js in the profile (the settings prefs, accounts, identities, calendars,\n# address-books and tags) and write what Betterbird rewrites itself: the account lists and each declared account's filters. They do\n# it only while no Betterbird of this account runs, because Betterbird rewrites its saved preferences when it quits; a\n# second start while one runs only hands over to it. What the preparation could not do it says here, and\n# betterbird_check says it too. Betterbird starts either way: its configuration never holds the mail back.\n/usr/local/lib/mesh-betterbird/betterbird-tools prepare || echo \"betterbird: the mesh's preparation did not complete; betterbird_check says why\" >&2\nexec /opt/betterbird/betterbird \"$@\"\n"
|
||||
},
|
||||
{
|
||||
"id": "launcher",
|
||||
"type": "file",
|
||||
"path": "/usr/local/share/applications/eu.betterbird.Betterbird.desktop",
|
||||
"mode": "0644",
|
||||
"content": "[Desktop Entry]\nVersion=1.0\nType=Application\nName=Betterbird\nGenericName=Mail Client\nComment=Send and receive mail with Betterbird\nExec=/usr/local/bin/betterbird %u\nIcon=/opt/betterbird/chrome/icons/default/default128.png\nTerminal=false\nCategories=Network;Email;\nMimeType=message/rfc822;x-scheme-handler/mailto;application/x-xpinstall;application/x-extension-ics;text/calendar;text/vcard;text/x-vcard;x-scheme-handler/webcal;x-scheme-handler/webcals;x-scheme-handler/mid;\nStartupNotify=true\nStartupWMClass=eu.betterbird.Betterbird\nKeywords=Email;E-mail;Calendar;Thunderbird;\nActions=ComposeMessage;OpenAddressBook;\n# Written by the mesh (module betterbird, novox/hq ADR 0316 §3), in place of the community package's entry:\n# it starts Betterbird through the module's start, and names the icon in the installation, with no link.\n\n[Desktop Action ComposeMessage]\nName=Write new message\nExec=/usr/local/bin/betterbird -compose\n\n[Desktop Action OpenAddressBook]\nName=Open address book\nExec=/usr/local/bin/betterbird -addressbook\n"
|
||||
}
|
||||
],
|
||||
"build": {
|
||||
@@ -49,6 +159,38 @@
|
||||
"loads": [
|
||||
"betterbird-tools"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "archive",
|
||||
"kind": "archive",
|
||||
"url": "https://www.betterbird.eu/downloads/LinuxArchive/betterbird-153.4.0esr-bb10.en-US.linux-x86_64.tar.xz",
|
||||
"sha256": "38dd24890a66b4cf9a45bc24e15192e00c26b929e1bb8df7974c16a4539129e2",
|
||||
"version": "153.4.0esr-bb10",
|
||||
"within": "betterbird",
|
||||
"names-on-purpose": {
|
||||
"www.betterbird.eu": "Betterbird's project publishes its Linux release there; the build seat fetches it once, by its pinned sha256 (novox/hq ADR 0316 §2)"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "owl",
|
||||
"kind": "archive",
|
||||
"url": "https://www.beonex.com/owl/owl.xpi",
|
||||
"sha256": "a54cc3d52113b34f97942ee19190947dee23561e2217f18db3d80ac023991221",
|
||||
"version": "1.5.6.1",
|
||||
"file": "owl.xpi",
|
||||
"names-on-purpose": {
|
||||
"www.beonex.com": "Owl's vendor publishes its xpi there; the build seat fetches it once, by its pinned sha256 (novox/hq ADR 0316 §6)"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "distribution",
|
||||
"kind": "archive",
|
||||
"from": "install/distribution"
|
||||
},
|
||||
{
|
||||
"name": "defaults",
|
||||
"kind": "archive",
|
||||
"from": "install/defaults-pref"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
@@ -7,9 +7,23 @@ machine that is off builds nothing and blocks nothing. The controller asks the r
|
||||
the outcome names the machine that built it.
|
||||
|
||||
What a holding machine needs is what the builder always needed, said here once: a container runtime
|
||||
(the socket is mounted), the artifact store and the package registry as provisions, a workspace, and
|
||||
the bus credential. The code is `cmd/mesh-builder` in the mesh-controller repository, compiled from
|
||||
(the socket is mounted), the artifact store as a provision, a workspace, and the bus credential. The code is `cmd/mesh-builder` in the mesh-controller repository, compiled from
|
||||
that repository's main (`build.artifacts[].context`); this module ships the packaging.
|
||||
|
||||
Assign it to every machine with a container runtime. It replaces `builder`, the one-holder form of the
|
||||
same thing; retire that once this is assigned where it was.
|
||||
|
||||
## The package registry (novox/hq ADR 0336)
|
||||
|
||||
The package registry (`npm-package-registry`, provided by the gitea module) is an optional requirement.
|
||||
Genesis assigns the build agent before anything provides it, so the agent is assignable on a mesh with no
|
||||
registry, and `mesh-controller.plan` and `node` list the requirement as unbound until the gitea module is
|
||||
assigned; it then binds with no further act.
|
||||
|
||||
Until it binds, the agent on genesis's node reads the `genesis-package-registry` own-secret, which the
|
||||
installer seals to it at the packages step: the forge it raised, the port, the npm path, the account
|
||||
`mesh-builder` and its password. Issued outside the mesh, so no other machine is given one. A real binding
|
||||
wins over it, and each build's log says which the build resolved from. A build that needs the registry (a
|
||||
`package` artifact, an image whose Dockerfile names `.npmrc`, a TypeScript artifact with an `@novox/`
|
||||
dependency) on an agent with neither fails before building, naming the machine. Go artifacts and images
|
||||
that do not name `.npmrc` build without it. `manifest_test.go` holds the manifest to this.
|
||||
|
||||
@@ -0,0 +1,3 @@
|
||||
module build-agent
|
||||
|
||||
go 1.22
|
||||
@@ -0,0 +1,117 @@
|
||||
// The build agent's manifest (novox/hq ADR 0336, issue 121). The build agent is assigned at genesis's
|
||||
// step 13, before anything provides the package registry, so that requirement is optional
|
||||
// (ADR 0330); until it binds, the agent on genesis's node reads the genesis document the installer
|
||||
// seals to it at the packages step. The program is mesh-controller's cmd/mesh-builder; only the
|
||||
// packaging lives here, and this is its test.
|
||||
package buildagent
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
type manifest struct {
|
||||
Requires []json.RawMessage `json:"requires"`
|
||||
Binds map[string]string `json:"binds"`
|
||||
Secrets map[string]string `json:"secrets"`
|
||||
OwnSecrets map[string]json.RawMessage `json:"own-secrets"`
|
||||
Resources []map[string]any `json:"resources"`
|
||||
}
|
||||
|
||||
func read(t *testing.T) manifest {
|
||||
t.Helper()
|
||||
raw, err := os.ReadFile("module.json")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var m manifest
|
||||
if err := json.Unmarshal(raw, &m); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return m
|
||||
}
|
||||
|
||||
// requirement is one entry of `requires`: its name, and whether it is optional.
|
||||
func requirements(t *testing.T, m manifest) map[string]bool {
|
||||
t.Helper()
|
||||
out := map[string]bool{}
|
||||
for _, raw := range m.Requires {
|
||||
var name string
|
||||
if json.Unmarshal(raw, &name) == nil {
|
||||
out[name] = false
|
||||
continue
|
||||
}
|
||||
var long struct {
|
||||
Name string `json:"name"`
|
||||
Optional bool `json:"optional"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &long); err != nil {
|
||||
t.Fatalf("requires: %s: %v", raw, err)
|
||||
}
|
||||
out[long.Name] = long.Optional
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func TestThePackageRegistryIsOptionalAndTheArtifactStoreStaysHard(t *testing.T) {
|
||||
r := requirements(t, read(t))
|
||||
optional, has := r["npm-package-registry"]
|
||||
if !has || !optional {
|
||||
t.Errorf("npm-package-registry is not an optional requirement (ADR 0336 §1): %v", r)
|
||||
}
|
||||
if optional, has := r["artifact-store"]; !has || optional {
|
||||
t.Errorf("artifact-store is not a hard requirement: %v", r)
|
||||
}
|
||||
}
|
||||
|
||||
// The binding and the credential of the registry are delivered only while it is bound: ADR 0330 §6
|
||||
// composes neither for an absent requirement, so no file of the module may name them through a
|
||||
// placeholder unless it is itself only-when-bound.
|
||||
func TestTheRegistrysBindsAndSecretsAreOnlyWhenBound(t *testing.T) {
|
||||
m := read(t)
|
||||
if m.Binds["npm-package-registry"] == "" || m.Secrets["npm-package-registry"] == "" {
|
||||
t.Fatalf("the agent is not told where the registry is once it binds: binds %v, secrets %v", m.Binds, m.Secrets)
|
||||
}
|
||||
for _, r := range m.Resources {
|
||||
content, _ := r["content"].(string)
|
||||
if strings.Contains(content, "${bound:npm-package-registry:") || strings.Contains(content, "${secret:npm-package-registry}") {
|
||||
if r["only-when-bound"] != "npm-package-registry" {
|
||||
t.Errorf("%v names the registry's binding and is delivered while it is absent", r["id"])
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The genesis document is an own-secret issued outside the mesh — by the installer at genesis's packages
|
||||
// step (ADR 0336 §3) — so the mesh never makes one of its own on any other machine, and the agent is
|
||||
// told where it lands.
|
||||
func TestTheGenesisDocumentIsDeclaredAndIssuedOutside(t *testing.T) {
|
||||
m := read(t)
|
||||
raw, has := m.OwnSecrets["genesis-package-registry"]
|
||||
if !has {
|
||||
t.Fatalf("the build agent declares no genesis-package-registry own-secret: %v", m.OwnSecrets)
|
||||
}
|
||||
var own struct {
|
||||
Path string `json:"path"`
|
||||
IssuedBy string `json:"issued-by"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &own); err != nil || own.IssuedBy != "outside" {
|
||||
t.Fatalf("the genesis document is not issued outside the mesh: %s (%v)", raw, err)
|
||||
}
|
||||
if !strings.HasPrefix(own.Path, "${dir:mesh-state}/") {
|
||||
t.Fatalf("the genesis document is not where the container reads /run/mesh: %s", own.Path)
|
||||
}
|
||||
inside := "/run/mesh/" + strings.TrimPrefix(own.Path, "${dir:mesh-state}/")
|
||||
for _, r := range m.Resources {
|
||||
if r["id"] == "agent-env" {
|
||||
content, _ := r["content"].(string)
|
||||
if !strings.Contains(content, "MESH_GENESIS_PACKAGE_REGISTRY="+inside+"\n") {
|
||||
t.Errorf("the agent is not told where the genesis document is (%s):\n%s", inside, content)
|
||||
}
|
||||
return
|
||||
}
|
||||
}
|
||||
t.Fatal("the build agent has no environment file")
|
||||
}
|
||||
@@ -14,7 +14,10 @@
|
||||
],
|
||||
"requires": [
|
||||
"artifact-store",
|
||||
"npm-package-registry"
|
||||
{
|
||||
"name": "npm-package-registry",
|
||||
"optional": true
|
||||
}
|
||||
],
|
||||
"binds": {
|
||||
"npm-package-registry": "${dir:mesh-state}/package-registry.json"
|
||||
@@ -23,7 +26,11 @@
|
||||
"npm-package-registry": "${dir:mesh-state}/package-registry.secret"
|
||||
},
|
||||
"own-secrets": {
|
||||
"broker": "${dir:mesh-state}/broker"
|
||||
"broker": "${dir:mesh-state}/broker",
|
||||
"genesis-package-registry": {
|
||||
"path": "${dir:mesh-state}/genesis-package-registry.json",
|
||||
"issued-by": "outside"
|
||||
}
|
||||
},
|
||||
"data": {
|
||||
"own": [
|
||||
@@ -52,7 +59,7 @@
|
||||
"type": "file",
|
||||
"path": "${dir:mesh-state}/build-agent.env",
|
||||
"mode": "0600",
|
||||
"content": "MESH_BROKER_FILE=/run/mesh/broker\nMESH_NODE=${machine:name}\nMESH_REGISTRY=${bound:artifact-store:at}:${bound:artifact-store:port}\nMESH_PACKAGE_BINDING=/run/mesh/package-registry.json\nMESH_NPM_TOKEN_FILE=/run/mesh/package-registry.secret\nMESH_WORKSPACE=${dir:workspace}\n"
|
||||
"content": "MESH_BROKER_FILE=/run/mesh/broker\nMESH_NODE=${machine:name}\nMESH_REGISTRY=${bound:artifact-store:at}:${bound:artifact-store:port}\nMESH_PACKAGE_BINDING=/run/mesh/package-registry.json\nMESH_NPM_TOKEN_FILE=/run/mesh/package-registry.secret\nMESH_GENESIS_PACKAGE_REGISTRY=/run/mesh/genesis-package-registry.json\nMESH_WORKSPACE=${dir:workspace}\n"
|
||||
},
|
||||
{
|
||||
"id": "server",
|
||||
@@ -70,7 +77,15 @@
|
||||
"restart-on": [
|
||||
"agent-env"
|
||||
],
|
||||
"network": "host"
|
||||
"network": "host",
|
||||
"health": {
|
||||
"kind": "exec",
|
||||
"command": "/usr/local/bin/mesh-builder ready",
|
||||
"interval": "10s",
|
||||
"timeout": "5s",
|
||||
"looks": 3,
|
||||
"grace": "60s"
|
||||
}
|
||||
}
|
||||
],
|
||||
"build": {
|
||||
|
||||
@@ -68,6 +68,7 @@ must see, a node that joins later included — kept, so it carries no secret eit
|
||||
| what this node should hold | the licence manager's `bindings` state, this node's key; on a newer generation this module asks `anthropic-licence-manager.current` for its token, sealed to the key it sends, and writes it access-token-only — so the agent here never refreshes. A node that was off reads its key when it is back |
|
||||
| an MCP server registered through this module before registering was refused (hq issue 340) | a key in the module's `servers` state — `all.<server>` for every node, `<node>.<server>` for one; every node watches it and renders what applies to it, a node's own entry over the one for every node. A node that joins later, or was off, reads the whole current set at start; unregistering is a delete. Nothing adds a key any more: a new tool server is the operator's, in the `mcp_servers` setting at the controller's terminal |
|
||||
| a proposed instruction section (hq ADR 0274) | a key in the module's `proposals` state per proposal, the ask published on `operator-channel` under this module's name, and the router's record of it read under that name alone; what a node approved is its own file, never the bus |
|
||||
| a session's question to the operator (hq ADR 0332) | a key `q-…` in the module's `questions` state per question, the ask published on `operator-channel` under this module's name, and the router's record of it read under that name alone; the router's word that it ended without a choice is kept on the question; two slot keys `slot-1`, `slot-2` name the questions open |
|
||||
| the agent's configuration (hq ADR 0216) | a key in the module's `config` state per registration — `mesh.<kind>.<name>` for every node, `node.<node>.<kind>.<name>` for one, `home.<node>.<kind>.<name>` for one account's own directory — the item and its files in one value, at most 256 KiB. Every node watches it and renders what applies to it, a node item over a mesh item of the same kind and name. A hook, settings, or an item that would run something by itself is never taken, whenever it was put there (hq issue 340) |
|
||||
|
||||
## The mesh's tools first (hq ADR 0245)
|
||||
@@ -160,6 +161,28 @@ a secret, more than 16 KiB, a name the terminal sets, and — at once, naming th
|
||||
that proves its sender is there. The controller's hand-act log records each warrant once (`warranted`), read from the
|
||||
router's record. The text is never logged.
|
||||
|
||||
**A session asks the operator a question on Telegram** (hq ADR 0332, issue 589). An agent that needs a decision while
|
||||
the operator may be away from the terminal calls `claude_code_ask` with a headline, the context in plain words, two to
|
||||
four options (a label and what choosing it means, exactly one recommended, with why), what it does if nobody
|
||||
answers, its name as who@node (required), and an expiry of 5 to 1430 minutes (60 when absent). The ask is published under this
|
||||
module's name on `operator-channel`; every option is at the level approve and binds the digest of the question and of
|
||||
itself, so only a channel that proves who answered (Telegram) carries it, never the desk. Anything shaped like a
|
||||
secret, or a character a phone does not show, in any of its words is refused before anything is kept. The tool waits
|
||||
a few seconds for the router's word and answers within seconds with the question's key, or the router's refusal (the content rule: no path,
|
||||
address, host name, commit id or secret's shape; `messenger_check` tests words first). `claude_code_answer` with the
|
||||
key says waiting, chosen (the option, who answered, on which channel, when), ended without a choice (expired,
|
||||
refused, cancelled, replaced), or not proven: the answer is read from the router's own record and held to the
|
||||
question as `questions` keeps it, so a question rewritten after it was asked, a warrant for another ask or a choice
|
||||
after expiry is never read as a choice. One call waits at most 60 s; the tool asks a session to read a question at
|
||||
most once a minute, and nothing enforces it. At most two questions wait at once, mesh-wide: each holds one of two
|
||||
slot keys in `questions`, taken by create or compare-and-set, so two nodes asking at once cannot both take the last,
|
||||
and a slot that cannot be read refuses the question. Questions alone never fill the router's three open asks, so one
|
||||
is left for a proposed instruction section while two are open. A question the router holds no record of, its refusal
|
||||
not heard here, keeps its slot and reads as waiting until it expires: the router may still take it from its queue,
|
||||
and no cancel can overtake it there (the queue offers work again after an error, and a cancel of an ask not yet
|
||||
recorded is dropped); once expired, the router refuses it if taken late. A refused question keeps none of
|
||||
its words. An answer is the operator's word to the session and authorises nothing; a question never defaults.
|
||||
|
||||
The home scope is registered through the tools **only where the home belongs to an agent account of its own**
|
||||
(hq ADR 0266, issue 340), and only for the node whose module serves the call. Where the agent runs as the operator,
|
||||
the home is the operator's own `~/.claude`, whose rules and skills load in every session the operator starts, and
|
||||
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
@@ -20,6 +21,19 @@ type Proposing struct {
|
||||
Record Recorder
|
||||
Now func() time.Time
|
||||
Write WriteManaged
|
||||
// Questions is the `questions` state (novox/hq ADR 0332); nil is the bus's, made on first use.
|
||||
Questions QuestionStore
|
||||
questionsMu sync.Mutex
|
||||
}
|
||||
|
||||
// questions is the `questions` state.
|
||||
func (d *Proposing) questions() QuestionStore {
|
||||
d.questionsMu.Lock()
|
||||
defer d.questionsMu.Unlock()
|
||||
if d.Questions == nil {
|
||||
d.Questions = keptQuestions{stdio.State(QuestionsState)}
|
||||
}
|
||||
return d.Questions
|
||||
}
|
||||
|
||||
var proposing *Proposing
|
||||
@@ -60,6 +74,10 @@ func (d *Proposing) watch(p Paths) {
|
||||
return stdio.Subscribe(asks.Seat+".decided."+Module, func(e stdio.Envelope) error {
|
||||
var w asks.Warrant
|
||||
if json.Unmarshal(e.Body, &w) == nil {
|
||||
if strings.HasPrefix(w.Ask, QuestionPrefix) {
|
||||
HeardQuestionEnd(d.questions(), w)
|
||||
return nil
|
||||
}
|
||||
if line, err := HeardEnd(p, d.Store, w, d.Now()); err != nil {
|
||||
say("taking the router's word on %s: %v", w.Ask, err)
|
||||
} else if line != "" {
|
||||
@@ -86,7 +104,7 @@ func (d *Proposing) watch(p Paths) {
|
||||
|
||||
func proposalTools(p Paths) []stdio.Tool {
|
||||
notLaunched := errors.New("this module was not launched with the bus; nothing can be proposed")
|
||||
return []stdio.Tool{
|
||||
return append([]stdio.Tool{
|
||||
{Name: "claude_code_instructions_propose",
|
||||
Description: "Propose an instruction section for every Claude session on the mesh (scope mesh) or on some nodes " +
|
||||
"(scope node): a rule, a hand-over, the context a later session needs — or, with remove, the removal of one " +
|
||||
@@ -127,5 +145,83 @@ func proposalTools(p Paths) []stdio.Tool {
|
||||
proposing.decideNow(p)
|
||||
return ListProposals(p, proposing.Store, proposing.Bus, strArg(a, "id"), proposing.Now())
|
||||
}},
|
||||
}, questionTools(p, notLaunched)...)
|
||||
}
|
||||
|
||||
// questionTools are a session's question to the operator and its answer (novox/hq ADR 0332).
|
||||
func questionTools(p Paths, notLaunched error) []stdio.Tool {
|
||||
return []stdio.Tool{
|
||||
{Name: "claude_code_ask",
|
||||
Description: "Ask the operator a question on Telegram when you need a decision and the operator may not be at the " +
|
||||
"terminal (novox/hq ADR 0332): a headline, the context in plain words, two to four options with exactly one " +
|
||||
"marked recommended and why, what you do if nobody answers, who you are (by, required), and when it expires (5 to " +
|
||||
"1430 minutes, 60 when absent). Answers within seconds, once the router has taken it, with the question's " +
|
||||
"key, or with the router's refusal; read the answer with claude_code_answer and keep working " +
|
||||
"meanwhile. Only a channel that proves who answered carries it, never the desk. The words leave the mesh, so the " +
|
||||
"router refuses a path, an address, a host name, a commit id or anything shaped like a secret: say it in " +
|
||||
"words (messenger_check tests words first); anything shaped like a secret is refused before it is kept. At most two " +
|
||||
"questions wait at once, mesh-wide. The answer is the " +
|
||||
"operator's word to you; it authorises no call that needs the operator's approval.",
|
||||
Input: map[string]any{
|
||||
"headline": str("what is decided, in a few plain words, at most 60 characters, e.g. 'Merge the resolver fix tonight?'"),
|
||||
"context": str("what the operator decides and why it matters, in plain words, at most 1500 bytes"),
|
||||
"options": map[string]any{"type": "array", "description": "two to four answers; exactly one recommended",
|
||||
"items": map[string]any{"type": "object", "properties": map[string]any{
|
||||
"label": str("the button's words, at most 24 characters"),
|
||||
"means": str("what you do when it is chosen, in plain words"),
|
||||
"recommended": map[string]any{"type": "boolean", "description": "true on the one you recommend"},
|
||||
}, "required": []string{"label", "means"}}},
|
||||
"why": str("why the recommended option is the one you recommend"),
|
||||
"on_expiry": str("what you do if nobody answers before it expires, e.g. 'I leave the pull request open'"),
|
||||
"by": str("required: your session's name, as who@node or who, e.g. fix-589@g14"),
|
||||
"expires_minutes": map[string]any{"type": "integer",
|
||||
"description": "minutes until it expires, 5 to 1430; 60 when absent"},
|
||||
},
|
||||
Run: func(a map[string]any) (any, error) {
|
||||
if proposing == nil {
|
||||
return nil, notLaunched
|
||||
}
|
||||
in := QuestionInput{Headline: strArg(a, "headline"), Context: strArg(a, "context"), Why: strArg(a, "why"),
|
||||
OnExpiry: strArg(a, "on_expiry"), By: strArg(a, "by"), Options: questionOptionsOf(a["options"])}
|
||||
if m, ok := a["expires_minutes"].(float64); ok {
|
||||
in.Expires = time.Duration(m * float64(time.Minute))
|
||||
}
|
||||
return AskQuestion(p, in, proposing.questions(), proposing.Bus, proposing.Now())
|
||||
}},
|
||||
{Name: "claude_code_answer",
|
||||
Description: "Where a question asked with claude_code_ask stands, by its key (novox/hq ADR 0332): waiting; chosen, " +
|
||||
"with the option, who answered, on which channel and when; ended without a choice (expired, refused, " +
|
||||
"cancelled, replaced) with the router's words; or not proven, when the router's record does not hold to the " +
|
||||
"question asked. Read from the router's own record. wait_seconds (at most 60) waits for an answer within the " +
|
||||
"call; read a question at most once a minute (asked of you, not enforced).",
|
||||
Input: map[string]any{
|
||||
"key": str("the key claude_code_ask gave, e.g. q-0123456789abcdef"),
|
||||
"wait_seconds": map[string]any{"type": "integer", "description": "wait up to this many seconds (at most 60) while it is waiting"},
|
||||
},
|
||||
Run: func(a map[string]any) (any, error) {
|
||||
if proposing == nil {
|
||||
return nil, notLaunched
|
||||
}
|
||||
wait := time.Duration(0)
|
||||
if s, ok := a["wait_seconds"].(float64); ok && s > 0 {
|
||||
wait = time.Duration(s * float64(time.Second))
|
||||
}
|
||||
return WaitForAnswer(proposing.questions(), proposing.Bus, strArg(a, "key"), wait, proposing.Now)
|
||||
}},
|
||||
}
|
||||
}
|
||||
|
||||
// questionOptionsOf reads the `options` argument: a list of objects with label, means and recommended.
|
||||
func questionOptionsOf(v any) []QuestionOption {
|
||||
list, _ := v.([]any)
|
||||
var out []QuestionOption
|
||||
for _, item := range list {
|
||||
m, _ := item.(map[string]any)
|
||||
if m == nil {
|
||||
out = append(out, QuestionOption{})
|
||||
continue
|
||||
}
|
||||
out = append(out, QuestionOption{Label: strArg(m, "label"), Means: strArg(m, "means"), Recommended: boolArg(m, "recommended")})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
@@ -838,14 +838,20 @@ func HeardEnd(p Paths, store ProposalStore, w asks.Warrant, now time.Time) (stri
|
||||
if !known || (w.About != "" && w.About != prop.Ask().About) {
|
||||
return "", nil
|
||||
}
|
||||
endedMu.Lock()
|
||||
ended[w.Ask] = w
|
||||
endedMu.Unlock()
|
||||
// Said to a proposer waiting on it only once the discard is kept: what it reads next is already there (the
|
||||
// flake of the router's refusal test under a loaded build agent, 2026-10-12).
|
||||
tell := func() {
|
||||
endedMu.Lock()
|
||||
ended[w.Ask] = w
|
||||
endedMu.Unlock()
|
||||
}
|
||||
approved, err := ReadApproved(p)
|
||||
if err != nil {
|
||||
tell()
|
||||
return "", err
|
||||
}
|
||||
if _, done := approved.Decided[w.Ask]; done {
|
||||
tell()
|
||||
return "", nil
|
||||
}
|
||||
// Heard, not read from a record: Reconcile still reads the router's record of it, so an ask the router took after
|
||||
@@ -856,7 +862,9 @@ func HeardEnd(p Paths, store ProposalStore, w asks.Warrant, now time.Time) (stri
|
||||
}
|
||||
approved.Decided[w.Ask] = d
|
||||
delete(approved.Refused, w.Ask)
|
||||
if err := writeApproved(p, approved); err != nil {
|
||||
err = writeApproved(p, approved)
|
||||
tell()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return "proposal " + w.Ask + ": " + d.Outcome, nil
|
||||
|
||||
@@ -0,0 +1,628 @@
|
||||
package main
|
||||
|
||||
// A question an agent session asks the operator, answered on a channel that proves who answered (novox/hq ADR 0332,
|
||||
// issue 589).
|
||||
//
|
||||
// A session that needs a yes, a no or a choice asks it with claude_code_ask instead of waiting at a terminal the
|
||||
// operator may not be watching. The ask is published under this module's name on the operator channel (ADR 0259):
|
||||
// the bus grants that subject to this module alone, so the asker is proven by the bus, and the router stays the judge
|
||||
// of answers, never an asker for others. Every option is at the level approve, so only a channel that proves who
|
||||
// answered (Telegram, today) carries it, and each option binds the digest of the question and of itself.
|
||||
//
|
||||
// The session reads the answer by the question's key with claude_code_answer: from the router's own record of the
|
||||
// ask, which only the router writes and only this module reads, held to the question as the `questions` state keeps
|
||||
// it (asks.Warrant.For and asks.Option.Performs). A record that does not hold is said as not proven, never as a
|
||||
// choice. An answer informs the session; it authorises nothing (ADR 0286 stays the way to an approved call).
|
||||
|
||||
import (
|
||||
"crypto/rand"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
)
|
||||
|
||||
// QuestionsState is the module's state the questions are kept in.
|
||||
const QuestionsState = "questions"
|
||||
|
||||
// QuestionPrefix starts every question's key, so the router's word on one is never taken for a proposal's.
|
||||
const QuestionPrefix = "q-"
|
||||
|
||||
// MostQuestions is how many questions may wait at once, mesh-wide: the router allows an asker three open asks
|
||||
// (asks.MostOpen), and questions alone never fill them, so one is left for a proposed instruction section (ADR 0274)
|
||||
// while two questions are open. Each open question holds one slot key in the `questions` state (slotKey), taken by
|
||||
// create or by compare-and-set, so two nodes asking at once cannot both take the last one.
|
||||
const MostQuestions = asks.MostOpen - 1
|
||||
|
||||
// A question the router holds no record of, and whose refusal was not heard here, keeps its slot until it expires:
|
||||
// the router may still have it in its queue (down, or backlogged), and neither the queue's order nor a cancel can
|
||||
// take it back — the queue offers a piece of work again after an error, so a cancel may be taken before the ask it
|
||||
// cancels, and a cancel of an ask not yet recorded is dropped. Once the question has expired the router refuses it
|
||||
// when it is taken (asks.Ask.Check), so the slot is free and the question has ended.
|
||||
|
||||
// slotKey is the key of one of the MostQuestions slots: not a question's key, so no question is ever read from it.
|
||||
func slotKey(n int) string { return fmt.Sprintf("slot-%d", n) }
|
||||
|
||||
// slot is what a slot key holds: the question holding it.
|
||||
type slot struct {
|
||||
Question string `json:"question"`
|
||||
At time.Time `json:"at"`
|
||||
}
|
||||
|
||||
// The bounds of a question's life (ADR 0332 §2): at least five minutes, at most a day less ten minutes (the
|
||||
// router's bound for an approval, with the margin a proposal keeps), an hour when none is given.
|
||||
const (
|
||||
QuestionShortest = 5 * time.Minute
|
||||
QuestionLongest = ProposalLasts
|
||||
QuestionDefault = time.Hour
|
||||
)
|
||||
|
||||
// The bounds of what a question says.
|
||||
const (
|
||||
FewestQuestionOptions = 2
|
||||
MostContextBytes = 1500
|
||||
MostMeansBytes = 200
|
||||
MostWhyBytes = 300
|
||||
MostOnExpiryBytes = 200
|
||||
)
|
||||
|
||||
// How long one claude_code_answer call may wait for the answer, and how often it looks meanwhile.
|
||||
var (
|
||||
AnswerWaitsAtMost = 60 * time.Second
|
||||
AnswerLooksEvery = 2 * time.Second
|
||||
)
|
||||
|
||||
// QuestionOption is one answer a session offers.
|
||||
type QuestionOption struct {
|
||||
ID string `json:"id"`
|
||||
Label string `json:"label"`
|
||||
Means string `json:"means"`
|
||||
Recommended bool `json:"recommended,omitempty"`
|
||||
}
|
||||
|
||||
// QuestionStore is the `questions` state: a value with its revision, created only when absent, updated only at the
|
||||
// revision read.
|
||||
type QuestionStore interface {
|
||||
Get(key string) (json.RawMessage, uint64, bool, error)
|
||||
Create(key string, value any) error
|
||||
Update(key string, value any, revision uint64) error
|
||||
Put(key string, value any) error
|
||||
}
|
||||
|
||||
// keptQuestions is the `questions` state on the bus.
|
||||
type keptQuestions struct{ s stdio.KeptState }
|
||||
|
||||
func (k keptQuestions) Get(key string) (json.RawMessage, uint64, bool, error) {
|
||||
e, err := k.s.Get(key)
|
||||
if err != nil || e == nil {
|
||||
return nil, 0, false, err
|
||||
}
|
||||
return e.Value, e.Revision, true, nil
|
||||
}
|
||||
func (k keptQuestions) Create(key string, value any) error {
|
||||
_, err := k.s.Create(key, value)
|
||||
return err
|
||||
}
|
||||
func (k keptQuestions) Update(key string, value any, revision uint64) error {
|
||||
_, err := k.s.Update(key, value, revision)
|
||||
return err
|
||||
}
|
||||
func (k keptQuestions) Put(key string, value any) error { _, err := k.s.Put(key, value); return err }
|
||||
|
||||
// Question is one question, as the `questions` state keeps it. Every field the ask is composed from is here, so the
|
||||
// answer is held to exactly the question asked.
|
||||
type Question struct {
|
||||
ID string `json:"id"`
|
||||
Headline string `json:"headline"`
|
||||
Context string `json:"context"`
|
||||
Options []QuestionOption `json:"options"`
|
||||
Why string `json:"why"`
|
||||
OnExpiry string `json:"on-expiry"`
|
||||
By string `json:"by"`
|
||||
From string `json:"from"`
|
||||
At time.Time `json:"at"`
|
||||
Expires time.Time `json:"expires"`
|
||||
// Asked is empty once the ask was published, or says why it was not; nothing composes from it.
|
||||
Asked string `json:"asked,omitempty"`
|
||||
// Ended is the router's word that the ask ended without a choice, as heard on the decided subject: a refused ask
|
||||
// leaves no record of the router's to read it from. It never makes a choice.
|
||||
Ended string `json:"ended,omitempty"`
|
||||
}
|
||||
|
||||
// recommended is the option marked as recommended.
|
||||
func (q Question) recommended() (QuestionOption, bool) {
|
||||
for _, o := range q.Options {
|
||||
if o.Recommended {
|
||||
return o, true
|
||||
}
|
||||
}
|
||||
return QuestionOption{}, false
|
||||
}
|
||||
|
||||
// Act is what an answer binds: this question, whole, and the option chosen.
|
||||
func (q Question) Act(option string) asks.Act {
|
||||
act := asks.Act{"verb": Module + ".question", "option": option, "question": q.ID, "headline": q.Headline,
|
||||
"context": q.Context, "why": q.Why, "on-expiry": q.OnExpiry, "by": q.By, "from": q.From,
|
||||
"at": q.At.UTC().Format(time.RFC3339Nano), "expires": q.Expires.UTC().Format(time.RFC3339Nano),
|
||||
"options": strconv.Itoa(len(q.Options))}
|
||||
for i, o := range q.Options {
|
||||
act[fmt.Sprintf("option.%d", i)] = o.ID + "\n" + o.Label + "\n" + o.Means + "\n" + strconv.FormatBool(o.Recommended)
|
||||
}
|
||||
return act
|
||||
}
|
||||
|
||||
// Ask is the ask for a question, composed from it alone.
|
||||
func (q Question) Ask() asks.Ask {
|
||||
var b strings.Builder
|
||||
b.WriteString(strings.TrimSpace(q.Context))
|
||||
if rec, ok := q.recommended(); ok {
|
||||
fmt.Fprintf(&b, "\n\nRecommended: %s, because %s", rec.Label, strings.TrimRight(strings.TrimSpace(q.Why), "."))
|
||||
b.WriteString(".")
|
||||
}
|
||||
fmt.Fprintf(&b, "\n\nAsked by the agent session %s on %s through the claude-code module, at %s UTC. "+
|
||||
"Your answer is told to that session; it performs nothing by itself.", q.By, q.From, q.At.UTC().Format("15:04 on 2 Jan"))
|
||||
a := asks.Ask{ID: q.ID, Headline: q.Headline, Explanation: b.String(), Who: asks.Operator, Expires: q.Expires,
|
||||
OnExpiry: q.OnExpiry}
|
||||
for _, o := range q.Options {
|
||||
does := o.Means
|
||||
if o.Recommended {
|
||||
does = "Recommended. " + does
|
||||
}
|
||||
binds, _ := asks.ActDigest(q.Act(o.ID))
|
||||
a.Options = append(a.Options, asks.Option{ID: o.ID, Label: o.Label, Does: does, Level: asks.Approve, Binds: binds})
|
||||
}
|
||||
return a
|
||||
}
|
||||
|
||||
var sessionName = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9_-]{0,47}$`)
|
||||
|
||||
// QuestionProblem says why a question cannot be asked, or "": every bound of ADR 0332 §2 that this module holds
|
||||
// itself. The words are held to the content rule by the router, which says its refusal back.
|
||||
func QuestionProblem(q Question) string {
|
||||
var problems []string
|
||||
add := func(format string, args ...any) { problems = append(problems, fmt.Sprintf(format, args...)) }
|
||||
switch {
|
||||
case strings.TrimSpace(q.Headline) == "":
|
||||
add("it has no headline")
|
||||
case len([]rune(q.Headline)) > asks.HeadlineLength:
|
||||
add("its headline is %d characters, more than %d", len([]rune(q.Headline)), asks.HeadlineLength)
|
||||
}
|
||||
switch {
|
||||
case strings.TrimSpace(q.Context) == "":
|
||||
add("it gives no context: say in plain words what the operator decides and why it matters")
|
||||
case len(q.Context) > MostContextBytes:
|
||||
add("its context is %d bytes, more than the %d a phone shows in one glance", len(q.Context), MostContextBytes)
|
||||
}
|
||||
if n := len(q.Options); n < FewestQuestionOptions || n > asks.MostOptions {
|
||||
add("it offers %d options; a question offers %d to %d", n, FewestQuestionOptions, asks.MostOptions)
|
||||
}
|
||||
recommended, labels := 0, map[string]bool{}
|
||||
for i, o := range q.Options {
|
||||
switch {
|
||||
case strings.TrimSpace(o.Label) == "":
|
||||
add("option %d has no label", i+1)
|
||||
case len([]rune(o.Label)) > asks.LabelLength:
|
||||
add("option %d's label is longer than %d characters", i+1, asks.LabelLength)
|
||||
case labels[strings.ToLower(o.Label)]:
|
||||
add("two options are labelled %q", o.Label)
|
||||
}
|
||||
labels[strings.ToLower(o.Label)] = true
|
||||
if strings.TrimSpace(o.Means) == "" {
|
||||
add("option %d does not say what choosing it means", i+1)
|
||||
} else if len(o.Means) > MostMeansBytes {
|
||||
add("what option %d means is longer than %d bytes", i+1, MostMeansBytes)
|
||||
}
|
||||
if !asks.UsableID(o.ID) {
|
||||
add("option %d's id is not one", i+1)
|
||||
}
|
||||
if o.Recommended {
|
||||
recommended++
|
||||
}
|
||||
}
|
||||
if recommended != 1 {
|
||||
add("%d options are marked recommended; exactly one is", recommended)
|
||||
}
|
||||
switch {
|
||||
case strings.TrimSpace(q.Why) == "":
|
||||
add("it does not say why the recommended option is recommended")
|
||||
case len(q.Why) > MostWhyBytes:
|
||||
add("why the option is recommended is longer than %d bytes", MostWhyBytes)
|
||||
}
|
||||
switch {
|
||||
case strings.TrimSpace(q.OnExpiry) == "":
|
||||
add("it does not say what the session does if nobody answers")
|
||||
case len(q.OnExpiry) > MostOnExpiryBytes:
|
||||
add("what happens if nobody answers is longer than %d bytes", MostOnExpiryBytes)
|
||||
}
|
||||
if !sessionName.MatchString(q.By) {
|
||||
add("who asks is not a session's name: letters, digits, - and _, such as fix-589")
|
||||
}
|
||||
if !nodeName.MatchString(q.From) {
|
||||
add("where it comes from is not a node's name")
|
||||
}
|
||||
if life := q.Expires.Sub(q.At); life < QuestionShortest || life > QuestionLongest {
|
||||
add("it lasts %s; a question lasts %s to %s", life.Round(time.Minute), QuestionShortest, QuestionLongest)
|
||||
}
|
||||
if problem := questionWordsProblem(q); problem != "" {
|
||||
add("%s", problem)
|
||||
}
|
||||
if len(problems) == 0 {
|
||||
return ""
|
||||
}
|
||||
return strings.Join(problems, "; ")
|
||||
}
|
||||
|
||||
// questionWordsProblem says why a question's words may not be kept at all, or "": anything shaped like a secret, or a
|
||||
// character a phone does not show, in any word the operator is shown — the headline, the context, why, what happens
|
||||
// on expiry, and each option's label and meaning. Judged before anything is kept: the `questions` state is read by
|
||||
// every node, and the router judges the content rule only after the question was kept.
|
||||
func questionWordsProblem(q Question) string {
|
||||
fields := []struct{ name, text string }{{"the headline", q.Headline}, {"the context", q.Context}, {"why", q.Why},
|
||||
{"what happens on expiry", q.OnExpiry}}
|
||||
for i, o := range q.Options {
|
||||
fields = append(fields, struct{ name, text string }{fmt.Sprintf("option %d's label", i+1), o.Label},
|
||||
struct{ name, text string }{fmt.Sprintf("what option %d means", i+1), o.Means})
|
||||
}
|
||||
for _, f := range fields {
|
||||
for _, re := range secretShaped {
|
||||
if re.MatchString(f.text) {
|
||||
return f.name + " holds something shaped like a secret, which is never kept or shown on a phone"
|
||||
}
|
||||
}
|
||||
if what := invisible(f.text); what != "" {
|
||||
return f.name + " holds " + what + ", which a phone does not show"
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func newQuestionID() string {
|
||||
var b [8]byte
|
||||
_, _ = rand.Read(b[:])
|
||||
return QuestionPrefix + hex.EncodeToString(b[:])
|
||||
}
|
||||
|
||||
// QuestionInput is what a session asks.
|
||||
type QuestionInput struct {
|
||||
Headline string
|
||||
Context string
|
||||
Options []QuestionOption
|
||||
Why string
|
||||
OnExpiry string
|
||||
By string
|
||||
Expires time.Duration
|
||||
}
|
||||
|
||||
// AskQuestion keeps a question and asks the operator. It answers within seconds, once the router has said whether it
|
||||
// took the question, with the question's key; the answer is read with ReadAnswer.
|
||||
func AskQuestion(p Paths, in QuestionInput, store QuestionStore, bus AskBus, now time.Time) (map[string]any, error) {
|
||||
refuse := func(why string) (map[string]any, error) { return nil, errors.New(why + ". Nothing was asked") }
|
||||
by := strings.TrimSpace(in.By)
|
||||
if who, node, found := strings.Cut(by, "@"); found {
|
||||
if node != p.Node {
|
||||
return refuse(fmt.Sprintf("who asks names the node %q, and this is %s: ask on the node the session runs on", node, p.Node))
|
||||
}
|
||||
by = who
|
||||
}
|
||||
if by == "" {
|
||||
return refuse("it does not say who asks: give by, your session's name as who@node, such as fix-589@g14")
|
||||
}
|
||||
life := in.Expires
|
||||
if life == 0 {
|
||||
life = QuestionDefault
|
||||
}
|
||||
q := Question{ID: newQuestionID(), Headline: strings.TrimSpace(in.Headline), Context: strings.TrimSpace(in.Context),
|
||||
Why: strings.TrimSpace(in.Why), OnExpiry: strings.TrimSpace(in.OnExpiry), By: by, From: p.Node, At: now.UTC(),
|
||||
Expires: now.UTC().Add(life)}
|
||||
for i, o := range in.Options {
|
||||
q.Options = append(q.Options, QuestionOption{ID: fmt.Sprintf("o%d", i+1), Label: strings.TrimSpace(o.Label),
|
||||
Means: strings.TrimSpace(o.Means), Recommended: o.Recommended})
|
||||
}
|
||||
if problem := QuestionProblem(q); problem != "" {
|
||||
return refuse(problem)
|
||||
}
|
||||
ask := q.Ask()
|
||||
if err := ask.Check(now); err != nil {
|
||||
return refuse(err.Error())
|
||||
}
|
||||
// The question is kept before its slot is taken, so a slot always names a question another node can judge.
|
||||
if err := store.Create(q.ID, q); err != nil {
|
||||
return nil, fmt.Errorf("the question could not be kept, so the operator was not asked: %w", err)
|
||||
}
|
||||
if open, err := takeSlot(store, bus, q, now); err != nil {
|
||||
_ = store.Put(q.ID, tombstone(q, "not asked: "+err.Error()))
|
||||
return refuse(err.Error())
|
||||
} else if open != nil {
|
||||
_ = store.Put(q.ID, tombstone(q, "not asked: two questions were open"))
|
||||
return refuse(fmt.Sprintf("%d questions already wait for the operator's answer (%s), and sessions hold at most %d "+
|
||||
"at once, so one of the router's three open asks is left for a proposed instruction section; read them with "+
|
||||
"claude_code_answer, or wait for one to end", len(open), strings.Join(open, ", "), MostQuestions))
|
||||
}
|
||||
if err := bus.Publish(asks.AskSubject(Module), ask, ask.ID); err != nil {
|
||||
_ = store.Put(q.ID, tombstone(q, "not asked: "+err.Error()))
|
||||
return nil, fmt.Errorf("the operator could not be asked: %v", err)
|
||||
}
|
||||
say("question %s from the session %s asked of the operator", q.ID, q.By)
|
||||
taken := "the router has not said yet whether it took the question; claude_code_answer says"
|
||||
defer forgetQuestionEnd(q.ID)
|
||||
for deadline := time.Now().Add(RouterAnswersWithin); time.Now().Before(deadline); {
|
||||
if w, ended := questionEnd(q.ID); ended {
|
||||
return nil, fmt.Errorf("the router did not ask the operator: the question %s (%s)", w.Outcome, w.Words)
|
||||
}
|
||||
if entry, err := bus.Record(q.ID); err == nil && entry != nil {
|
||||
taken = "the router took the question and shows it where who answers is proven (Telegram)"
|
||||
break
|
||||
}
|
||||
time.Sleep(RouterAnswersEvery)
|
||||
}
|
||||
return map[string]any{
|
||||
"key": q.ID,
|
||||
"router": taken,
|
||||
"asked": "\"" + q.Headline + "\", with the answers " + labelsOf(q),
|
||||
"expires": q.Expires.Format(time.RFC3339),
|
||||
"next": "read the answer with claude_code_answer {\"key\": \"" + q.ID + "\"} (it may wait up to a minute); " +
|
||||
"keep working meanwhile, and if it expires do what you said: " + q.OnExpiry,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func labelsOf(q Question) string {
|
||||
var out []string
|
||||
for _, o := range q.Options {
|
||||
out = append(out, o.Label)
|
||||
}
|
||||
return strings.Join(out, ", ")
|
||||
}
|
||||
|
||||
// questionOpen says whether a question still waits: kept, asked, not ended, not expired, and open in the router's
|
||||
// record or not recorded yet. An unreadable record is open: a slot is never freed on a guess.
|
||||
func questionOpen(store QuestionStore, bus AskBus, id string, now time.Time) (bool, error) {
|
||||
q, ok, err := readQuestionErr(store, id)
|
||||
if err != nil {
|
||||
return true, err
|
||||
}
|
||||
if !ok || q.Asked != "" || q.Ended != "" || !now.Before(q.Expires) {
|
||||
return false, nil
|
||||
}
|
||||
entry, err := bus.Record(q.ID)
|
||||
if err != nil {
|
||||
return true, err
|
||||
}
|
||||
if entry == nil {
|
||||
return true, nil
|
||||
}
|
||||
var rec routerRecord
|
||||
if json.Unmarshal(entry.Value, &rec) != nil {
|
||||
return true, nil
|
||||
}
|
||||
return rec.State == "" || rec.State == "open", nil
|
||||
}
|
||||
|
||||
// takeSlot takes one of the MostQuestions slots for q: a slot no key holds, by create, or one whose question no longer
|
||||
// waits, by compare-and-set at the revision read. Of two nodes taking the same slot only one wins; the other tries
|
||||
// the next. It answers the open questions when every slot is held, and an error — refusing, never guessing — when a
|
||||
// slot or its question cannot be read.
|
||||
func takeSlot(store QuestionStore, bus AskBus, q Question, now time.Time) ([]string, error) {
|
||||
mine := slot{Question: q.ID, At: q.At}
|
||||
var open []string
|
||||
for n := 1; n <= MostQuestions; n++ {
|
||||
key := slotKey(n)
|
||||
raw, rev, found, err := store.Get(key)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the open questions could not be counted (%v), so none more is asked", err)
|
||||
}
|
||||
if !found {
|
||||
if err := store.Create(key, mine); err == nil {
|
||||
return nil, nil
|
||||
} else if !errors.Is(err, stdio.ErrChanged) {
|
||||
return nil, fmt.Errorf("the open questions could not be counted (%v), so none more is asked", err)
|
||||
}
|
||||
if raw, rev, found, err = store.Get(key); err != nil || !found {
|
||||
return nil, fmt.Errorf("the open questions could not be counted, so none more is asked")
|
||||
}
|
||||
}
|
||||
var held slot
|
||||
if json.Unmarshal(raw, &held) == nil && held.Question != "" {
|
||||
waits, err := questionOpen(store, bus, held.Question, now)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the open questions could not be counted (%v), so none more is asked", err)
|
||||
}
|
||||
if waits {
|
||||
open = append(open, held.Question)
|
||||
continue
|
||||
}
|
||||
}
|
||||
if err := store.Update(key, mine, rev); err == nil {
|
||||
return nil, nil
|
||||
} else if !errors.Is(err, stdio.ErrChanged) {
|
||||
return nil, fmt.Errorf("the open questions could not be counted (%v), so none more is asked", err)
|
||||
}
|
||||
open = append(open, "one taken just now")
|
||||
}
|
||||
sort.Strings(open)
|
||||
return open, nil
|
||||
}
|
||||
|
||||
// tombstone is what is kept of a question that was not asked or that the router refused: its key, who asked, when, and
|
||||
// why it ended — none of its words, which the router judged unfit to leave the mesh or never saw.
|
||||
func tombstone(q Question, ended string) Question {
|
||||
return Question{ID: q.ID, By: q.By, From: q.From, At: q.At, Expires: q.Expires, Ended: ended}
|
||||
}
|
||||
|
||||
func readQuestion(store QuestionStore, key string) (Question, bool) {
|
||||
q, ok, _ := readQuestionErr(store, key)
|
||||
return q, ok
|
||||
}
|
||||
|
||||
func readQuestionErr(store QuestionStore, key string) (Question, bool, error) {
|
||||
if !strings.HasPrefix(key, QuestionPrefix) {
|
||||
return Question{}, false, nil
|
||||
}
|
||||
raw, _, found, err := store.Get(key)
|
||||
if err != nil || !found {
|
||||
return Question{}, false, err
|
||||
}
|
||||
var q Question
|
||||
if json.Unmarshal(raw, &q) != nil || q.ID != key {
|
||||
return Question{}, false, nil
|
||||
}
|
||||
return q, true, nil
|
||||
}
|
||||
|
||||
// The router's word that a question ended without a choice, as heard on the decided subject.
|
||||
var (
|
||||
questionEndsMu sync.Mutex
|
||||
questionEnds = map[string]asks.Warrant{}
|
||||
)
|
||||
|
||||
func questionEnd(id string) (asks.Warrant, bool) {
|
||||
questionEndsMu.Lock()
|
||||
defer questionEndsMu.Unlock()
|
||||
w, ok := questionEnds[id]
|
||||
return w, ok
|
||||
}
|
||||
|
||||
func forgetQuestionEnd(id string) {
|
||||
questionEndsMu.Lock()
|
||||
delete(questionEnds, id)
|
||||
questionEndsMu.Unlock()
|
||||
}
|
||||
|
||||
// HeardQuestionEnd takes the router's word that a question ended without a choice — refused, expired, cancelled or
|
||||
// replaced — and keeps it on the question, because a refused ask leaves no record of the router's. A refused question
|
||||
// keeps none of its words (tombstone). A choice is never taken from this word: only from the router's record, held to
|
||||
// the question (ReadAnswer).
|
||||
func HeardQuestionEnd(store QuestionStore, w asks.Warrant) {
|
||||
if w.Asker != Module || w.Outcome == asks.OutcomeChosen || !strings.HasPrefix(w.Ask, QuestionPrefix) {
|
||||
return
|
||||
}
|
||||
q, known := readQuestion(store, w.Ask)
|
||||
if !known {
|
||||
return
|
||||
}
|
||||
if q.Ended == "" {
|
||||
ended := string(w.Outcome)
|
||||
if w.Words != "" {
|
||||
ended += ": " + w.Words
|
||||
}
|
||||
if w.Outcome == asks.OutcomeRefused {
|
||||
q = tombstone(q, ended)
|
||||
} else {
|
||||
q.Ended = ended
|
||||
}
|
||||
_ = store.Put(q.ID, q)
|
||||
}
|
||||
// Kept on the question first, then said to a session waiting on it: what it reads next is already there.
|
||||
questionEndsMu.Lock()
|
||||
questionEnds[w.Ask] = w
|
||||
questionEndsMu.Unlock()
|
||||
}
|
||||
|
||||
// The answer's states, as claude_code_answer says them.
|
||||
const (
|
||||
AnswerWaiting = "waiting"
|
||||
AnswerChosen = "chosen"
|
||||
AnswerEnded = "ended"
|
||||
AnswerNotProven = "not proven"
|
||||
)
|
||||
|
||||
// ReadAnswer is where a question stands: waiting, chosen (and what), ended without a choice, or not proven.
|
||||
func ReadAnswer(store QuestionStore, bus AskBus, key string, now time.Time) (map[string]any, error) {
|
||||
q, ok := readQuestion(store, strings.TrimSpace(key))
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("no question %q is kept: the key is the one claude_code_ask gave", key)
|
||||
}
|
||||
out := map[string]any{"key": q.ID, "headline": q.Headline, "asked": q.At.Format(time.RFC3339),
|
||||
"expires": q.Expires.Format(time.RFC3339)}
|
||||
if q.Asked != "" {
|
||||
out["state"], out["words"] = AnswerEnded, q.Asked
|
||||
return out, nil
|
||||
}
|
||||
entry, err := bus.Record(q.ID)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the router's record of the question could not be read now (%v); ask again", err)
|
||||
}
|
||||
var rec routerRecord
|
||||
if entry != nil {
|
||||
if err := json.Unmarshal(entry.Value, &rec); err != nil {
|
||||
out["state"], out["words"] = AnswerNotProven, "the router's record of it cannot be read: "+err.Error()
|
||||
return out, nil
|
||||
}
|
||||
}
|
||||
switch {
|
||||
case entry == nil && q.Ended != "":
|
||||
out["state"], out["words"] = AnswerEnded, q.Ended
|
||||
case entry == nil && !now.Before(q.Expires):
|
||||
out["state"], out["words"] = AnswerEnded, "expired without the router recording it: refused (its refusal was not "+
|
||||
"heard here) or never taken; the router refuses it now if it is taken late"
|
||||
out["now"] = "do what you said: " + q.OnExpiry
|
||||
case entry == nil:
|
||||
out["state"] = AnswerWaiting
|
||||
out["words"] = "not yet recorded by the router: it may still be in the router's queue, or was refused without the " +
|
||||
"refusal being heard here. It holds its slot, and the operator may still be asked, until it expires at " +
|
||||
q.Expires.Format(time.RFC3339)
|
||||
case entry == nil, rec.State == "open", rec.Warrant == nil:
|
||||
out["state"] = AnswerWaiting
|
||||
if q.Ended != "" {
|
||||
out["words"] = "the router said it ended (" + q.Ended + "), and its record says it is open; read again"
|
||||
}
|
||||
default:
|
||||
w := *rec.Warrant
|
||||
if w.Outcome != asks.OutcomeChosen {
|
||||
out["state"], out["words"] = AnswerEnded, string(w.Outcome)
|
||||
if w.Words != "" {
|
||||
out["words"] = string(w.Outcome) + ": " + w.Words
|
||||
}
|
||||
if w.Outcome == asks.OutcomeExpired {
|
||||
out["now"] = "do what you said: " + q.OnExpiry
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
if problem := QuestionProblem(q); problem != "" {
|
||||
out["state"], out["words"] = AnswerNotProven, "the question kept is not one this module asks: "+problem
|
||||
return out, nil
|
||||
}
|
||||
option, err := w.For(Module, q.Ask())
|
||||
if err == nil {
|
||||
err = option.Performs(q.Act(option.ID))
|
||||
}
|
||||
if err != nil {
|
||||
say("REFUSED the answer to question %s: %v", q.ID, err)
|
||||
out["state"], out["words"] = AnswerNotProven, "the router's record does not hold to the question asked: "+err.Error()
|
||||
return out, nil
|
||||
}
|
||||
chosen := map[string]any{"id": option.ID, "label": option.Label}
|
||||
for _, o := range q.Options {
|
||||
if o.ID == option.ID {
|
||||
chosen["means"], chosen["recommended"] = o.Means, o.Recommended
|
||||
}
|
||||
}
|
||||
out["state"], out["chosen"], out["by"], out["channel"], out["at"] = AnswerChosen, chosen, w.Says(), w.Channel,
|
||||
w.At.Format(time.RFC3339)
|
||||
out["note"] = "the operator's word to this session; it authorises no call that needs the operator's approval"
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// WaitForAnswer reads the answer, and while it is waiting reads it again until wait has passed.
|
||||
func WaitForAnswer(store QuestionStore, bus AskBus, key string, wait time.Duration, now func() time.Time) (map[string]any, error) {
|
||||
if wait > AnswerWaitsAtMost {
|
||||
wait = AnswerWaitsAtMost
|
||||
}
|
||||
deadline := time.Now().Add(wait)
|
||||
for {
|
||||
out, err := ReadAnswer(store, bus, key, now())
|
||||
if err != nil || out["state"] != AnswerWaiting || !time.Now().Before(deadline) {
|
||||
return out, err
|
||||
}
|
||||
time.Sleep(AnswerLooksEvery)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,513 @@
|
||||
package main
|
||||
|
||||
// A session's question to the operator (novox/hq ADR 0332, issue 589): asked under this module's name at the level
|
||||
// approve, refused outside its bounds, at most two waiting, and its answer read back by its key only when the
|
||||
// router's record holds to the question asked.
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
)
|
||||
|
||||
func mergeTonight() QuestionInput {
|
||||
return QuestionInput{
|
||||
Headline: "Merge the resolver fix tonight?",
|
||||
Context: "The fix for the resolver on the laptop is reviewed and both checks are green. Merging now rolls it to every node within the hour.",
|
||||
Options: []QuestionOption{
|
||||
{Label: "Merge now", Means: "I merge it and follow the rollout", Recommended: true},
|
||||
{Label: "Wait", Means: "I leave the pull request open until you say"},
|
||||
},
|
||||
Why: "it is reviewed and the checks are green",
|
||||
OnExpiry: "I leave the pull request open",
|
||||
By: "fix-589@g14",
|
||||
}
|
||||
}
|
||||
|
||||
func askMergeTonight(t *testing.T, p Paths, store *memQuestions, r *router, at time.Time) string {
|
||||
t.Helper()
|
||||
out, err := AskQuestion(p, mergeTonight(), store, r, at)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return out["key"].(string)
|
||||
}
|
||||
|
||||
func TestAQuestionIsAskedUnderTheModulesNameAtTheLevelApprove(t *testing.T) {
|
||||
p, _ := node(t, "g14")
|
||||
store, r := newMemQuestions(), newRouter()
|
||||
out, err := AskQuestion(p, mergeTonight(), store, r, t0)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
key := out["key"].(string)
|
||||
if !strings.HasPrefix(key, QuestionPrefix) || !strings.Contains(out["router"].(string), "took the question") {
|
||||
t.Fatalf("answered %v", out)
|
||||
}
|
||||
if len(r.subjects) != 1 || r.subjects[0] != asks.AskSubject(Module) {
|
||||
t.Fatalf("published on %v", r.subjects)
|
||||
}
|
||||
a := r.published[key]
|
||||
if a.Who != asks.Operator || a.OnExpiry != "I leave the pull request open" || !a.Expires.Equal(t0.Add(QuestionDefault)) {
|
||||
t.Errorf("ask %+v", a)
|
||||
}
|
||||
if len(a.Options) != 2 {
|
||||
t.Fatalf("options %+v", a.Options)
|
||||
}
|
||||
for _, o := range a.Options {
|
||||
if o.Level != asks.Approve || o.Binds == "" {
|
||||
t.Errorf("option %s is %s binding %q: every option is an approval bound to the question", o.ID, o.Level, o.Binds)
|
||||
}
|
||||
}
|
||||
if !strings.HasPrefix(a.Options[0].Does, "Recommended. ") || strings.HasPrefix(a.Options[1].Does, "Recommended") {
|
||||
t.Errorf("the recommendation is not on the recommended option alone: %+v", a.Options)
|
||||
}
|
||||
for _, want := range []string{"Recommended: Merge now, because it is reviewed and the checks are green.",
|
||||
"the agent session fix-589 on g14", "performs nothing by itself"} {
|
||||
if !strings.Contains(a.Explanation, want) {
|
||||
t.Errorf("the explanation does not say %q:\n%s", want, a.Explanation)
|
||||
}
|
||||
}
|
||||
if err := a.Check(t0); err != nil {
|
||||
t.Errorf("the ask does not pass the asks contract: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheChosenAnswerIsReadBackByItsKey(t *testing.T) {
|
||||
p, _ := node(t, "g14")
|
||||
store, r := newMemQuestions(), newRouter()
|
||||
key := askMergeTonight(t, p, store, r, t0)
|
||||
out, err := ReadAnswer(store, r, key, t0.Add(time.Minute))
|
||||
if err != nil || out["state"] != AnswerWaiting {
|
||||
t.Fatalf("before an answer: %v %v", out, err)
|
||||
}
|
||||
r.close(r.chose(key, "o2", t0.Add(10*time.Minute)))
|
||||
out, err = ReadAnswer(store, r, key, t0.Add(11*time.Minute))
|
||||
if err != nil || out["state"] != AnswerChosen {
|
||||
t.Fatalf("after the choice: %v %v", out, err)
|
||||
}
|
||||
chosen := out["chosen"].(map[string]any)
|
||||
if chosen["id"] != "o2" || chosen["label"] != "Wait" || chosen["means"] != "I leave the pull request open until you say" {
|
||||
t.Errorf("chosen %v", chosen)
|
||||
}
|
||||
if !strings.Contains(out["by"].(string), "the operator, via telegram") || out["channel"] != "telegram" {
|
||||
t.Errorf("by %v on %v", out["by"], out["channel"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnAnswerThatDoesNotHoldToTheQuestionIsNotProven(t *testing.T) {
|
||||
p, _ := node(t, "g14")
|
||||
store, r := newMemQuestions(), newRouter()
|
||||
key := askMergeTonight(t, p, store, r, t0)
|
||||
w := r.chose(key, "o1", t0.Add(time.Minute))
|
||||
|
||||
// The question rewritten on the bus after it was asked: the operator was shown other words.
|
||||
var q Question
|
||||
_ = json.Unmarshal(store.m[key], &q)
|
||||
q.Context = "Merging now deletes nothing and rolls it to every node."
|
||||
_ = store.Put(key, q)
|
||||
r.close(w)
|
||||
if out, _ := ReadAnswer(store, r, key, t0.Add(2*time.Minute)); out["state"] != AnswerNotProven {
|
||||
t.Errorf("a question changed after it was asked read as %v", out)
|
||||
}
|
||||
|
||||
// A warrant for another ask's digest.
|
||||
store2, r2 := newMemQuestions(), newRouter()
|
||||
key2 := askMergeTonight(t, p, store2, r2, t0)
|
||||
w2 := r2.chose(key2, "o1", t0.Add(time.Minute))
|
||||
w2.AskDigest = "sha256:0000"
|
||||
r2.close(w2)
|
||||
if out, _ := ReadAnswer(store2, r2, key2, t0.Add(2*time.Minute)); out["state"] != AnswerNotProven {
|
||||
t.Errorf("a warrant for another ask read as %v", out)
|
||||
}
|
||||
|
||||
// A choice made after the question expired.
|
||||
store3, r3 := newMemQuestions(), newRouter()
|
||||
key3 := askMergeTonight(t, p, store3, r3, t0)
|
||||
r3.close(r3.chose(key3, "o1", t0.Add(QuestionDefault+time.Minute)))
|
||||
if out, _ := ReadAnswer(store3, r3, key3, t0.Add(2*QuestionDefault)); out["state"] != AnswerNotProven {
|
||||
t.Errorf("a choice after expiry read as %v", out)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAQuestionIsRefusedOutsideItsBounds(t *testing.T) {
|
||||
p, _ := node(t, "g14")
|
||||
for name, change := range map[string]func(*QuestionInput){
|
||||
"one option": func(in *QuestionInput) { in.Options = in.Options[:1] },
|
||||
"five options": func(in *QuestionInput) { in.Options = append(in.Options, in.Options[1], in.Options[1], in.Options[1]) },
|
||||
"none recommended": func(in *QuestionInput) { in.Options[0].Recommended = false },
|
||||
"two recommended": func(in *QuestionInput) { in.Options[1].Recommended = true },
|
||||
"no why": func(in *QuestionInput) { in.Why = "" },
|
||||
"no context": func(in *QuestionInput) { in.Context = "" },
|
||||
"no on-expiry": func(in *QuestionInput) { in.OnExpiry = "" },
|
||||
"a long headline": func(in *QuestionInput) { in.Headline = strings.Repeat("Merge ", 11) },
|
||||
"a long label": func(in *QuestionInput) { in.Options[0].Label = "Merge it now and follow the rollout" },
|
||||
"an option unsaid": func(in *QuestionInput) { in.Options[1].Means = "" },
|
||||
"too short": func(in *QuestionInput) { in.Expires = time.Minute },
|
||||
"too long": func(in *QuestionInput) { in.Expires = 2 * 24 * time.Hour },
|
||||
"another node": func(in *QuestionInput) { in.By = "fix-589@shanks" },
|
||||
"an invisible char": func(in *QuestionInput) { in.Context += "\u200b" },
|
||||
"invisible label": func(in *QuestionInput) { in.Options[0].Label = "Merge\u200bnow" },
|
||||
"invisible means": func(in *QuestionInput) { in.Options[1].Means += "\u202e" },
|
||||
"a token, context": func(in *QuestionInput) { in.Context += " The key is ghp_abcdefghijklmnopqrstuvwxyz0123456789." },
|
||||
"a token, headline": func(in *QuestionInput) { in.Headline = "Use sk-abcdefghijklmnopqrstuv?" },
|
||||
"a token, why": func(in *QuestionInput) { in.Why = "password: hunter2 works" },
|
||||
"a token, expiry": func(in *QuestionInput) { in.OnExpiry = "I keep AKIAABCDEFGHIJKLMNOP" },
|
||||
"a token, label": func(in *QuestionInput) { in.Options[0].Label = "token=abc" },
|
||||
"a token, means": func(in *QuestionInput) { in.Options[1].Means = "-----BEGIN PRIVATE KEY-----" },
|
||||
"no by": func(in *QuestionInput) { in.By = "" },
|
||||
"61 characters": func(in *QuestionInput) { in.Headline = strings.Repeat("x", 60) + "?" },
|
||||
"1431 minutes": func(in *QuestionInput) { in.Expires = 1431 * time.Minute },
|
||||
} {
|
||||
store, r := newMemQuestions(), newRouter()
|
||||
in := mergeTonight()
|
||||
in.Options = append([]QuestionOption(nil), in.Options...)
|
||||
change(&in)
|
||||
if _, err := AskQuestion(p, in, store, r, t0); err == nil || !strings.Contains(err.Error(), "Nothing was asked") {
|
||||
t.Errorf("%s: %v", name, err)
|
||||
}
|
||||
if len(r.subjects) != 0 || len(store.m) != 0 {
|
||||
t.Errorf("%s: asked or kept anyway", name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAtMostTwoQuestionsWaitSoAProposalFindsRoom(t *testing.T) {
|
||||
p, _ := node(t, "g14")
|
||||
store, r := newMemQuestions(), newRouter()
|
||||
first := askMergeTonight(t, p, store, r, t0)
|
||||
askMergeTonight(t, p, store, r, t0)
|
||||
if _, err := AskQuestion(p, mergeTonight(), store, r, t0); err == nil || !strings.Contains(err.Error(), "2 questions already wait") {
|
||||
t.Fatalf("a third question: %v", err)
|
||||
}
|
||||
r.close(r.chose(first, "o1", t0.Add(time.Minute)))
|
||||
if _, err := AskQuestion(p, mergeTonight(), store, r, t0.Add(2*time.Minute)); err != nil {
|
||||
t.Fatalf("a question once one was answered: %v", err)
|
||||
}
|
||||
// A proposal is still asked beside two waiting questions.
|
||||
pstore := &memProposals{m: map[string]json.RawMessage{}}
|
||||
if _, err := Propose(p, ProposeInput{Scope: ScopeMesh, Name: "x", Text: "Text.\n"}, pstore, r, t0); err != nil {
|
||||
t.Errorf("a proposal beside two questions: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// questionRefusingRouter refuses every ask as the router does for words that may not leave the mesh: on the decided
|
||||
// subject, with no record.
|
||||
type questionRefusingRouter struct {
|
||||
*router
|
||||
store QuestionStore
|
||||
id string
|
||||
}
|
||||
|
||||
func (r *questionRefusingRouter) Publish(subject string, body any, id string) error {
|
||||
r.id = id
|
||||
go func() {
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
HeardQuestionEnd(r.store, asks.Warrant{Ask: id, Asker: Module, Outcome: asks.OutcomeRefused,
|
||||
Words: "its words carry path (a file path), which may not leave the mesh", At: t0})
|
||||
}()
|
||||
return nil
|
||||
}
|
||||
|
||||
func TestTheRoutersRefusalIsSaidAndKept(t *testing.T) {
|
||||
p, _ := node(t, "g14")
|
||||
store := newMemQuestions()
|
||||
refusing := &questionRefusingRouter{router: newRouter(), store: store}
|
||||
within, every := RouterAnswersWithin, RouterAnswersEvery
|
||||
RouterAnswersWithin, RouterAnswersEvery = time.Second, 10*time.Millisecond
|
||||
t.Cleanup(func() { RouterAnswersWithin, RouterAnswersEvery = within, every })
|
||||
_, err := AskQuestion(p, mergeTonight(), store, refusing, t0)
|
||||
if err == nil || !strings.Contains(err.Error(), "may not leave the mesh") {
|
||||
t.Fatalf("refusal: %v", err)
|
||||
}
|
||||
out, err := ReadAnswer(store, refusing, refusing.id, t0.Add(time.Minute))
|
||||
if err != nil || out["state"] != AnswerEnded || !strings.Contains(out["words"].(string), "refused") {
|
||||
t.Errorf("read after the refusal: %v %v", out, err)
|
||||
}
|
||||
if waits, err := questionOpen(store, refusing, refusing.id, t0.Add(time.Minute)); waits || err != nil {
|
||||
t.Errorf("a refused question still counts as waiting: %v %v", waits, err)
|
||||
}
|
||||
var kept Question
|
||||
_ = json.Unmarshal(store.m[refusing.id], &kept)
|
||||
if kept.Headline != "" || kept.Context != "" || len(kept.Options) != 0 || kept.Why != "" || kept.OnExpiry != "" {
|
||||
t.Errorf("a refused question keeps its words: %+v", kept)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnExpiredQuestionSaysWhatTheSessionDoes(t *testing.T) {
|
||||
p, _ := node(t, "g14")
|
||||
store, r := newMemQuestions(), newRouter()
|
||||
key := askMergeTonight(t, p, store, r, t0)
|
||||
r.close(asks.Warrant{Ask: key, Outcome: asks.OutcomeExpired, At: t0.Add(QuestionDefault)})
|
||||
out, err := ReadAnswer(store, r, key, t0.Add(QuestionDefault+time.Minute))
|
||||
if err != nil || out["state"] != AnswerEnded || out["now"] != "do what you said: I leave the pull request open" {
|
||||
t.Errorf("expired: %v %v", out, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheQuestionToolsAreServed(t *testing.T) {
|
||||
p, _ := node(t, "g14")
|
||||
r := newRouter()
|
||||
questions := newMemQuestions()
|
||||
proposing = &Proposing{Store: &memProposals{m: map[string]json.RawMessage{}}, Bus: r, Now: func() time.Time { return t0 },
|
||||
Write: writer(map[string]string{}), Questions: questions}
|
||||
t.Cleanup(func() { proposing = nil })
|
||||
s := serve(t, p, &bus{kept: map[string]map[string]any{}}, memConfig{})
|
||||
text, isErr := s.call(t, "claude_code_ask", map[string]any{
|
||||
"headline": "Merge the resolver fix tonight?", "context": "Reviewed and green.",
|
||||
"options": []any{
|
||||
map[string]any{"label": "Merge now", "means": "I merge it", "recommended": true},
|
||||
map[string]any{"label": "Wait", "means": "I leave it open"},
|
||||
},
|
||||
"why": "it is reviewed", "on_expiry": "I leave it open", "by": "fix-589@g14", "expires_minutes": 30,
|
||||
})
|
||||
if isErr || !strings.Contains(text, `"key"`) {
|
||||
t.Fatalf("%v %s", isErr, text)
|
||||
}
|
||||
var asked map[string]any
|
||||
_ = json.Unmarshal([]byte(text), &asked)
|
||||
key, _ := asked["key"].(string)
|
||||
if a := r.published[key]; !a.Expires.Equal(t0.Add(30 * time.Minute)) {
|
||||
t.Errorf("expires %v", a.Expires)
|
||||
}
|
||||
r.close(r.chose(key, "o1", t0.Add(time.Minute)))
|
||||
text, isErr = s.call(t, "claude_code_answer", map[string]any{"key": key})
|
||||
if isErr || !strings.Contains(text, `"chosen"`) || !strings.Contains(text, "Merge now") {
|
||||
t.Fatalf("%v %s", isErr, text)
|
||||
}
|
||||
}
|
||||
|
||||
// One read waits at most AnswerWaitsAtMost (60 s), whatever the session asks for, and says waiting when nobody
|
||||
// answered meanwhile.
|
||||
func TestAReadWaitsNoLongerThanItsBound(t *testing.T) {
|
||||
if AnswerWaitsAtMost != 60*time.Second {
|
||||
t.Fatalf("a read may wait %s; ADR 0332 says 60 s", AnswerWaitsAtMost)
|
||||
}
|
||||
p, _ := node(t, "g14")
|
||||
store, r := newMemQuestions(), newRouter()
|
||||
key := askMergeTonight(t, p, store, r, t0)
|
||||
most, every := AnswerWaitsAtMost, AnswerLooksEvery
|
||||
AnswerWaitsAtMost, AnswerLooksEvery = 100*time.Millisecond, 10*time.Millisecond
|
||||
t.Cleanup(func() { AnswerWaitsAtMost, AnswerLooksEvery = most, every })
|
||||
start := time.Now()
|
||||
out, err := WaitForAnswer(store, r, key, time.Hour, func() time.Time { return t0.Add(time.Minute) })
|
||||
if err != nil || out["state"] != AnswerWaiting {
|
||||
t.Fatalf("%v %v", out, err)
|
||||
}
|
||||
if took := time.Since(start); took > 2*time.Second {
|
||||
t.Errorf("a read asked to wait an hour took %s", took)
|
||||
}
|
||||
}
|
||||
|
||||
// memQuestions is the `questions` state: values with revisions, create-only and compare-and-set, safe for the
|
||||
// goroutines a test starts.
|
||||
type memQuestions struct {
|
||||
mu sync.Mutex
|
||||
m map[string]json.RawMessage
|
||||
rev map[string]uint64
|
||||
n uint64
|
||||
// failGet fails every read of a key with this prefix.
|
||||
failGet string
|
||||
// afterGet runs after a read, outside the lock: another node writing between a read and a write.
|
||||
afterGet func(key string)
|
||||
}
|
||||
|
||||
func newMemQuestions() *memQuestions {
|
||||
return &memQuestions{m: map[string]json.RawMessage{}, rev: map[string]uint64{}}
|
||||
}
|
||||
|
||||
func (s *memQuestions) Get(key string) (json.RawMessage, uint64, bool, error) {
|
||||
s.mu.Lock()
|
||||
if s.failGet != "" && strings.HasPrefix(key, s.failGet) {
|
||||
s.mu.Unlock()
|
||||
return nil, 0, false, errors.New("the bus did not answer")
|
||||
}
|
||||
raw, ok := s.m[key]
|
||||
rev, after := s.rev[key], s.afterGet
|
||||
s.mu.Unlock()
|
||||
if after != nil {
|
||||
after(key)
|
||||
}
|
||||
return raw, rev, ok, nil
|
||||
}
|
||||
func (s *memQuestions) put(key string, value any) error {
|
||||
raw, err := json.Marshal(value)
|
||||
s.n++
|
||||
s.m[key], s.rev[key] = raw, s.n
|
||||
return err
|
||||
}
|
||||
func (s *memQuestions) Create(key string, value any) error {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if _, ok := s.m[key]; ok {
|
||||
return stdio.ErrChanged
|
||||
}
|
||||
return s.put(key, value)
|
||||
}
|
||||
func (s *memQuestions) Update(key string, value any, revision uint64) error {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if s.rev[key] != revision {
|
||||
return stdio.ErrChanged
|
||||
}
|
||||
return s.put(key, value)
|
||||
}
|
||||
func (s *memQuestions) Put(key string, value any) error {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
return s.put(key, value)
|
||||
}
|
||||
|
||||
// The edges of ADR 0332 §2 that are allowed: a 60-character headline and 1430 minutes.
|
||||
func TestTheEdgesOfAQuestionAreAsked(t *testing.T) {
|
||||
p, _ := node(t, "g14")
|
||||
in := mergeTonight()
|
||||
in.Headline = strings.Repeat("x", 59) + "?"
|
||||
in.Expires = 1430 * time.Minute
|
||||
if _, err := AskQuestion(p, in, newMemQuestions(), newRouter(), t0); err != nil {
|
||||
t.Fatalf("a 60-character headline lasting 1430 minutes: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// A warrant naming another asker or another ask, or an option the question did not offer, is no answer to it.
|
||||
func TestAWarrantForAnotherAskOrOptionIsNotProven(t *testing.T) {
|
||||
p, _ := node(t, "g14")
|
||||
for name, change := range map[string]func(*asks.Warrant){
|
||||
"another asker": func(w *asks.Warrant) { w.Asker = "mesh-controller" },
|
||||
"another ask": func(w *asks.Warrant) { w.Ask = "q-0000000000000000" },
|
||||
"an option unoffered": func(w *asks.Warrant) { w.Option, w.Label = "o9", "Delete everything" },
|
||||
} {
|
||||
store, r := newMemQuestions(), newRouter()
|
||||
key := askMergeTonight(t, p, store, r, t0)
|
||||
w := r.chose(key, "o1", t0.Add(time.Minute))
|
||||
change(&w)
|
||||
r.records[key], _ = json.Marshal(map[string]any{"state": "chosen", "warrant": w})
|
||||
if out, _ := ReadAnswer(store, r, key, t0.Add(2*time.Minute)); out["state"] != AnswerNotProven {
|
||||
t.Errorf("%s read as %v", name, out)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Two nodes asking at the same moment take two slots at most: of three sessions racing, one is refused.
|
||||
func TestTheCapHoldsAcrossNodesAskingAtOnce(t *testing.T) {
|
||||
store, r := newMemQuestions(), &lockedRouter{router: newRouter()}
|
||||
nodes := []string{"g14", "shanks", "ace"}
|
||||
var wg sync.WaitGroup
|
||||
errs := make([]error, len(nodes))
|
||||
for i, name := range nodes {
|
||||
p, _ := node(t, name)
|
||||
wg.Add(1)
|
||||
go func(i int, p Paths) {
|
||||
defer wg.Done()
|
||||
in := mergeTonight()
|
||||
in.By = "fix-589"
|
||||
_, errs[i] = AskQuestion(p, in, store, r, t0)
|
||||
}(i, p)
|
||||
}
|
||||
wg.Wait()
|
||||
refused := 0
|
||||
for _, err := range errs {
|
||||
if err != nil {
|
||||
if !strings.Contains(err.Error(), "questions already wait") {
|
||||
t.Errorf("refused for another reason: %v", err)
|
||||
}
|
||||
refused++
|
||||
}
|
||||
}
|
||||
if refused != 1 {
|
||||
t.Errorf("%d of three questions asked at once were refused; one is", refused)
|
||||
}
|
||||
}
|
||||
|
||||
// A slot that cannot be read refuses the question: the cap fails closed.
|
||||
func TestTheCapFailsClosed(t *testing.T) {
|
||||
p, _ := node(t, "g14")
|
||||
store, r := newMemQuestions(), newRouter()
|
||||
store.failGet = "slot-"
|
||||
if _, err := AskQuestion(p, mergeTonight(), store, r, t0); err == nil || !strings.Contains(err.Error(), "could not be counted") {
|
||||
t.Fatalf("asked with the slots unreadable: %v", err)
|
||||
}
|
||||
if len(r.subjects) != 0 {
|
||||
t.Error("the operator was asked anyway")
|
||||
}
|
||||
}
|
||||
|
||||
// A question whose refusal was not heard here, and of which the router holds no record, keeps its slot and reads as
|
||||
// waiting until it expires — the router may still take it from its queue — and only then frees its slot and ends.
|
||||
func TestAQuestionTheRouterHasNotRecordedHoldsItsSlotUntilItExpires(t *testing.T) {
|
||||
p, _ := node(t, "g14")
|
||||
store, r := newMemQuestions(), newRouter()
|
||||
first := askMergeTonight(t, p, store, r, t0)
|
||||
askMergeTonight(t, p, store, r, t0)
|
||||
delete(r.records, first) // the router down or backlogged, or its refusal never heard here
|
||||
later := t0.Add(QuestionDefault - time.Minute)
|
||||
if _, err := AskQuestion(p, mergeTonight(), store, r, later); err == nil {
|
||||
t.Fatal("the slot of a question the router may still show was freed before it expired")
|
||||
}
|
||||
out, _ := ReadAnswer(store, r, first, later)
|
||||
if out["state"] != AnswerWaiting || !strings.Contains(out["words"].(string), "not yet recorded") {
|
||||
t.Errorf("before expiry read as %v", out)
|
||||
}
|
||||
expired := t0.Add(QuestionDefault)
|
||||
if out, _ := ReadAnswer(store, r, first, expired); out["state"] != AnswerEnded ||
|
||||
out["now"] != "do what you said: I leave the pull request open" {
|
||||
t.Errorf("at expiry read as %v", out)
|
||||
}
|
||||
if _, err := AskQuestion(p, mergeTonight(), store, r, expired); err != nil {
|
||||
t.Errorf("the slot of an expired question was not freed: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// A slot is taken only at the revision read: when another node takes both slots between this node's read and its
|
||||
// write, this node's writes are refused and the question is refused — a plain write would take a slot it lost.
|
||||
func TestASlotIsTakenOnlyAtTheRevisionRead(t *testing.T) {
|
||||
p, _ := node(t, "g14")
|
||||
store, r := newMemQuestions(), newRouter()
|
||||
// Two questions that ended: their slots are free to take again.
|
||||
for _, id := range []string{askMergeTonight(t, p, store, r, t0), askMergeTonight(t, p, store, r, t0)} {
|
||||
r.close(asks.Warrant{Ask: id, Outcome: asks.OutcomeExpired, At: t0.Add(time.Minute)})
|
||||
}
|
||||
shanks, _ := node(t, "shanks")
|
||||
other := newRouter()
|
||||
store.afterGet = func(key string) {
|
||||
if !strings.HasPrefix(key, "slot-") {
|
||||
return
|
||||
}
|
||||
// Another node asks between this node's read of the slot and its write: its question waits.
|
||||
q := Question{ID: newQuestionID(), Headline: "Another", Context: "c", Why: "w", OnExpiry: "e", By: "other",
|
||||
From: shanks.Node, At: t0, Expires: t0.Add(time.Hour),
|
||||
Options: []QuestionOption{{ID: "o1", Label: "A", Means: "a", Recommended: true}, {ID: "o2", Label: "B", Means: "b"}}}
|
||||
_ = store.Put(q.ID, q)
|
||||
_ = other.Publish(asks.AskSubject(Module), q.Ask(), q.ID)
|
||||
r.records[q.ID] = other.records[q.ID]
|
||||
_ = store.Put(key, slot{Question: q.ID, At: t0})
|
||||
}
|
||||
_, err := AskQuestion(p, mergeTonight(), store, r, t0.Add(2*time.Minute))
|
||||
store.afterGet = nil
|
||||
if err == nil || !strings.Contains(err.Error(), "questions already wait") {
|
||||
t.Fatalf("a slot lost to another node was taken: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// lockedRouter is the router fake, safe for sessions asking at once.
|
||||
type lockedRouter struct {
|
||||
mu sync.Mutex
|
||||
*router
|
||||
}
|
||||
|
||||
func (l *lockedRouter) Publish(subject string, body any, id string) error {
|
||||
l.mu.Lock()
|
||||
defer l.mu.Unlock()
|
||||
return l.router.Publish(subject, body, id)
|
||||
}
|
||||
func (l *lockedRouter) Record(id string) (*stdio.StateEntry, error) {
|
||||
l.mu.Lock()
|
||||
defer l.mu.Unlock()
|
||||
return l.router.Record(id)
|
||||
}
|
||||
@@ -18,7 +18,8 @@
|
||||
"servers",
|
||||
"holdings",
|
||||
"config",
|
||||
"proposals"
|
||||
"proposals",
|
||||
"questions"
|
||||
],
|
||||
"reads": [
|
||||
"claude-licence-manager.bindings"
|
||||
@@ -54,6 +55,8 @@
|
||||
"claude_code_instructions_unregister",
|
||||
"claude_code_instructions_propose",
|
||||
"claude_code_proposals",
|
||||
"claude_code_ask",
|
||||
"claude_code_answer",
|
||||
"claude_code_settings_get",
|
||||
"claude_code_settings_set",
|
||||
"claude_code_settings_clear",
|
||||
|
||||
@@ -17,6 +17,7 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"net"
|
||||
"net/netip"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
@@ -147,6 +148,8 @@ type JailSettings struct {
|
||||
// Fail2ban is the daemon as this machine has it, through its own client.
|
||||
type Fail2ban struct {
|
||||
Run Runner
|
||||
// Root is where the machine's files are read from: "" on the machine, a directory in a test.
|
||||
Root string
|
||||
}
|
||||
|
||||
func (f Fail2ban) client(ctx context.Context, args ...string) (string, error) {
|
||||
@@ -229,6 +232,13 @@ type BanOutcome struct {
|
||||
}
|
||||
|
||||
// Ban bans one address in one jail now. The daemon's own answer is how many addresses it added.
|
||||
//
|
||||
// **Never one of the operator's own addresses, nor one the jail spares** (novox/hq ADR 0339 §3, narrowing
|
||||
// ADR 0179 §4): an address on the setting `own-addresses`, or on any entry of the jail's own ignore list — the
|
||||
// private ranges ADR 0186 spares among them — is refused, naming the entry it lies on, and nothing reaches the
|
||||
// daemon. fail2ban's `banip` bans whatever it is given, ignore list or not: the 2026-10-04 lockout of the
|
||||
// operator's home (issue 238) needed only a jail; this verb could have done it in one call. When the setting
|
||||
// cannot be read the ban is refused too: whether the address is the operator's cannot then be told.
|
||||
func (f Fail2ban) Ban(ctx context.Context, ip, jail string) (*BanOutcome, error) {
|
||||
if err := address(ip); err != nil {
|
||||
return nil, err
|
||||
@@ -236,6 +246,9 @@ func (f Fail2ban) Ban(ctx context.Context, ip, jail string) (*BanOutcome, error)
|
||||
if err := jailName(jail); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := f.spared(ctx, ip, jail); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out, err := f.client(ctx, "set", jail, "banip", ip)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -255,6 +268,38 @@ func (f Fail2ban) Ban(ctx context.Context, ip, jail string) (*BanOutcome, error)
|
||||
return outcome, nil
|
||||
}
|
||||
|
||||
// spared refuses an address the operator owns or the jail ignores, saying which entry it lies on.
|
||||
func (f Fail2ban) spared(ctx context.Context, ip, jail string) error {
|
||||
a, err := netip.ParseAddr(ip)
|
||||
if err != nil {
|
||||
return fmt.Errorf("%q is not an address", ip)
|
||||
}
|
||||
own, err := ReadOwn(f.Root)
|
||||
if err != nil {
|
||||
return fmt.Errorf("refused, and nothing was banned: %v, so whether %s is one of them cannot be told", err, ip)
|
||||
}
|
||||
if on := own.Lies(a); on != "" {
|
||||
return fmt.Errorf("refused, and nothing was banned: %s lies on %s, one of the operator's own addresses "+
|
||||
"(the setting %s), which no jail bans (novox/hq ADR 0339)", ip, on, SettingKey)
|
||||
}
|
||||
out, err := f.client(ctx, "get", jail, "ignoreip")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var entries []netip.Prefix
|
||||
for _, word := range listed(out) {
|
||||
// A file of entries (the operator's own, read above) or a name the daemon resolves is not an entry here.
|
||||
if p, err := entry(word); err == nil {
|
||||
entries = append(entries, p)
|
||||
}
|
||||
}
|
||||
if on := liesOn(a, entries); on != "" {
|
||||
return fmt.Errorf("refused, and nothing was banned: %s lies on %s, which the jail %s never bans — the "+
|
||||
"machine itself, the mesh's own range or a private one (novox/hq ADR 0186, ADR 0339)", ip, on, jail)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Released is how many bans the daemon let go, of which address, from where.
|
||||
type Released struct {
|
||||
Released int `json:"released"`
|
||||
|
||||
@@ -108,7 +108,8 @@ func TestBannedGathersEveryJailsBansSoonestToEndFirst(t *testing.T) {
|
||||
|
||||
func TestBanAsksByJailAndAnswersTheBanAsHeldRefusingANonAddressFirst(t *testing.T) {
|
||||
var calls [][]string
|
||||
f := Fail2ban{Run: fake(map[string]string{
|
||||
f := Fail2ban{Root: t.TempDir(), Run: fake(map[string]string{
|
||||
"get recidive ignoreip": ignoring,
|
||||
"set recidive banip 198.51.100.7": "1\n",
|
||||
"get recidive banip --with-time": withTime + "198.51.100.7 \t2026-10-02 17:00:00 + 604800 = 2026-10-09 17:00:00\n",
|
||||
}, &calls)}
|
||||
@@ -119,8 +120,8 @@ func TestBanAsksByJailAndAnswersTheBanAsHeldRefusingANonAddressFirst(t *testing.
|
||||
if r.Added != 1 || r.Banned == nil || r.Banned.Until != "2026-10-09 17:00:00" {
|
||||
t.Errorf("%+v", r)
|
||||
}
|
||||
if !reflect.DeepEqual(calls[0], []string{"fail2ban-client", "set", "recidive", "banip", "198.51.100.7"}) {
|
||||
t.Errorf("first call %v", calls[0])
|
||||
if !reflect.DeepEqual(calls[1], []string{"fail2ban-client", "set", "recidive", "banip", "198.51.100.7"}) {
|
||||
t.Errorf("the ban %v", calls)
|
||||
}
|
||||
if _, err := f.Ban(ctx, "not-an-ip", "recidive"); err == nil || !strings.Contains(err.Error(), "is not an address") {
|
||||
t.Errorf("a non-address: %v", err)
|
||||
@@ -128,7 +129,7 @@ func TestBanAsksByJailAndAnswersTheBanAsHeldRefusingANonAddressFirst(t *testing.
|
||||
if _, err := f.Ban(ctx, "198.51.100.7", "a jail; rm"); err == nil || !strings.Contains(err.Error(), "is not a jail's name") {
|
||||
t.Errorf("a non-name: %v", err)
|
||||
}
|
||||
if len(calls) != 2 {
|
||||
if len(calls) != 3 {
|
||||
t.Errorf("a refused ban reached the daemon: %v", calls)
|
||||
}
|
||||
}
|
||||
@@ -189,7 +190,7 @@ func TestTheSeatsVerbsAndTheModulesOwnToolAreServed(t *testing.T) {
|
||||
names = append(names, tool.Name)
|
||||
}
|
||||
want := []string{"node-intrusion-prevention.status", "node-intrusion-prevention.banned", "node-intrusion-prevention.ban",
|
||||
"node-intrusion-prevention.unban", "fail2ban_settings"}
|
||||
"node-intrusion-prevention.unban", "fail2ban_settings", "fail2ban_health"}
|
||||
if !reflect.DeepEqual(names, want) {
|
||||
t.Errorf("%v", names)
|
||||
}
|
||||
|
||||
@@ -5,6 +5,10 @@
|
||||
// by the mesh from the modules a machine runs and written as declared resources; these touch only
|
||||
// what the running daemon holds.
|
||||
//
|
||||
// Started as `fail2ban-tools watch` it is the module's long-running process, as root (watch.go, novox/hq
|
||||
// ADR 0339): it writes the operator's own addresses into the list every jail ignores, checks that every jail
|
||||
// names that list, and keeps every failure a jail ignored from one of them, for the health tool to say.
|
||||
//
|
||||
// stdout is the MCP channel; everything this module says, it says on stderr.
|
||||
package main
|
||||
|
||||
@@ -12,7 +16,10 @@ import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/signal"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
)
|
||||
@@ -21,6 +28,20 @@ import (
|
||||
const Seat = "node-intrusion-prevention"
|
||||
|
||||
func main() {
|
||||
if len(os.Args) > 1 {
|
||||
if os.Args[1] != "watch" || len(os.Args) != 2 {
|
||||
fmt.Fprintln(os.Stderr, "usage: fail2ban-tools [watch]")
|
||||
os.Exit(2)
|
||||
}
|
||||
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGTERM, os.Interrupt)
|
||||
defer stop()
|
||||
w := &Watcher{F: Fail2ban{Run: execRunner}, Now: time.Now}
|
||||
if err := w.Watch(ctx); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "[fail2ban] %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
return
|
||||
}
|
||||
if err := stdio.Serve("", tools(Fail2ban{Run: execRunner})); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "[fail2ban] %v\n", err)
|
||||
os.Exit(1)
|
||||
@@ -60,5 +81,9 @@ func tools(f Fail2ban) []stdio.Tool {
|
||||
Description: "One jail's effective settings on this machine: ban time, window, tries, the addresses it never bans, its actions and what it reads.",
|
||||
Input: map[string]any{"jail": str("the jail")},
|
||||
Run: func(a map[string]any) (any, error) { return f.Settings(ctx, arg(a, "jail")) }},
|
||||
{Name: "fail2ban_health",
|
||||
Description: "Whether every jail on this machine ignores the operator's own addresses (the mesh-wide setting own-addresses, novox/hq ADR 0339) and none of them failed in the last day: each jail and address with the failures it ignored, how many and when the last was, and what the module's watch last saw. Not healthy while one did, so the mesh raises it as a warning; it clears a day after the last.",
|
||||
Input: map[string]any{},
|
||||
Run: func(map[string]any) (any, error) { return ReadHealth(f.Root, time.Now()), nil }},
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,186 @@
|
||||
package main
|
||||
|
||||
// The operator's own addresses (novox/hq ADR 0339, issue 238): the public addresses and ranges the operator
|
||||
// names as theirs, in the mesh-wide setting `own-addresses`. Every jail ignores them beside the private ranges
|
||||
// ADR 0186 spares, the ban verb refuses them, and a failure a jail counted from one is said as a warning.
|
||||
//
|
||||
// **The setting reaches the machine as a mergeable file, so it may be unset.** A `${setting:own-addresses}`
|
||||
// would leave the module out of every machine until the operator gave a value (ADR 0262: an operator's value
|
||||
// has no default), and an empty list is what the record says the mesh starts from. The mesh writes the
|
||||
// setting into SettingFile, whose own content is the empty list; the watch, as root, writes the entries
|
||||
// into ListFile, one per line, and every jail reads that through `ignoreip = … file:<ListFile>`, which fail2ban
|
||||
// (1.1 and later) reloads within a second of the file changing — no restart, no ban lost. A failure ignored
|
||||
// because of it is logged as `Ignore <address> by file:<ListFile>`, which is how the watch tells the
|
||||
// operator's own addresses from the private ranges, which are logged `by ip`.
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"net/netip"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// The files, as the machine has them.
|
||||
const (
|
||||
// SettingFile is the setting as the mesh writes it: {"own-addresses": […]} or a string of entries.
|
||||
SettingFile = "/etc/fail2ban/mesh-own-addresses.json"
|
||||
// StateDir is the module's own directory, where the watch writes.
|
||||
StateDir = "/var/lib/mesh-fail2ban"
|
||||
// ListFile is the entries one per line, as fail2ban reads them.
|
||||
ListFile = StateDir + "/own-addresses"
|
||||
// StateFile is what the watch saw, for the health tool to read.
|
||||
StateFile = StateDir + "/watch.json"
|
||||
// LogFile is the daemon's log, where it says what it ignored.
|
||||
LogFile = "/var/log/fail2ban.log"
|
||||
// SettingKey is the setting's name (ADR 0339 §1).
|
||||
SettingKey = "own-addresses"
|
||||
// IgnoredBy is how every jail's ignore list names the operator's own addresses, and how the daemon's log
|
||||
// names them as the reason it ignored a failure.
|
||||
IgnoredBy = "file:" + ListFile
|
||||
)
|
||||
|
||||
// Own is the setting read: its entries, and each one that is not an address or a range, with why.
|
||||
type Own struct {
|
||||
Entries []netip.Prefix `json:"-"`
|
||||
Listed []string `json:"entries"`
|
||||
Problems []string `json:"problems,omitempty"`
|
||||
}
|
||||
|
||||
// ReadOwn reads the setting from SettingFile under root. A file not written yet is no entry: the mesh has
|
||||
// not written the setting there, and the record starts from none. Any other failure to read it is an error,
|
||||
// so a caller that refuses on it fails closed.
|
||||
func ReadOwn(root string) (Own, error) {
|
||||
raw, err := os.ReadFile(filepath.Join(root, SettingFile))
|
||||
if errors.Is(err, fs.ErrNotExist) {
|
||||
return Own{Listed: []string{}}, nil
|
||||
}
|
||||
if err != nil {
|
||||
return Own{}, fmt.Errorf("the operator's own addresses cannot be read from %s: %w", SettingFile, err)
|
||||
}
|
||||
var doc map[string]any
|
||||
if err := json.Unmarshal(raw, &doc); err != nil {
|
||||
return Own{}, fmt.Errorf("the operator's own addresses in %s are not JSON: %w", SettingFile, err)
|
||||
}
|
||||
return ParseOwn(doc[SettingKey])
|
||||
}
|
||||
|
||||
// ParseOwn reads the setting's value: a list of entries, or one string of entries separated by spaces or
|
||||
// commas, as `settings set` may give either. Each entry is an address or a range in either family; anything
|
||||
// else is a problem, named, and left out — a typo never widens what is spared.
|
||||
func ParseOwn(value any) (Own, error) {
|
||||
var words []string
|
||||
switch v := value.(type) {
|
||||
case nil:
|
||||
case string:
|
||||
words = strings.FieldsFunc(v, func(r rune) bool { return r == ',' || r == ' ' || r == '\t' || r == '\n' })
|
||||
case []any:
|
||||
for _, item := range v {
|
||||
s, ok := item.(string)
|
||||
if !ok {
|
||||
return Own{}, fmt.Errorf("%s holds %v, which is not text: each entry is an address or a range", SettingKey, item)
|
||||
}
|
||||
words = append(words, strings.TrimSpace(s))
|
||||
}
|
||||
default:
|
||||
return Own{}, fmt.Errorf("%s is %T: it is a list of addresses and ranges, or one line of them", SettingKey, value)
|
||||
}
|
||||
own := Own{Listed: []string{}}
|
||||
seen := map[netip.Prefix]bool{}
|
||||
for _, w := range words {
|
||||
if w == "" {
|
||||
continue
|
||||
}
|
||||
p, err := entry(w)
|
||||
if err != nil {
|
||||
own.Problems = append(own.Problems, err.Error())
|
||||
continue
|
||||
}
|
||||
if seen[p] {
|
||||
continue
|
||||
}
|
||||
seen[p] = true
|
||||
own.Entries = append(own.Entries, p)
|
||||
}
|
||||
sort.Slice(own.Entries, func(i, j int) bool { return own.Entries[i].String() < own.Entries[j].String() })
|
||||
for _, p := range own.Entries {
|
||||
own.Listed = append(own.Listed, show(p))
|
||||
}
|
||||
return own, nil
|
||||
}
|
||||
|
||||
// entry is one address or range; an address is the range of itself.
|
||||
func entry(word string) (netip.Prefix, error) {
|
||||
if strings.Contains(word, "/") {
|
||||
p, err := netip.ParsePrefix(word)
|
||||
if err != nil {
|
||||
return netip.Prefix{}, fmt.Errorf("%q is not an address or a range", word)
|
||||
}
|
||||
return p.Masked(), nil
|
||||
}
|
||||
a, err := netip.ParseAddr(word)
|
||||
if err != nil {
|
||||
return netip.Prefix{}, fmt.Errorf("%q is not an address or a range", word)
|
||||
}
|
||||
a = a.Unmap()
|
||||
return netip.PrefixFrom(a, a.BitLen()), nil
|
||||
}
|
||||
|
||||
// show is an entry as a person reads it: a single address without its length.
|
||||
func show(p netip.Prefix) string {
|
||||
if p.IsSingleIP() {
|
||||
return p.Addr().String()
|
||||
}
|
||||
return p.String()
|
||||
}
|
||||
|
||||
// Lies is the entry ip lies on, or "".
|
||||
func (o Own) Lies(ip netip.Addr) string {
|
||||
return liesOn(ip, o.Entries)
|
||||
}
|
||||
|
||||
func liesOn(ip netip.Addr, entries []netip.Prefix) string {
|
||||
ip = ip.Unmap()
|
||||
for _, p := range entries {
|
||||
if p.Contains(ip) {
|
||||
return show(p)
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// ListText is the entries as fail2ban reads ListFile: one per line, under a line saying whose it is.
|
||||
func (o Own) ListText() string {
|
||||
var b strings.Builder
|
||||
b.WriteString("# The operator's own addresses (novox/hq ADR 0339), written by the fail2ban module's watch from the\n")
|
||||
b.WriteString("# mesh-wide setting own-addresses. Every jail ignores them. Replaced on every change; edit nothing here.\n")
|
||||
for _, l := range o.Listed {
|
||||
b.WriteString(l + "\n")
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// WriteList writes ListFile under root when its content differs, by a rename so the daemon never reads half
|
||||
// of it. Says whether it wrote.
|
||||
func WriteList(root string, o Own) (bool, error) {
|
||||
path := filepath.Join(root, ListFile)
|
||||
want := o.ListText()
|
||||
if have, err := os.ReadFile(path); err == nil && string(have) == want {
|
||||
return false, nil
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
|
||||
return false, err
|
||||
}
|
||||
tmp := path + ".new"
|
||||
if err := os.WriteFile(tmp, []byte(want), 0o644); err != nil {
|
||||
return false, err
|
||||
}
|
||||
if err := os.Rename(tmp, path); err != nil {
|
||||
return false, err
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
@@ -0,0 +1,305 @@
|
||||
package main
|
||||
|
||||
// The operator's own addresses (novox/hq ADR 0339, issue 238): the setting read, every jail's ignore list
|
||||
// naming it, the ban verb refusing them, the watch keeping what a jail ignored, and the health saying it.
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/netip"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// ignoring is a jail's ignore list as fail2ban-client 1.1 prints it, with the base the module writes.
|
||||
const ignoring = "These IP addresses/networks are ignored:\n|- 127.0.0.0/8\n|- 10.10.0.0/24\n|- 10.0.0.0/8\n" +
|
||||
"|- 172.16.0.0/12\n|- 192.168.0.0/16\n|- 169.254.0.0/16\n|- fc00::/7\n|- fe80::/10\n" +
|
||||
"|- file:/var/lib/mesh-fail2ban/own-addresses\n`- ::1\n"
|
||||
|
||||
// home is the operator's home uplink in these tests (RFC 5737).
|
||||
const home = "203.0.113.5"
|
||||
|
||||
func writeSetting(t *testing.T, root, content string) {
|
||||
t.Helper()
|
||||
path := filepath.Join(root, SettingFile)
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(path, []byte(content), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheSettingIsAListOrALineAndAnEntryThatIsNoAddressIsLeftOutAndSaid(t *testing.T) {
|
||||
list, err := ParseOwn([]any{"203.0.113.5", "198.51.100.0/24", "2001:db8::/32", "203.0.113.5", "not-an-address"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !reflect.DeepEqual(list.Listed, []string{"198.51.100.0/24", "2001:db8::/32", "203.0.113.5"}) {
|
||||
t.Errorf("listed %v", list.Listed)
|
||||
}
|
||||
if len(list.Problems) != 1 || !strings.Contains(list.Problems[0], "not-an-address") {
|
||||
t.Errorf("problems %v", list.Problems)
|
||||
}
|
||||
line, err := ParseOwn("203.0.113.5, 198.51.100.7")
|
||||
if err != nil || !reflect.DeepEqual(line.Listed, []string{"198.51.100.7", "203.0.113.5"}) {
|
||||
t.Errorf("a line: %v %v", line.Listed, err)
|
||||
}
|
||||
if on := list.Lies(netip.MustParseAddr("198.51.100.77")); on != "198.51.100.0/24" {
|
||||
t.Errorf("an address in a range lies on %q", on)
|
||||
}
|
||||
if on := list.Lies(netip.MustParseAddr("192.0.2.1")); on != "" {
|
||||
t.Errorf("a stranger lies on %q", on)
|
||||
}
|
||||
if _, err := ParseOwn(3.0); err == nil {
|
||||
t.Error("a number was taken as the setting")
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheSettingNotYetWrittenIsNoEntryAndAnUnreadableOneIsAnError(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
own, err := ReadOwn(root)
|
||||
if err != nil || len(own.Entries) != 0 {
|
||||
t.Fatalf("no file: %+v %v", own, err)
|
||||
}
|
||||
writeSetting(t, root, "{\n \"own-addresses\": []\n}\n")
|
||||
if own, err := ReadOwn(root); err != nil || len(own.Entries) != 0 {
|
||||
t.Fatalf("the module's own content: %+v %v", own, err)
|
||||
}
|
||||
writeSetting(t, root, "{not json")
|
||||
if _, err := ReadOwn(root); err == nil {
|
||||
t.Fatal("a broken file read as no entry")
|
||||
}
|
||||
}
|
||||
|
||||
// The verb of 2026-10-04's remedy, used the other way: fail2ban's banip bans what it is given, ignore list or not.
|
||||
func TestBanRefusesTheOperatorsOwnAddressAndWhatTheJailSparesBeforeTheDaemonIsAsked(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
writeSetting(t, root, `{"own-addresses": ["203.0.113.5", "198.51.100.0/24"]}`)
|
||||
var calls [][]string
|
||||
f := Fail2ban{Root: root, Run: fake(map[string]string{"get gitea ignoreip": ignoring}, &calls)}
|
||||
for ip, on := range map[string]string{home: home, "198.51.100.9": "198.51.100.0/24", "192.168.1.1": "192.168.0.0/16",
|
||||
"10.10.0.4": "10.10.0.0/24"} {
|
||||
_, err := f.Ban(ctx, ip, "gitea")
|
||||
if err == nil || !strings.Contains(err.Error(), "refused") || !strings.Contains(err.Error(), "lies on "+on) {
|
||||
t.Errorf("%s: %v", ip, err)
|
||||
}
|
||||
}
|
||||
for _, c := range calls {
|
||||
if len(c) > 2 && c[1] == "set" {
|
||||
t.Fatalf("a refused ban reached the daemon: %v", calls)
|
||||
}
|
||||
}
|
||||
if _, err := f.Ban(ctx, home, "gitea"); err == nil || !strings.Contains(err.Error(), "own-addresses") {
|
||||
t.Errorf("the refusal does not name the setting: %v", err)
|
||||
}
|
||||
writeSetting(t, root, "{broken")
|
||||
if _, err := f.Ban(ctx, "192.0.2.1", "gitea"); err == nil || !strings.Contains(err.Error(), "cannot be told") {
|
||||
t.Errorf("an unreadable setting let a ban through: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheWatchWritesTheListEveryJailIgnoresAndRewritesItOnlyOnAChange(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
own, _ := ParseOwn([]any{home})
|
||||
wrote, err := WriteList(root, own)
|
||||
if err != nil || !wrote {
|
||||
t.Fatalf("%v %v", wrote, err)
|
||||
}
|
||||
raw, _ := os.ReadFile(filepath.Join(root, ListFile))
|
||||
var entries []string
|
||||
for _, l := range strings.Split(string(raw), "\n") {
|
||||
if l != "" && !strings.HasPrefix(l, "#") {
|
||||
entries = append(entries, l)
|
||||
}
|
||||
}
|
||||
if !reflect.DeepEqual(entries, []string{home}) {
|
||||
t.Errorf("the list fail2ban reads: %q", raw)
|
||||
}
|
||||
if wrote, _ := WriteList(root, own); wrote {
|
||||
t.Error("the same list was written again")
|
||||
}
|
||||
none, _ := ParseOwn(nil)
|
||||
if wrote, _ := WriteList(root, none); !wrote {
|
||||
t.Error("an emptied setting did not empty the list")
|
||||
}
|
||||
}
|
||||
|
||||
// logAt is a line as fail2ban 1.1 logs a failure it ignored.
|
||||
func logAt(at time.Time, jail, ip, by string) string {
|
||||
return at.Format("2006-01-02 15:04:05") + ",123 fail2ban.filter [1234]: INFO [" + jail + "] Ignore " + ip + " by " + by + "\n"
|
||||
}
|
||||
|
||||
func appendLog(t *testing.T, root, text string) {
|
||||
t.Helper()
|
||||
path := filepath.Join(root, LogFile)
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f, err := os.OpenFile(path, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0o640)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer f.Close()
|
||||
if _, err := f.WriteString(text); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func daemon(missing bool) Runner {
|
||||
gitea := ignoring
|
||||
if missing {
|
||||
gitea = strings.Replace(ignoring, "|- file:/var/lib/mesh-fail2ban/own-addresses\n", "", 1)
|
||||
}
|
||||
return fake(map[string]string{"status": "Status\n|- Number of jail:\t2\n`- Jail list:\tgitea, recidive\n",
|
||||
"get gitea ignoreip": gitea, "get recidive ignoreip": ignoring}, nil)
|
||||
}
|
||||
|
||||
// The 2026-10-04 loop as it now runs: forty failures from the operator's home, ignored and never banned, and
|
||||
// said by the health as one jail, one address and a count; a private address ignored is not the operator's.
|
||||
func TestAFailureIgnoredFromTheOperatorsOwnAddressIsSaidUntilADayAfterTheLast(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
writeSetting(t, root, `{"own-addresses": ["203.0.113.5"]}`)
|
||||
now := time.Date(2026, 10, 13, 10, 0, 0, 0, time.Local)
|
||||
clock := now
|
||||
w := &Watcher{F: Fail2ban{Run: daemon(false)}, Root: root, Now: func() time.Time { return clock }}
|
||||
|
||||
appendLog(t, root, logAt(now.Add(-48*time.Hour), "gitea", home, IgnoredBy)+
|
||||
logAt(now.Add(-time.Minute), "sshd", "192.168.1.1", "ip"))
|
||||
s := w.Look(context.Background())
|
||||
if err := WriteState(root, s); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if h := ReadHealth(root, now); !h.Healthy || !strings.Contains(h.Why, "every jail (2)") {
|
||||
t.Fatalf("nothing of the operator's in the last day, yet: %+v", h)
|
||||
}
|
||||
if !reflect.DeepEqual(s.Own, []string{home}) {
|
||||
t.Errorf("own %v", s.Own)
|
||||
}
|
||||
|
||||
var loop strings.Builder
|
||||
for i := 0; i < 40; i++ {
|
||||
loop.WriteString(logAt(now.Add(time.Duration(i)*15*time.Second), "gitea", home, IgnoredBy))
|
||||
}
|
||||
appendLog(t, root, loop.String()[:loop.Len()-10]) // the daemon is still writing the last line
|
||||
clock = now.Add(10 * time.Minute)
|
||||
s = w.Look(context.Background())
|
||||
if len(s.Ignored) != 1 || s.Ignored[0].Count != 39 {
|
||||
t.Fatalf("a line still being written was counted, or the loop was not: %+v", s.Ignored)
|
||||
}
|
||||
appendLog(t, root, loop.String()[loop.Len()-10:])
|
||||
s = w.Look(context.Background())
|
||||
want := IgnoredFailures{Jail: "gitea", Address: home, Count: 40, First: now, Last: now.Add(39 * 15 * time.Second)}
|
||||
if len(s.Ignored) != 1 || s.Ignored[0] != want {
|
||||
t.Fatalf("ignored %+v", s.Ignored)
|
||||
}
|
||||
if err := WriteState(root, s); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
h := ReadHealth(root, clock)
|
||||
if h.Healthy || !strings.Contains(h.Why, "the jail gitea counted 40 failure(s) from 203.0.113.5") ||
|
||||
!strings.Contains(h.Why, "banned none") {
|
||||
t.Fatalf("the loop was not said: %+v", h)
|
||||
}
|
||||
|
||||
// A watch started again reads the log whole and still knows the last day.
|
||||
again := &Watcher{F: Fail2ban{Run: daemon(false)}, Root: root, Now: func() time.Time { return clock }}
|
||||
if s := again.Look(context.Background()); len(s.Ignored) != 1 || s.Ignored[0].Count != 40 {
|
||||
t.Errorf("after a start: %+v", s.Ignored)
|
||||
}
|
||||
|
||||
// A day after the last, it clears.
|
||||
clock = now.Add(39*15*time.Second + Remembered + time.Second)
|
||||
s = w.Look(context.Background())
|
||||
if err := WriteState(root, s); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if h := ReadHealth(root, clock); !h.Healthy {
|
||||
t.Errorf("a day after the last it still says: %+v", h)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAJailThatLeavesTheListOutABadEntryAndAStoppedWatchAreSaid(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
writeSetting(t, root, `{"own-addresses": "203.0.113.5 nonsense"}`)
|
||||
now := time.Date(2026, 10, 13, 10, 0, 0, 0, time.Local)
|
||||
appendLog(t, root, "")
|
||||
w := &Watcher{F: Fail2ban{Run: daemon(true)}, Root: root, Now: func() time.Time { return now }}
|
||||
if err := WriteState(root, w.Look(context.Background())); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
h := ReadHealth(root, now)
|
||||
if h.Healthy || !strings.Contains(h.Why, "the jail(s) gitea do not ignore") || !strings.Contains(h.Why, `"nonsense"`) {
|
||||
t.Errorf("%+v", h)
|
||||
}
|
||||
if h := ReadHealth(root, now.Add(StaleAfter+time.Second)); h.Healthy || !strings.Contains(h.Why, "not looking") {
|
||||
t.Errorf("a stopped watch: %+v", h)
|
||||
}
|
||||
if h := ReadHealth(t.TempDir(), now); h.Healthy {
|
||||
t.Errorf("no watch at all: %+v", h)
|
||||
}
|
||||
}
|
||||
|
||||
// The manifest: every jail's base ignore list names the list, the setting reaches the machine without a
|
||||
// default and only from the controller's terminal, and the watch is judged by the health tool.
|
||||
func TestTheManifestGivesEveryJailTheListAndTheWatchItsHealth(t *testing.T) {
|
||||
raw, err := os.ReadFile("../../module.json")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var m struct {
|
||||
Tools []string `json:"tools"`
|
||||
Resources []map[string]any `json:"resources"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &m); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
byID := map[string]map[string]any{}
|
||||
for _, r := range m.Resources {
|
||||
byID[r["id"].(string)] = r
|
||||
}
|
||||
base, _ := byID["jail-local"]["content"].(string)
|
||||
var ignoreip []string
|
||||
for _, l := range strings.Split(base, "\n") {
|
||||
if strings.HasPrefix(l, "ignoreip") {
|
||||
ignoreip = append(ignoreip, l)
|
||||
}
|
||||
}
|
||||
if len(ignoreip) != 1 || !strings.HasSuffix(ignoreip[0], " "+IgnoredBy) {
|
||||
t.Errorf("the base ignore list: %q", ignoreip)
|
||||
}
|
||||
for id, r := range byID {
|
||||
if content, _ := r["content"].(string); id != "jail-local" && strings.Contains(content, "ignoreip") {
|
||||
t.Errorf("%s sets an ignore list of its own, which leaves the operator's own addresses out", id)
|
||||
}
|
||||
}
|
||||
setting := byID["own-addresses"]
|
||||
if setting["path"] != SettingFile || setting["merge"] != "json" || setting["trusted"] != true ||
|
||||
!reflect.DeepEqual(setting["takes-settings"], []any{SettingKey}) {
|
||||
t.Errorf("the setting's file: %v", setting)
|
||||
}
|
||||
var content map[string]any
|
||||
if err := json.Unmarshal([]byte(setting["content"].(string)), &content); err != nil ||
|
||||
!reflect.DeepEqual(content, map[string]any{SettingKey: []any{}}) {
|
||||
t.Errorf("the setting starts from %v (%v)", content, err)
|
||||
}
|
||||
watch := byID["watch"]
|
||||
health, _ := watch["health"].(map[string]any)
|
||||
if watch["type"] != "process" || !reflect.DeepEqual(watch["run"], []any{"./fail2ban-tools", "watch"}) ||
|
||||
health["kind"] != "tool" || health["tool"] != "fail2ban_health" || watch["user"] != nil {
|
||||
t.Errorf("the watch: %v", watch)
|
||||
}
|
||||
if list := byID["own-addresses-list"]; list["path"] != ListFile || list["create-once"] != true {
|
||||
t.Errorf("the list is there, empty, before the daemon first reads it: %v", list)
|
||||
}
|
||||
if byID["state-dir"]["path"] != StateDir {
|
||||
t.Errorf("the watch's directory: %v", byID["state-dir"])
|
||||
}
|
||||
if !contains(m.Tools, "fail2ban_health") {
|
||||
t.Errorf("tools %v", m.Tools)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,297 @@
|
||||
package main
|
||||
|
||||
// The watch (novox/hq ADR 0339 §2 and §4, issue 238): the module's long-running process, as root. Every look it
|
||||
//
|
||||
// - writes the operator's own addresses from the setting into the list every jail ignores (own.go);
|
||||
// - asks the daemon whether every jail's ignore list names that list, so no jail leaves it out;
|
||||
// - reads what the daemon logged since the last look, and keeps every failure a jail ignored because the
|
||||
// address is one of the operator's own, for a day.
|
||||
//
|
||||
// What it saw goes to StateFile, which the health tool reads as the operator's account. A failure from one
|
||||
// of the operator's own addresses is never banned — that is the point — and never silent: the module's health
|
||||
// says it, so the controller raises it as a warning naming the node, the jail, the address and the count, and
|
||||
// it clears a day after the last one. A loop like the one of 2026-10-04 is then seen within minutes.
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// The watch's timing.
|
||||
var (
|
||||
// LookEvery is how often the watch looks.
|
||||
LookEvery = 30 * time.Second
|
||||
// Remembered is how long an ignored failure is said: a day, as the record has it.
|
||||
Remembered = 24 * time.Hour
|
||||
// StaleAfter is how old the watch's last look may be before the health says it stopped looking.
|
||||
StaleAfter = 3 * time.Minute
|
||||
// keptPerAddress bounds what is remembered of one address in one jail; the count says when it met it.
|
||||
keptPerAddress = 10000
|
||||
)
|
||||
|
||||
// IgnoredFailures is the failures one jail ignored from one of the operator's own addresses, in the last day.
|
||||
type IgnoredFailures struct {
|
||||
Jail string `json:"jail"`
|
||||
Address string `json:"address"`
|
||||
Count int `json:"count"`
|
||||
First time.Time `json:"first"`
|
||||
Last time.Time `json:"last"`
|
||||
// AtLeast says the count met what is remembered of one address and is a floor.
|
||||
AtLeast bool `json:"at-least,omitempty"`
|
||||
}
|
||||
|
||||
// State is what the watch saw at its last look.
|
||||
type State struct {
|
||||
Looked time.Time `json:"looked"`
|
||||
// Own is the operator's own addresses as written into the list, and Problems the entries left out.
|
||||
Own []string `json:"own"`
|
||||
Problems []string `json:"problems,omitempty"`
|
||||
// Unread is why something the watch reads could not be read at this look.
|
||||
Unread []string `json:"unread,omitempty"`
|
||||
// Jails is how many jails the daemon runs; Missing those whose ignore list does not name the list.
|
||||
Jails int `json:"jails"`
|
||||
Missing []string `json:"missing,omitempty"`
|
||||
// Ignored is every jail and address of the operator's own with a failure ignored in the last day.
|
||||
Ignored []IgnoredFailures `json:"ignored"`
|
||||
}
|
||||
|
||||
// ignoredLine is the daemon's line for a failure it ignored: `<time>,<ms> fail2ban.filter [<pid>]: INFO
|
||||
// [<jail>] Ignore <address> by <source>` (fail2ban 1.1, server/filter.py logIgnoreIp).
|
||||
var ignoredLine = regexp.MustCompile(`^(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}),\d+ .*?\]:\s+\S+\s+\[([^\]]+)\] Ignore (\S+) by (\S+)\s*$`)
|
||||
|
||||
// Watcher is the watch's memory between looks: where it is in the log and what it saw there.
|
||||
type Watcher struct {
|
||||
F Fail2ban
|
||||
Root string
|
||||
Now func() time.Time
|
||||
|
||||
logFile os.FileInfo
|
||||
offset int64
|
||||
partial string
|
||||
seen map[[2]string][]time.Time
|
||||
}
|
||||
|
||||
// Look is one look: the list written, the jails asked, the log read, the state answered.
|
||||
func (w *Watcher) Look(ctx context.Context) State {
|
||||
now := w.Now()
|
||||
s := State{Looked: now, Own: []string{}, Ignored: []IgnoredFailures{}}
|
||||
own, err := ReadOwn(w.Root)
|
||||
if err != nil {
|
||||
s.Unread = append(s.Unread, err.Error())
|
||||
} else {
|
||||
s.Own, s.Problems = own.Listed, own.Problems
|
||||
if _, err := WriteList(w.Root, own); err != nil {
|
||||
s.Unread = append(s.Unread, fmt.Sprintf("the list every jail ignores (%s) cannot be written: %v", ListFile, err))
|
||||
}
|
||||
}
|
||||
if jails, err := w.F.Jails(ctx); err != nil {
|
||||
s.Unread = append(s.Unread, fmt.Sprintf("the daemon cannot be asked for its jails: %v", err))
|
||||
} else {
|
||||
s.Jails = len(jails)
|
||||
for _, j := range jails {
|
||||
out, err := w.F.client(ctx, "get", j, "ignoreip")
|
||||
if err != nil {
|
||||
s.Unread = append(s.Unread, fmt.Sprintf("the jail %s's ignore list cannot be read: %v", j, err))
|
||||
continue
|
||||
}
|
||||
if !contains(listed(out), IgnoredBy) {
|
||||
s.Missing = append(s.Missing, j)
|
||||
}
|
||||
}
|
||||
}
|
||||
if err := w.readLog(now); err != nil {
|
||||
s.Unread = append(s.Unread, err.Error())
|
||||
}
|
||||
s.Ignored = w.ignored(now)
|
||||
return s
|
||||
}
|
||||
|
||||
func contains(list []string, want string) bool {
|
||||
for _, l := range list {
|
||||
if l == want {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// readLog reads what the daemon logged since the last look. A log rotated away is read from its start; the
|
||||
// first look reads the whole of it, so a watch started again still knows the last day.
|
||||
func (w *Watcher) readLog(now time.Time) error {
|
||||
path := filepath.Join(w.Root, LogFile)
|
||||
f, err := os.Open(path)
|
||||
if err != nil {
|
||||
return fmt.Errorf("the daemon's log %s cannot be read: %w", LogFile, err)
|
||||
}
|
||||
defer f.Close()
|
||||
info, err := f.Stat()
|
||||
if err != nil {
|
||||
return fmt.Errorf("the daemon's log %s cannot be read: %w", LogFile, err)
|
||||
}
|
||||
if w.logFile == nil || !os.SameFile(w.logFile, info) || info.Size() < w.offset {
|
||||
w.offset, w.partial = 0, ""
|
||||
}
|
||||
w.logFile = info
|
||||
if _, err := f.Seek(w.offset, io.SeekStart); err != nil {
|
||||
return fmt.Errorf("the daemon's log %s cannot be read: %w", LogFile, err)
|
||||
}
|
||||
r := bufio.NewReader(f)
|
||||
for {
|
||||
line, err := r.ReadString('\n')
|
||||
w.offset += int64(len(line))
|
||||
if err != nil {
|
||||
// A line the daemon is still writing is kept for the next look.
|
||||
w.partial += line
|
||||
break
|
||||
}
|
||||
w.take(w.partial+strings.TrimRight(line, "\n"), now)
|
||||
w.partial = ""
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// take keeps one line of the log when it is a failure ignored because of the operator's own addresses.
|
||||
func (w *Watcher) take(line string, now time.Time) {
|
||||
m := ignoredLine.FindStringSubmatch(line)
|
||||
if m == nil || m[4] != IgnoredBy {
|
||||
return
|
||||
}
|
||||
at, err := time.ParseInLocation("2006-01-02 15:04:05", m[1], time.Local)
|
||||
if err != nil || now.Sub(at) > Remembered {
|
||||
return
|
||||
}
|
||||
if w.seen == nil {
|
||||
w.seen = map[[2]string][]time.Time{}
|
||||
}
|
||||
key := [2]string{m[2], m[3]}
|
||||
times := append(w.seen[key], at)
|
||||
if len(times) > keptPerAddress {
|
||||
times = times[len(times)-keptPerAddress:]
|
||||
}
|
||||
w.seen[key] = times
|
||||
}
|
||||
|
||||
// ignored is what was seen in the last day, by jail and address; anything older is forgotten.
|
||||
func (w *Watcher) ignored(now time.Time) []IgnoredFailures {
|
||||
out := []IgnoredFailures{}
|
||||
for key, times := range w.seen {
|
||||
kept := times[:0]
|
||||
for _, at := range times {
|
||||
if now.Sub(at) <= Remembered {
|
||||
kept = append(kept, at)
|
||||
}
|
||||
}
|
||||
if len(kept) == 0 {
|
||||
delete(w.seen, key)
|
||||
continue
|
||||
}
|
||||
w.seen[key] = kept
|
||||
sort.Slice(kept, func(i, j int) bool { return kept[i].Before(kept[j]) })
|
||||
out = append(out, IgnoredFailures{Jail: key[0], Address: key[1], Count: len(kept), First: kept[0],
|
||||
Last: kept[len(kept)-1], AtLeast: len(kept) >= keptPerAddress})
|
||||
}
|
||||
sort.Slice(out, func(i, j int) bool {
|
||||
if out[i].Jail != out[j].Jail {
|
||||
return out[i].Jail < out[j].Jail
|
||||
}
|
||||
return out[i].Address < out[j].Address
|
||||
})
|
||||
return out
|
||||
}
|
||||
|
||||
// WriteState writes the state for the health tool, readable by the operator's account, by a rename.
|
||||
func WriteState(root string, s State) error {
|
||||
raw, err := json.MarshalIndent(s, "", " ")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
path := filepath.Join(root, StateFile)
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
|
||||
return err
|
||||
}
|
||||
tmp := path + ".new"
|
||||
if err := os.WriteFile(tmp, append(raw, '\n'), 0o644); err != nil {
|
||||
return err
|
||||
}
|
||||
return os.Rename(tmp, path)
|
||||
}
|
||||
|
||||
// Watch looks until it is stopped.
|
||||
func (w *Watcher) Watch(ctx context.Context) error {
|
||||
for {
|
||||
if err := WriteState(w.Root, w.Look(ctx)); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "[fail2ban] what the watch saw cannot be written to %s: %v\n", StateFile, err)
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil
|
||||
case <-time.After(LookEvery):
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Health is the module's health as the node-engine reads it (novox/hq ADR 0240): healthy, and why.
|
||||
type Health struct {
|
||||
Healthy bool `json:"healthy"`
|
||||
Why string `json:"why"`
|
||||
State *State `json:"seen,omitempty"`
|
||||
}
|
||||
|
||||
// ReadHealth judges what the watch last saw. Not healthy when the watch is not looking, when something it
|
||||
// reads cannot be read, when the setting holds an entry that is not an address, when a jail's ignore list
|
||||
// leaves the operator's own addresses out, and when a jail ignored a failure from one in the last day — each
|
||||
// said, so the controller's warning names the jail, the address and the count (ADR 0339 §4).
|
||||
func ReadHealth(root string, now time.Time) Health {
|
||||
raw, err := os.ReadFile(filepath.Join(root, StateFile))
|
||||
if err != nil {
|
||||
return Health{Why: fmt.Sprintf("the watch has written nothing this tool can read (%v): it is not looking", err)}
|
||||
}
|
||||
var s State
|
||||
if err := json.Unmarshal(raw, &s); err != nil {
|
||||
return Health{Why: fmt.Sprintf("what the watch wrote is not its state: %v", err)}
|
||||
}
|
||||
if age := now.Sub(s.Looked); age > StaleAfter {
|
||||
return Health{State: &s, Why: fmt.Sprintf("the watch last looked at %s, %s ago: it is not looking",
|
||||
s.Looked.Local().Format("2006-01-02 15:04:05"), age.Round(time.Second))}
|
||||
}
|
||||
var wrong []string
|
||||
wrong = append(wrong, s.Unread...)
|
||||
if len(s.Problems) > 0 {
|
||||
wrong = append(wrong, fmt.Sprintf("the setting %s holds what is not an address or a range, left out: %s",
|
||||
SettingKey, strings.Join(s.Problems, ", ")))
|
||||
}
|
||||
if len(s.Missing) > 0 {
|
||||
wrong = append(wrong, fmt.Sprintf("the jail(s) %s do not ignore the operator's own addresses: their ignore "+
|
||||
"list lacks %s (novox/hq ADR 0339 §2)", strings.Join(s.Missing, ", "), IgnoredBy))
|
||||
}
|
||||
for _, i := range s.Ignored {
|
||||
count := fmt.Sprint(i.Count)
|
||||
if i.AtLeast {
|
||||
count = "at least " + count
|
||||
}
|
||||
wrong = append(wrong, fmt.Sprintf("the jail %s counted %s failure(s) from %s, one of the operator's own "+
|
||||
"addresses, in the last day, the last at %s, and banned none (novox/hq ADR 0339); it clears a day after "+
|
||||
"the last", i.Jail, count, i.Address, i.Last.Local().Format("2006-01-02 15:04:05")))
|
||||
}
|
||||
if len(wrong) > 0 {
|
||||
return Health{State: &s, Why: strings.Join(wrong, "; ")}
|
||||
}
|
||||
return Health{Healthy: true, State: &s, Why: fmt.Sprintf("every jail (%d) ignores the operator's own addresses "+
|
||||
"(%d entr%s), and none failed in the last day", s.Jails, len(s.Own), plural(len(s.Own)))}
|
||||
}
|
||||
|
||||
func plural(n int) string {
|
||||
if n == 1 {
|
||||
return "y"
|
||||
}
|
||||
return "ies"
|
||||
}
|
||||
@@ -17,7 +17,8 @@
|
||||
}
|
||||
],
|
||||
"tools": [
|
||||
"fail2ban_settings"
|
||||
"fail2ban_settings",
|
||||
"fail2ban_health"
|
||||
],
|
||||
"jailing": {
|
||||
"into": "/etc/fail2ban/jail.d/mesh.conf",
|
||||
@@ -29,6 +30,20 @@
|
||||
"type": "package",
|
||||
"package": "fail2ban"
|
||||
},
|
||||
{
|
||||
"id": "state-dir",
|
||||
"type": "directory",
|
||||
"path": "/var/lib/mesh-fail2ban",
|
||||
"mode": "0755"
|
||||
},
|
||||
{
|
||||
"id": "own-addresses-list",
|
||||
"type": "file",
|
||||
"path": "/var/lib/mesh-fail2ban/own-addresses",
|
||||
"mode": "0644",
|
||||
"create-once": true,
|
||||
"content": ""
|
||||
},
|
||||
{
|
||||
"id": "jail-d",
|
||||
"type": "directory",
|
||||
@@ -58,7 +73,7 @@
|
||||
"type": "file",
|
||||
"path": "/etc/fail2ban/jail.local",
|
||||
"mode": "0644",
|
||||
"content": "[INCLUDES]\n\nbefore = paths-arch.conf\n\n[DEFAULT]\n\n# Never act on the machine itself or on a tunnel peer: the mesh's private range is\n# ${machine:mesh-range}, named here rather than written as a value the module cannot\n# know (novox/hq ADR 0112). Without this, fail2ban could ban the mesh's own nodes.\n# **A ban list never holds a neighbour.** The mesh's own range is named rather than written\n# (novox/hq ADR 0112), and every private range beside it: a source on one is somebody's own\n# network, not the internet. On a machine behind a router that reflects local traffic, every\n# client in the house arrives as the gateway's address — so one mistyped local request banned\n# 192.168.1.1 on the home server and would have cut the whole house off from it (ADR 0186).\nignoreip = 127.0.0.1/8 ::1 ${machine:mesh-range} 10.0.0.0/8 172.16.0.0/12 192.168.0.0/16 169.254.0.0/16 fc00::/7 fe80::/10\n\n# Three failures in a day ban for a day (novox/hq ADR 0179). The attackers this mesh sees pace\n# themselves at one try every ten minutes, under any ten-minute window; a day's window counts\n# them, and a day's ban costs a person who mistyped three times once, from one address, while\n# the mesh's own range is never banned at all.\nbantime = 1d\nfindtime = 1d\nmaxretry = 3\n\n# Ban through iptables, not through a firewall front-end the machine may not have. ufw is\n# installed on two of this mesh's machines and absent on the other two, and fail2ban finds out\n# only at ban time: the service reports healthy, the jail counts the attempt, the ban command\n# exits 127, and nothing is blocked. Proven on 2026-09-28 -- 'ufw: command not found' on a\n# machine the mesh reported as protected.\n#\n# The action below is this module's own, already used by the recidive jail on every machine\n# here, and it bans in DOCKER-USER as well as INPUT, so a container's published port is\n# covered too.\nbanaction = iptables-allports-dualchain\nbanaction_allports = iptables-allports-dualchain\n\n[sshd]\nenabled = true\nport = ssh\nlogpath = %(sshd_log)s\nbackend = %(sshd_backend)s\n"
|
||||
"content": "[INCLUDES]\n\nbefore = paths-arch.conf\n\n[DEFAULT]\n\n# Never act on the machine itself or on a tunnel peer: the mesh's private range is\n# ${machine:mesh-range}, named here rather than written as a value the module cannot\n# know (novox/hq ADR 0112). Without this, fail2ban could ban the mesh's own nodes.\n# **A ban list never holds a neighbour.** The mesh's own range is named rather than written\n# (novox/hq ADR 0112), and every private range beside it: a source on one is somebody's own\n# network, not the internet. On a machine behind a router that reflects local traffic, every\n# client in the house arrives as the gateway's address — so one mistyped local request banned\n# 192.168.1.1 on the home server and would have cut the whole house off from it (ADR 0186).\n#\n# **Nor the operator's own addresses** (novox/hq ADR 0339, issue 238): the public addresses the operator names\n# as theirs, in the mesh-wide setting own-addresses. An agent polling the forge from the operator's home banned\n# that home for four weeks on 2026-10-04, and nothing in the house reached the control-node until it was found\n# among 400 bans. The file is written by this module's watch from the setting, one entry per line; fail2ban\n# reads it again within a second of a change, so a new value needs no restart. Empty until the operator\n# fills the setting. A jail that sets its own ignoreip leaves all of this out; the watch says so.\nignoreip = 127.0.0.1/8 ::1 ${machine:mesh-range} 10.0.0.0/8 172.16.0.0/12 192.168.0.0/16 169.254.0.0/16 fc00::/7 fe80::/10 file:/var/lib/mesh-fail2ban/own-addresses\n\n# Three failures in a day ban for a day (novox/hq ADR 0179). The attackers this mesh sees pace\n# themselves at one try every ten minutes, under any ten-minute window; a day's window counts\n# them, and a day's ban costs a person who mistyped three times once, from one address, while\n# the mesh's own range is never banned at all.\nbantime = 1d\nfindtime = 1d\nmaxretry = 3\n\n# Ban through iptables, not through a firewall front-end the machine may not have. ufw is\n# installed on two of this mesh's machines and absent on the other two, and fail2ban finds out\n# only at ban time: the service reports healthy, the jail counts the attempt, the ban command\n# exits 127, and nothing is blocked. Proven on 2026-09-28 -- 'ufw: command not found' on a\n# machine the mesh reported as protected.\n#\n# The action below is this module's own, already used by the recidive jail on every machine\n# here, and it bans in DOCKER-USER as well as INPUT, so a container's published port is\n# covered too.\nbanaction = iptables-allports-dualchain\nbanaction_allports = iptables-allports-dualchain\n\n[sshd]\nenabled = true\nport = ssh\nlogpath = %(sshd_log)s\nbackend = %(sshd_backend)s\n"
|
||||
},
|
||||
{
|
||||
"id": "jail-sshd",
|
||||
@@ -67,6 +82,36 @@
|
||||
"mode": "0644",
|
||||
"content": "[sshd]\nenabled = true\nport = ssh\nlogpath = %(sshd_log)s\nbackend = %(sshd_backend)s\nmaxretry = 3\nfindtime = 1d\nbantime = 1d\n"
|
||||
},
|
||||
{
|
||||
"id": "own-addresses",
|
||||
"type": "file",
|
||||
"path": "/etc/fail2ban/mesh-own-addresses.json",
|
||||
"mode": "0644",
|
||||
"merge": "json",
|
||||
"takes-settings": [
|
||||
"own-addresses"
|
||||
],
|
||||
"trusted": true,
|
||||
"content": "{\n \"own-addresses\": []\n}\n"
|
||||
},
|
||||
{
|
||||
"id": "watch",
|
||||
"type": "process",
|
||||
"name": "fail2ban-watch",
|
||||
"artifact": "tools",
|
||||
"run": [
|
||||
"./fail2ban-tools",
|
||||
"watch"
|
||||
],
|
||||
"health": {
|
||||
"kind": "tool",
|
||||
"tool": "fail2ban_health",
|
||||
"interval": "60s",
|
||||
"timeout": "10s",
|
||||
"looks": 2,
|
||||
"grace": "90s"
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "log",
|
||||
"type": "file",
|
||||
@@ -108,7 +153,10 @@
|
||||
"jail-recidive",
|
||||
"action-dualchain",
|
||||
"composed-jails"
|
||||
]
|
||||
],
|
||||
"health": {
|
||||
"kind": "unit"
|
||||
}
|
||||
}
|
||||
],
|
||||
"build": {
|
||||
|
||||
+84
-3
@@ -49,6 +49,23 @@ export interface GiteaIssue {
|
||||
body?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* **The branch a pull request was opened from, also once it is deleted** (novox/hq issue 578). The forge says it as
|
||||
* `head.ref` while the branch exists; once it is gone — the merge tool deletes it after the merge by default —
|
||||
* `head.ref` reads `refs/pull/<n>/head`, and only `head.label` still names the branch. A delivery group is the pull
|
||||
* requests sharing a head branch name (novox/hq ADR 0249), and the controller finds a merge's group, and joins the
|
||||
* group's merges in one batch, by the head its `pull.merged` names (ADR 0276 decision 5, ADR 0306 decision 2). With
|
||||
* the pull ref it found no group: on 2026-10-12 mesh-catalog #251's walk was cut at once and went before
|
||||
* mesh-controller #271's, which the group's order put first.
|
||||
*/
|
||||
export function headBranchOf(head: { ref?: unknown; label?: unknown } | null | undefined): string | undefined {
|
||||
const said = (v: unknown): string | undefined => (typeof v === "string" && v !== "" ? v : undefined);
|
||||
const ref = said(head?.ref);
|
||||
const label = said(head?.label);
|
||||
if (ref === undefined || /^refs\/pull\/\d+\/head$/.test(ref)) return label ?? ref;
|
||||
return ref;
|
||||
}
|
||||
|
||||
/** A pull request, trimmed to the fields a reviewer or an event body needs. */
|
||||
export interface GiteaPull {
|
||||
number: number;
|
||||
@@ -121,6 +138,17 @@ function meshConfig(file?: string): Record<string, string> {
|
||||
catch { return {}; }
|
||||
}
|
||||
|
||||
/**
|
||||
* A password, token or API key is the vault's, never a setting (novox/hq issue 342): a value for one of
|
||||
* them in the settings file is ignored, and that is said once — by its key, never with its value.
|
||||
*/
|
||||
const saidIgnored = new Set<string>();
|
||||
function ignoreSecretSetting(cfg: Record<string, unknown>, key: string): void {
|
||||
if (cfg[key] === undefined || saidIgnored.has(key)) return;
|
||||
saidIgnored.add(key);
|
||||
console.error(`the setting ${key} is ignored: it comes from the vault`);
|
||||
}
|
||||
|
||||
export class GiteaClient {
|
||||
readonly baseUrl: string;
|
||||
private readonly tokens: TokenSource;
|
||||
@@ -134,7 +162,8 @@ export class GiteaClient {
|
||||
/**
|
||||
* Build from the module's resolved environment. The URL comes from MESH_GITEA_URL (the mesh's own
|
||||
* name), falling back to the bare GITEA_URL and to the forge's loopback port. The token, in order:
|
||||
* one configured in settings or the environment (MESH_GITEA_TOKEN / GITEA_TOKEN), which wins; else
|
||||
* one configured in the environment (MESH_GITEA_TOKEN / GITEA_TOKEN), which wins — never a setting, which is
|
||||
* ignored and said (hq issue 342); else
|
||||
* one the module mints for itself with the admin account the vault delivered and keeps in its own
|
||||
* state (token.ts; hq issue 100). Throws only when neither is possible, naming what is missing,
|
||||
* rather than hand back a client that fails on first use.
|
||||
@@ -142,7 +171,8 @@ export class GiteaClient {
|
||||
static fromEnv(env: NodeJS.ProcessEnv = process.env): GiteaClient {
|
||||
const cfg = meshConfig(env.MESH_GITEA_CONFIG_FILE);
|
||||
const url = cfg.url ?? env.MESH_GITEA_URL ?? env.GITEA_URL ?? `http://127.0.0.1:${env.GITEA_PORT ?? "3000"}`;
|
||||
const configured = cfg.token ?? env.MESH_GITEA_TOKEN ?? env.GITEA_TOKEN;
|
||||
ignoreSecretSetting(cfg, "token");
|
||||
const configured = env.MESH_GITEA_TOKEN ?? env.GITEA_TOKEN;
|
||||
if (configured) return new GiteaClient(url, new ConfiguredToken(configured));
|
||||
return new GiteaClient(url, MintedToken.fromEnv(url, env));
|
||||
}
|
||||
@@ -341,6 +371,57 @@ export class GiteaClient {
|
||||
return out;
|
||||
}
|
||||
|
||||
/** **Every open pull request of a repository**, page by page (novox/hq ADR 0323 §4): what a trunk's move puts out of
|
||||
* date, however many are open, whatever their update time (a pass announces from listOpenPullsSince). Bounded at
|
||||
* 20 pages of 50; a repository with more is said, and the rest are left to the merge tool's own comparison. */
|
||||
async listOpenPulls(owner: string, repo: string): Promise<GiteaPull[]> {
|
||||
const out: GiteaPull[] = [];
|
||||
for (let page = 1; page <= 20; page++) {
|
||||
const batch = await this.listPullRequests(owner, repo, { state: "open", limit: "50", page: String(page) });
|
||||
out.push(...batch);
|
||||
if (batch.length < 50) return out;
|
||||
}
|
||||
console.error(`[gitea] ${owner}/${repo} has more than 1000 open pull requests; only the first 1000 are read`);
|
||||
return out;
|
||||
}
|
||||
|
||||
/** **The open pull requests of a repository updated since a moment**, newest first, page by page (novox/hq issue
|
||||
* 579): what a pass announces new heads from. The forge sorts `recentupdate` by update time, newest first, so the
|
||||
* paging stops at the first page that reaches back past `since`; with no moment — a sweep or the first pass —
|
||||
* every page is read. Bounded like listOpenPulls at 20 pages of 50. A pull request seen twice, pushed up a page
|
||||
* while the pages were read, is answered once. */
|
||||
async listOpenPullsSince(owner: string, repo: string, since: string): Promise<GiteaPull[]> {
|
||||
const out: GiteaPull[] = [];
|
||||
const seen = new Set<number>();
|
||||
const floor = since ? Date.parse(since) : NaN;
|
||||
for (let page = 1; page <= 20; page++) {
|
||||
const batch = await this.listPullRequests(owner, repo, { state: "open", sort: "recentupdate", limit: "50", page: String(page) });
|
||||
for (const p of batch) {
|
||||
if (seen.has(p.number)) continue;
|
||||
seen.add(p.number);
|
||||
out.push(p);
|
||||
}
|
||||
if (batch.length < 50) return out;
|
||||
const oldest = batch[batch.length - 1]?.updated_at;
|
||||
if (!Number.isNaN(floor) && oldest && Date.parse(oldest) < floor) return out;
|
||||
}
|
||||
console.error(`[gitea] ${owner}/${repo} has more than 1000 open pull requests updated since ${since || "ever"}; only the newest 1000 are read`);
|
||||
return out;
|
||||
}
|
||||
|
||||
/** The commit a branch is at now: a trunk's head (novox/hq ADR 0323). */
|
||||
async branchHead(owner: string, repo: string, branch: string): Promise<string> {
|
||||
const b = await this.request<any>(`${repoPath(owner, repo)}/branches/${pathSegments("branch", branch)}`);
|
||||
const sha = String(b?.commit?.id ?? "");
|
||||
if (!/^[0-9a-f]{40,64}$/i.test(sha)) throw new Error(`the forge named no commit for ${owner}/${repo}'s ${branch}`);
|
||||
return sha;
|
||||
}
|
||||
|
||||
/** One repository, as the forge holds it: its clone URL above all, for an announcement. */
|
||||
async getRepo(owner: string, repo: string): Promise<GiteaRepo> {
|
||||
return GiteaClient.mapRepo(await this.request<any>(repoPath(owner, repo)));
|
||||
}
|
||||
|
||||
async getPullRequest(owner: string, repo: string, index: number): Promise<GiteaPull> {
|
||||
indexOf(index);
|
||||
return GiteaClient.mapPull(await this.request<any>(`${repoPath(owner, repo)}/pulls/${index}`));
|
||||
@@ -646,7 +727,7 @@ export class GiteaClient {
|
||||
merge_commit_sha: p.merge_commit_sha ?? undefined,
|
||||
merged_at: p.merged_at ?? undefined,
|
||||
user: p.user?.login,
|
||||
head: p.head?.ref,
|
||||
head: headBranchOf(p.head),
|
||||
head_sha: p.head?.sha ?? undefined,
|
||||
base: p.base?.ref,
|
||||
updated_at: p.updated_at ?? undefined,
|
||||
|
||||
+23
-1
@@ -18,7 +18,7 @@
|
||||
// restart, when the check was opened is read from the forge's own history of the gate.
|
||||
|
||||
import { CHECK_CONTEXT, pendingWords } from "./pulls.js";
|
||||
import type { CommitStatus } from "./client.js";
|
||||
import type { CommitStatus, HeadCheck } from "./client.js";
|
||||
|
||||
/** The build seat's `started`, as mesh-controller's link.BuildStart publishes it. */
|
||||
export interface BuildStarted {
|
||||
@@ -37,6 +37,8 @@ export interface GateForge {
|
||||
gateOf(owner: string, repo: string, sha: string): Promise<{ state: string; updated_at?: string } | undefined>;
|
||||
/** The head's mesh/merge-gate statuses, every one the forge keeps, newest first (novox/hq issue 456). */
|
||||
gateHistory(owner: string, repo: string, sha: string): Promise<GateStatus[]>;
|
||||
/** The head's statuses by context, the newest of each, with their words (novox/hq ADR 0323). */
|
||||
headChecks(owner: string, repo: string, sha: string): Promise<Record<string, HeadCheck>>;
|
||||
}
|
||||
|
||||
/** One status of a head's gate as the forge keeps it: its state, its words, and when it was set. */
|
||||
@@ -174,6 +176,26 @@ export class GateWriter {
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* **A trunk's move puts the head's verdict out of date** (novox/hq ADR 0323 §4): its merge check statuses are put back
|
||||
* to pending, with the reason, in their turn after every write of the head already asked for — a verdict in flight
|
||||
* lands first. What is written is decided in the turn, from the statuses the forge holds then (`decide`, trunk.ts's
|
||||
* outOfDateStatuses): a check on its way, or a verdict already judged against the trunk's head, is left as it is. It
|
||||
* opens no check — nothing is asked — so a verdict of a check asked before it keeps its weight. Answers what it wrote;
|
||||
* a write that fails fails the caller, and the move is weighed again on the next pass.
|
||||
*/
|
||||
outOfDate(owner: string, repo: string, sha: string,
|
||||
decide: (checks: Record<string, HeadCheck>) => CommitStatus[] | Promise<CommitStatus[]>, target?: string): Promise<CommitStatus[]> {
|
||||
const h = this.head(owner, repo, sha);
|
||||
const written: CommitStatus[] = [];
|
||||
return this.turn(h, async () => {
|
||||
for (const s of await decide(await this.forge.headChecks(owner, repo, sha))) {
|
||||
await this.forge.setCommitStatus(owner, repo, sha, target ? { ...s, target_url: target } : s);
|
||||
written.push(s);
|
||||
}
|
||||
}).then(() => written);
|
||||
}
|
||||
|
||||
/**
|
||||
* A build machine took an ask of this head: said while no verdict was heard, and only for the check now pending —
|
||||
* a start from before the pending words were set (a run before a recheck, the stream's history) is not this check's.
|
||||
|
||||
+71
-35
@@ -17,8 +17,13 @@
|
||||
// Consumes node-build-agent.started — a build machine took an ask — and says, on the head a pending check is of,
|
||||
// which machine builds it since when (novox/hq issue 446). This module is the one writer of mesh/merge-gate.
|
||||
//
|
||||
// Serves gitea_commit_status from this bundle, not the tools', so a recheck's pending reaches that one writer
|
||||
// (novox/hq issue 456).
|
||||
// **A trunk's move puts every open verdict into it out of date** (novox/hq ADR 0323 §4, trunk.ts): each pass reads the
|
||||
// head of every trunk an open pull request merges into, and when one moved, the merge check statuses of every open pull
|
||||
// request into it go back to pending with the reason. Nothing is run for it. A verdict that lands judged against a trunk
|
||||
// commit that is no longer the trunk's head is said out of date in place of its pass or failure.
|
||||
//
|
||||
// Serves gitea_commit_status and gitea_merge_pull_request from this bundle, not the tools', so a recheck's pending and
|
||||
// the merge tool's ask of a check reach that one writer (novox/hq issue 456, ADR 0323).
|
||||
//
|
||||
// Every pull request the forge holds is announced, whatever its repository: the controller holds the
|
||||
// module graph and decides what is checked — a repository is never asked to opt in.
|
||||
@@ -34,10 +39,12 @@
|
||||
import { emit, on } from "@novox/mesh-sdk/events";
|
||||
import { registerModuleTools } from "@novox/mesh-sdk/tools";
|
||||
import { commitStatusTool } from "./delivery.js";
|
||||
import { mergeTool } from "./merge.js";
|
||||
import { GiteaClient, movedSince } from "./client.js";
|
||||
import { CHECK_CONTEXT, REPO_CHECK_CONTEXT, commentFor, headsToAnnounce, statusesFor, type Announced, type Checked,
|
||||
type RepoCheckFacts, reposToLook } from "./pulls.js";
|
||||
import { GateWriter, clonedAs, type BuildStarted } from "./gate.js";
|
||||
import { announcementOf, keepTook, lateVerdictComment, outOfDateWords, sameCommit, tookFile, weighTrunks } from "./trunk.js";
|
||||
|
||||
// Without a way to a token — configured, or mintable with the admin account (token.ts) — there is
|
||||
// nothing to watch; log and stay quiet rather than crash the runtime. With one, the first poll mints
|
||||
@@ -245,42 +252,15 @@ async function pollPulls(client: GiteaClient): Promise<void> {
|
||||
}
|
||||
if (sweep) lastSweep = Date.now();
|
||||
for (const repo of reposToLook(await client.listAllRepos(), floor, updated, sweep)) {
|
||||
const open = await client.listPullRequests(repo.owner, repo.name, { state: "open", sort: "recentupdate", limit: "20" });
|
||||
// Every open pull request updated since the floor, paged (novox/hq issue 579): the newest 20 alone left a head
|
||||
// beyond them unannounced. A sweep and the first pass read every page.
|
||||
const open = await client.listOpenPullsSince(repo.owner, repo.name, sweep ? "" : floor);
|
||||
for (const pull of headsToAnnounce(repo.full_name, open, heads)) {
|
||||
const key = `${repo.full_name}#${pull.number}`;
|
||||
const fresh = primedPulls || (!!pull.updated_at && pull.updated_at > dayAgo);
|
||||
if (fresh) {
|
||||
const files = await client.listPullFiles(repo.owner, repo.name, pull.number);
|
||||
const head = String(pull.head_sha);
|
||||
// What the controller maps the change onto the mesh's module graph with (novox/hq ADR 0237 as
|
||||
// amended): which changed directories hold a module at the head — the merge's rule (issue 278) —
|
||||
// and whether the head holds the repository's own merge-check.sh. Not said when it could not be
|
||||
// read; the controller then reads the change as touching everything built from the repository.
|
||||
let moduleDirs: string[] | null = null;
|
||||
let mergeCheck: boolean | null = null;
|
||||
try {
|
||||
if (!files.truncated) moduleDirs = await client.moduleDirsAt(repo.owner, repo.name, head, files.paths);
|
||||
mergeCheck = await client.holdsFile(repo.owner, repo.name, head, "merge-check.sh");
|
||||
} catch (err) {
|
||||
console.error(`[gitea] ${key}: what the head holds could not be read — ${err instanceof Error ? err.message : err}`);
|
||||
}
|
||||
await emit("pull.updated", {
|
||||
owner: repo.owner,
|
||||
repo: repo.name,
|
||||
number: pull.number,
|
||||
title: pull.title,
|
||||
body: pull.body,
|
||||
base: pull.base,
|
||||
head: pull.head,
|
||||
head_sha: pull.head_sha,
|
||||
clone_url: repo.clone_url,
|
||||
html_url: pull.html_url,
|
||||
paths: files.paths,
|
||||
paths_truncated: files.truncated,
|
||||
removed: files.removed,
|
||||
...(moduleDirs ? { module_dirs: moduleDirs, module_dirs_said: true } : {}),
|
||||
...(mergeCheck !== null ? { merge_check: mergeCheck, merge_check_said: true } : {}),
|
||||
});
|
||||
// The announcement the merge tool makes too when it asks a check (trunk.ts, novox/hq ADR 0323 §5).
|
||||
await emit("pull.updated", await announcementOf(client, repo.owner, repo.name, repo.clone_url, pull));
|
||||
// Said, so an operator knows the mesh was asked to check it.
|
||||
console.log(`[gitea] announced ${key} at ${String(pull.head_sha).slice(0, 8)} to be checked before it merges`);
|
||||
// Pending until the verdict comes, so the pull request says a check is running rather than nothing — and
|
||||
@@ -290,17 +270,52 @@ async function pollPulls(client: GiteaClient): Promise<void> {
|
||||
heads[key] = String(pull.head_sha);
|
||||
changed = true;
|
||||
}
|
||||
if (await trunksMoved(client, repo.owner, repo.name)) changedTrunks = true;
|
||||
}
|
||||
if (!primedPulls || changed) keepHeads();
|
||||
if (changedTrunks) keepTrunks();
|
||||
changedTrunks = false;
|
||||
primedPulls = true;
|
||||
lastPullLook = began;
|
||||
}
|
||||
|
||||
// **The trunks' heads, as this module last read them** (novox/hq ADR 0323 §4), by `owner/repo:branch`: kept beside the
|
||||
// other records, so a move made while this module was away is weighed on its first pass back. With no record — the
|
||||
// first pass after this rolls out — a trunk's head is taken as it is and nothing is put out of date: the verdicts
|
||||
// then standing are weighed on the trunk's next move, and the merge tool compares commits whatever this holds.
|
||||
const trunksRecord = process.env.MESH_GITEA_STATE_DIR ? join(process.env.MESH_GITEA_STATE_DIR, "trunks-seen.json") : null;
|
||||
let trunks: Record<string, string> = {};
|
||||
let changedTrunks = false;
|
||||
if (trunksRecord && existsSync(trunksRecord)) {
|
||||
try {
|
||||
trunks = (JSON.parse(readFileSync(trunksRecord, "utf8")) as { trunks: Record<string, string> }).trunks ?? {};
|
||||
} catch {
|
||||
// An unreadable record is no record: the next pass takes each trunk's head as it is.
|
||||
}
|
||||
}
|
||||
function keepTrunks(): void {
|
||||
if (!trunksRecord) return;
|
||||
mkdirSync(join(trunksRecord, ".."), { recursive: true });
|
||||
const tmp = trunksRecord + ".tmp";
|
||||
writeFileSync(tmp, JSON.stringify({ trunks }));
|
||||
renameSync(tmp, trunksRecord);
|
||||
}
|
||||
|
||||
/** Whether a trunk of one repository moved, weighed with this module's forge and its one writer (trunk.ts). */
|
||||
function trunksMoved(client: GiteaClient, owner: string, name: string): Promise<boolean> {
|
||||
return weighTrunks(client, gates, trunks, owner, name);
|
||||
}
|
||||
|
||||
// **The verdict, set where the pull request shows it.** Every verdict is the head commit's status; one
|
||||
// that is not a pass also leaves the check's own account as a comment, so the reason is read where the
|
||||
// change is reviewed. An error — the check could not run — is the forge's `error`, never a success.
|
||||
async function setVerdict(client: GiteaClient, event: { body: unknown }): Promise<void> {
|
||||
const c = (event.body ?? {}) as Checked;
|
||||
if (!c.group && c.owner && c.repo && c.took) {
|
||||
// How long it took, for the merge tool to say how long the next one may (novox/hq ADR 0323 §5).
|
||||
keepTook(tookFile(process.env.MESH_GITEA_STATE_DIR), `${c.owner}/${c.repo}`,
|
||||
{ took: c.took, at: new Date().toISOString(), head: c.commit });
|
||||
}
|
||||
if (c.group) {
|
||||
// A delivery group's composed check (novox/hq ADR 0239): its verdict is the group owner's to say, on each
|
||||
// member's head, as mesh/delivery-group — never this head's merge gate.
|
||||
@@ -318,6 +333,26 @@ async function setVerdict(client: GiteaClient, event: { body: unknown }): Promis
|
||||
target = pull?.html_url || undefined;
|
||||
base = pull?.base || undefined;
|
||||
}
|
||||
// **A verdict judged against a trunk commit that is no longer the trunk's head is out of date when it lands**
|
||||
// (novox/hq ADR 0323 §4): a check asked before the trunk moved. It is said so, pending with the reason, in place of
|
||||
// its pass or failure; a trunk that cannot be read lets the verdict stand, and the merge tool compares the commits.
|
||||
const trunk = base ?? c.plan?.base;
|
||||
if (c.trunk && trunk) {
|
||||
const head = await client.branchHead(c.owner, c.repo, trunk).catch(() => undefined);
|
||||
if (head && !sameCommit(c.trunk, head)) {
|
||||
const description = outOfDateWords(c.trunk, head);
|
||||
for (const context of [CHECK_CONTEXT, ...(c["repo-check"] ? [REPO_CHECK_CONTEXT] : [])]) {
|
||||
await client.setCommitStatus(c.owner, c.repo, c.commit, { state: "pending", context, description,
|
||||
...(target ? { target_url: target } : {}) });
|
||||
}
|
||||
// Its account is kept and shown all the same (ADR 0323 §7): what it judged, and that it is out of date.
|
||||
const said = commentFor(c, await repoCheckFacts(client, c, trunk));
|
||||
if (c.number) await client.addComment(c.owner, c.repo, c.number, lateVerdictComment(c, said, description));
|
||||
console.log(`[gitea] ${c.owner}/${c.repo}#${c.number ?? "?"} at ${c.commit.slice(0, 8)}: merge check ${c.verdict}, ` +
|
||||
`${description}`);
|
||||
return;
|
||||
}
|
||||
}
|
||||
const facts = await repoCheckFacts(client, c, base ?? c.plan?.base);
|
||||
for (const status of statusesFor(c, facts)) {
|
||||
await client.setCommitStatus(c.owner, c.repo, c.commit, target ? { ...status, target_url: target } : status);
|
||||
@@ -350,6 +385,7 @@ const gates = new GateWriter({
|
||||
setCommitStatus: (owner, repo, sha, status) => (gitea as GiteaClient).setCommitStatus(owner, repo, sha, status),
|
||||
gateOf: async (owner, repo, sha) => (await (gitea as GiteaClient).headChecks(owner, repo, sha))[CHECK_CONTEXT],
|
||||
gateHistory: (owner, repo, sha) => (gitea as GiteaClient).contextHistory(owner, repo, sha, CHECK_CONTEXT),
|
||||
headChecks: (owner, repo, sha) => (gitea as GiteaClient).headChecks(owner, repo, sha),
|
||||
});
|
||||
|
||||
// **The status tool is served here, beside the writer** (novox/hq issue 456): the runtime starts each bundle as its own
|
||||
@@ -358,7 +394,7 @@ const gates = new GateWriter({
|
||||
registerModuleTools("gitea", () => (gitea ? [commitStatusTool({
|
||||
setCommitStatus: (owner, repo, sha, status) => (gitea as GiteaClient).setCommitStatus(owner, repo, sha, status),
|
||||
gates,
|
||||
})] : []));
|
||||
}), mergeTool(gitea, { gates })] : []));
|
||||
|
||||
// **A build machine took an ask** (the build seat's `started`, novox/hq ADR 0157): said on the head it checks, when
|
||||
// the head is an open pull request's this module announced. Any other build — a branch, a merge's — says nothing.
|
||||
|
||||
@@ -0,0 +1,183 @@
|
||||
// The forge's merge tool, `gitea_merge_pull_request` (novox/hq ADR 0266, issue 381, ADR 0323).
|
||||
//
|
||||
// **Served from the watcher's bundle, beside the merge check's one writer** (index.ts, gate.ts; novox/hq issue 456):
|
||||
// when a verdict is judged against an earlier trunk commit than the trunk's head, the tool asks the head's check and
|
||||
// puts its statuses back to pending through that writer, so where the check runs is said and no second check is asked.
|
||||
// Each bundle the runtime serves is its own process; from the tools' bundle the pending bypassed the writer's chain.
|
||||
|
||||
import { emit } from "@novox/mesh-sdk/events";
|
||||
import type { ToolDefinition } from "@novox/mesh-sdk/tools";
|
||||
import type { CommitStatus, GiteaClient } from "./client.js";
|
||||
import { mergeableUnder, newestByContext } from "./checks.js";
|
||||
import { MERGE_METHODS, Refused, checkName } from "./guard.js";
|
||||
import { CHECK_CONTEXT, REPO_CHECK_CONTEXT } from "./pulls.js";
|
||||
import { announcementOf, askedWords, outOfDateWords, tookFile, tookWords, twelve, verdictAtTrunk } from "./trunk.js";
|
||||
|
||||
/** A head named for a merge: at least 12 hex characters of its sha (novox/hq issue 381), up to a sha-256 commit's 64. */
|
||||
const REVIEWED_HEAD = /^[0-9a-f]{12,64}$/;
|
||||
|
||||
/** What the merge tool reaches beside the forge: the merge check's one writer, how an event is said, the module's
|
||||
* state directory, and the clock. */
|
||||
export interface MergeDeps {
|
||||
gates: { rechecked(owner: string, repo: string, sha: string, status: CommitStatus): Promise<void> };
|
||||
emit?: (type: string, body: unknown) => Promise<void>;
|
||||
stateDir?: string;
|
||||
now?: () => Date;
|
||||
}
|
||||
|
||||
export function mergeTool(gitea: GiteaClient, given: MergeDeps): ToolDefinition {
|
||||
const deps = { ...given, emit: given.emit ?? emit, stateDir: given.stateDir ?? process.env.MESH_GITEA_STATE_DIR,
|
||||
now: given.now ?? (() => new Date()) };
|
||||
return {
|
||||
name: "gitea_merge_pull_request",
|
||||
description: "Merge a pull request at the head you name — the commit you reviewed — when that is still its head " +
|
||||
"and every status its base branch's protection requires succeeded there (mesh/merge-gate, mesh/repo-check), " +
|
||||
"judged against the trunk's head as it is now, optionally deleting the source branch afterwards. Refused " +
|
||||
"otherwise, with nothing merged: no tool merges past a check, and none merges a head that moved after it was " +
|
||||
"reviewed. A verdict judged against an earlier trunk commit is refused, naming both, and the head's check against " +
|
||||
"the trunk's head is asked once, with how long the last check of the repository took: ask again at the same head " +
|
||||
"once it passes — the tool never merges by itself.",
|
||||
input: {
|
||||
owner: { type: "string", description: "the repository owner" },
|
||||
repo: { type: "string", description: "the repository name" },
|
||||
number: { type: "number", description: "the PR number" },
|
||||
head: { type: "string", description: "the commit you reviewed, as its sha or at least its first 12 characters: " +
|
||||
"refused unless it is the pull request's head" },
|
||||
method: { type: "string", description: "merge (default) | fast-forward-only | squash | rebase | rebase-merge. " +
|
||||
"Only merge and fast-forward-only can be read back to prove the head you named is what landed" },
|
||||
delete_branch: { type: "boolean", description: "delete the source branch after merge (default true)" },
|
||||
},
|
||||
run: async (args) => {
|
||||
const owner = checkName("owner", args.owner);
|
||||
const repo = checkName("name", args.repo);
|
||||
const number = Number(args.number);
|
||||
const method = args.method ? String(args.method) : "merge";
|
||||
if (!MERGE_METHODS.has(method)) throw new Refused(`${method} is not a merge method: ${[...MERGE_METHODS].join(", ")}`);
|
||||
const deleteBranch = args.delete_branch === undefined ? true : Boolean(args.delete_branch);
|
||||
// **Only a head whose required statuses all succeeded** (novox/hq ADR 0266): read here rather than left
|
||||
// to the forge, whose administrator — the account this tool's token is minted for — may merge past a
|
||||
// rule that lets it. A rule that cannot be read is no rule passed.
|
||||
// **Only the head that was reviewed** (novox/hq issue 381): mesh-controller#197 was reviewed at 065cfa0d,
|
||||
// force-pushed to 3ced96f, checked green there, and merged at 3ced96f, because this tool merged whatever the
|
||||
// head was and nothing told it which head the review had approved. The caller names it; a head that moved
|
||||
// since is refused, and the forge is handed the same head, so a push between this read and the merge is
|
||||
// refused by the forge too.
|
||||
const named = args.head === undefined ? "" : String(args.head).trim().toLowerCase();
|
||||
if (!named) {
|
||||
throw new Refused(`${owner}/${repo}#${number} is not merged: name the head you reviewed (head), so that only ` +
|
||||
`that commit can land (novox/hq issue 381). Nothing was merged`);
|
||||
}
|
||||
// Twelve characters, not seven: an author who force-pushes can make a commit whose first seven match the
|
||||
// reviewed one in seconds, and the review would then merge it. Twelve cannot be matched that way.
|
||||
if (!REVIEWED_HEAD.test(named)) {
|
||||
throw new Refused(`${owner}/${repo}#${number} is not merged: head ${named} does not name one commit — give its ` +
|
||||
`sha, or at least its first 12 characters. Nothing was merged`);
|
||||
}
|
||||
const pull = await gitea.getPullRequest(owner, repo, number);
|
||||
if (pull.merged) throw new Refused(`${owner}/${repo}#${number} is already merged`);
|
||||
if (!pull.head_sha || !pull.base) throw new Refused(`${owner}/${repo}#${number} names no head commit or base branch`);
|
||||
if (!pull.head_sha.toLowerCase().startsWith(named)) {
|
||||
throw new Refused(`${owner}/${repo}#${number} is not merged: its head is ${pull.head_sha.slice(0, 12)}, not the ` +
|
||||
`${named} you named. If you reviewed an earlier head, review the one it has now, then name it. Nothing was merged`);
|
||||
}
|
||||
const rule = await gitea.branchProtection(owner, repo, pull.base);
|
||||
// Only into a branch whose protection requires the checks: a pull request whose base was moved to a
|
||||
// branch nothing guards would merge past every check (novox/hq ADR 0266).
|
||||
if (!rule || !rule.enable_status_check || !(rule.status_check_contexts ?? []).filter(Boolean).length) {
|
||||
throw new Refused(`${owner}/${repo}#${number} merges into ${pull.base}, which no protection rule requiring ` +
|
||||
`the checks guards: a tool merges only into such a branch (novox/hq ADR 0266). Nothing was merged`);
|
||||
}
|
||||
// **The verdict must be judged against the trunk's head** (novox/hq ADR 0323 §5, issue 341): a check judges the
|
||||
// head merged into a named trunk commit, and a merge of another trunk is not the one that check judged. Compared
|
||||
// here at the moment the merge is asked, not left to the statuses the trunk's move resets a pass later. When they
|
||||
// differ nothing is merged, and the head's check against the trunk's head is asked once — the announcement a new
|
||||
// head makes — never again while one is on its way. The caller asks again at the reviewed head once it passes.
|
||||
const trunkHead = await gitea.branchHead(owner, repo, pull.base);
|
||||
const checks = await gitea.headChecks(owner, repo, pull.head_sha);
|
||||
const at = verdictAtTrunk(checks, trunkHead);
|
||||
if (!at.current) {
|
||||
const judged = at.judged ? `was judged against trunk ${twelve(at.judged)}` : "names no trunk commit it was judged against";
|
||||
const took = tookWords(tookFile(deps.stateDir), `${owner}/${repo}`);
|
||||
const head = twelve(pull.head_sha), now = twelve(trunkHead);
|
||||
if (at.onItsWay) {
|
||||
throw new Refused(`${owner}/${repo}#${number} is not merged: its verdict ${judged}, and ${pull.base} is now ` +
|
||||
`${now}. Its check is already asked and not yet answered (${checks[CHECK_CONTEXT]?.description}), so it is not ` +
|
||||
`asked again; ${took}. Ask again at ${head} once both statuses pass: this tool never merges by itself ` +
|
||||
`(novox/hq ADR 0323). A check that never answers is asked again with mesh-delivery.recheck. Nothing was merged`);
|
||||
}
|
||||
// Pending first, so a second ask finds the check on its way; then the head is announced as a new head is. An
|
||||
// announcement that cannot be made puts the statuses back to out of date, so the next ask asks again rather
|
||||
// than wait for ever on a check nobody asked.
|
||||
const contexts = [CHECK_CONTEXT, ...(checks[REPO_CHECK_CONTEXT] ? [REPO_CHECK_CONTEXT] : [])];
|
||||
// Through the module's one writer of the merge check (gate.ts, novox/hq issue 456): the pending opens a new
|
||||
// check of the head, so where it runs is said ("building on …") and its verdict is not weighed as older.
|
||||
const say = async (description: string) => {
|
||||
for (const context of contexts) {
|
||||
await deps.gates.rechecked(owner, repo, String(pull.head_sha), { state: "pending", context, description,
|
||||
...(pull.html_url ? { target_url: pull.html_url } : {}) });
|
||||
}
|
||||
};
|
||||
await say(askedWords(deps.now(), trunkHead));
|
||||
try {
|
||||
const cloneURL = (await gitea.getRepo(owner, repo)).clone_url;
|
||||
await deps.emit("pull.updated", await announcementOf(gitea, owner, repo, cloneURL, pull));
|
||||
} catch (err) {
|
||||
await say(outOfDateWords(at.judged, trunkHead)).catch(() => {});
|
||||
throw new Error(`${owner}/${repo}#${number} is not merged: its verdict ${judged}, and ${pull.base} is now ` +
|
||||
`${now}, and its check could not be asked: ${err instanceof Error ? err.message : err}. Nothing was merged`);
|
||||
}
|
||||
throw new Refused(`${owner}/${repo}#${number} is not merged: its verdict ${judged}, and ${pull.base} is now ` +
|
||||
`${now}. The check of head ${head} merged into trunk ${now} was asked; ${took}. Ask again at ${head} once both ` +
|
||||
`statuses pass: this tool never merges by itself (novox/hq ADR 0323). In a busy repository another merge may ` +
|
||||
`move the trunk again first; this is then refused and asked once more. A check that never answers is asked ` +
|
||||
`again with mesh-delivery.recheck. Nothing was merged`);
|
||||
}
|
||||
const statuses = newestByContext((await gitea.combinedStatus(owner, repo, pull.head_sha)).statuses);
|
||||
const verdict = mergeableUnder(pull.base, rule, statuses);
|
||||
if (verdict.mergeable !== true) {
|
||||
throw new Refused(`${owner}/${repo}#${number} is not merged: ${verdict.says}. A pull request merges once ` +
|
||||
`its checks succeed; no tool merges past them (novox/hq ADR 0266)`);
|
||||
}
|
||||
await gitea.mergePullRequest(owner, repo, number, method, deleteBranch, pull.head_sha);
|
||||
// The merge commit only exists now; answered so the caller can follow what is built from it.
|
||||
// pull.merged is the events entrypoint's to announce (index.ts), once, within its poll.
|
||||
// From here the merge has happened. Nothing below may read as if it had not: a read-back that fails or cannot
|
||||
// be done is said beside merged: true, and only a proven other head is an error — one that says the merge happened.
|
||||
const answer: Record<string, unknown> = { merged: true, number, method, head: pull.head_sha, deleted_branch: deleteBranch };
|
||||
let landed: string | undefined;
|
||||
try {
|
||||
const merged = await gitea.getPullRequest(owner, repo, number);
|
||||
answer.merge_commit_sha = merged.merge_commit_sha;
|
||||
// What landed, read back from the commit the merge made rather than from the forge's record of the pull
|
||||
// request, which is the very thing a race would leave behind. A merge commit's second parent is the head it
|
||||
// joined; a fast-forward lands the head itself. A squash or a rebase makes new commits that name no head.
|
||||
if (!merged.merge_commit_sha) {
|
||||
answer.read_back = "the forge named no merge commit, so what landed cannot be read back";
|
||||
} else if (method === "merge") {
|
||||
const parents = await gitea.commitParents(owner, repo, merged.merge_commit_sha);
|
||||
if (parents.length === 2) landed = parents[1];
|
||||
else answer.read_back = `the merge commit has ${parents.length} parent(s), not two: what landed cannot be read back`;
|
||||
} else if (method === "fast-forward-only") {
|
||||
// That the forge names the head itself as a fast-forward's merge commit is assumed, not seen on this forge:
|
||||
// a different sha is said, not raised as another head having landed.
|
||||
if (merged.merge_commit_sha === pull.head_sha) landed = merged.merge_commit_sha;
|
||||
else answer.read_back = `the forge named ${merged.merge_commit_sha.slice(0, 12)} as this fast-forward's commit, ` +
|
||||
`not the head: that the head landed could not be confirmed`;
|
||||
} else {
|
||||
answer.read_back = `a ${method} makes new commits that name no head: what landed cannot be read back`;
|
||||
}
|
||||
} catch (err) {
|
||||
answer.read_back = `the merge happened, but reading back what landed failed: ${(err as Error).message}`;
|
||||
}
|
||||
// Loud, not a field beside merged: true that a caller reading only that would miss. The merge cannot be taken
|
||||
// back by this tool; what must not happen is that it reads as the merge that was asked for.
|
||||
if (landed !== undefined && landed !== pull.head_sha) {
|
||||
throw new Error(`${owner}/${repo}#${number} WAS merged (${answer.merge_commit_sha}), but the forge merged ` +
|
||||
`${landed.slice(0, 12)}, not the ${pull.head_sha.slice(0, 12)} it was handed and you reviewed: a commit nobody ` +
|
||||
`reviewed has landed. Read it before anything is built from it (novox/hq issue 381)`);
|
||||
}
|
||||
if (landed !== undefined) answer.landed = landed;
|
||||
return answer;
|
||||
},
|
||||
};
|
||||
}
|
||||
+22
-2
@@ -59,6 +59,12 @@ export interface Checked {
|
||||
plan?: ChangePlan;
|
||||
/** Set on a delivery group's composed check (novox/hq ADR 0239): not this head's merge gate. */
|
||||
group?: string;
|
||||
/** The trunk commit the head was merged into and judged as, twelve characters (novox/hq ADR 0323 §3); absent from a
|
||||
* controller whose build seat judged the head alone. Beside are the other repositories' commits the check read,
|
||||
* by directory; took how long it ran. */
|
||||
trunk?: string;
|
||||
beside?: Record<string, string>;
|
||||
took?: string;
|
||||
}
|
||||
|
||||
/** A change plan, as the controller says it (mesh-controller internal/link, ChangePlan). */
|
||||
@@ -199,17 +205,25 @@ export function statusFor(c: Checked): CommitStatus {
|
||||
const description = builds(c.plan)
|
||||
? described(gate.verdict, `${c.plan!.summary}; ${gate.summary}`)
|
||||
: described(gate.verdict, gate.summary, gate.modules, gate.dependents);
|
||||
return { state: stateOf(gate.verdict), context: CHECK_CONTEXT, description };
|
||||
return judgedFirst(c, { state: stateOf(gate.verdict), context: CHECK_CONTEXT, description });
|
||||
}
|
||||
|
||||
/** Every status a verdict sets: the gate's, and the repository's own check's when it was said. */
|
||||
export function statusesFor(c: Checked, facts: RepoCheckFacts = {}): CommitStatus[] {
|
||||
const out = [statusFor(c)];
|
||||
const repo = c["repo-check"];
|
||||
if (repo) out.push(repoStatus(repo, facts));
|
||||
if (repo) out.push(judgedFirst(c, repoStatus(repo, facts)));
|
||||
return out;
|
||||
}
|
||||
|
||||
/** **A status names what its verdict judged, first** (novox/hq ADR 0323 §3): "head 1a2b3c4d5e6f merged into trunk
|
||||
* 7a8b9c0d1e2f — pass: …", so the forge's 140 characters never clip it away; the trunk's move is weighed by it. A
|
||||
* verdict that names no trunk commit is said as before. */
|
||||
function judgedFirst(c: Checked, status: CommitStatus): CommitStatus {
|
||||
if (!c.trunk) return status;
|
||||
return { ...status, description: clipped(`head ${c.commit.slice(0, 12)} merged into trunk ${c.trunk.slice(0, 12)} — ${status.description}`) };
|
||||
}
|
||||
|
||||
/** The comment a verdict leaves on its pull request, with the check's own account: the change plan of a
|
||||
* change that builds something (novox/hq ADR 0238), and why, when the gate is not a pass or the
|
||||
* repository's own check failed or could not run. A repository with no merge-check.sh, touching nothing,
|
||||
@@ -220,6 +234,12 @@ export function commentFor(c: Checked, facts: RepoCheckFacts = {}): string | nul
|
||||
const wrong = repoWrong(repo, facts);
|
||||
if (gate.verdict === "pass" && !wrong && !builds(c.plan)) return null;
|
||||
const lines = [`**Merge check** at \`${c.commit.slice(0, 8)}\``, ""];
|
||||
if (c.trunk) {
|
||||
// What was judged (novox/hq ADR 0323 §3): the head merged into the trunk commit, and what was read beside it.
|
||||
const beside = Object.entries(c.beside ?? {}).map(([d, sha]) => `${d} \`${sha}\``).join(", ");
|
||||
lines.splice(1, 0, `Judged: head \`${c.commit.slice(0, 12)}\` merged into trunk \`${c.trunk.slice(0, 12)}\`` +
|
||||
(beside ? `; read beside it: ${beside}` : ""));
|
||||
}
|
||||
lines.push(`- \`${CHECK_CONTEXT}\`: **${saidAs(gate.verdict).toUpperCase()}** — ${gate.summary}` +
|
||||
(gate.modules?.length ? ` (modules: ${gate.modules.join(", ")}` +
|
||||
(gate.dependents?.length ? `; built after them: ${gate.dependents.join(", ")}` : "") + ")" : ""));
|
||||
|
||||
@@ -93,3 +93,47 @@ test("a pull request or issue number that is not one is refused before the forge
|
||||
server.close();
|
||||
}
|
||||
});
|
||||
|
||||
// **A merge names the branch it came from, also once that branch is deleted** (novox/hq issue 578). On 2026-10-12
|
||||
// the merge tool merged mesh-controller #271 and mesh-catalog #251, one delivery group, and deleted each branch. The
|
||||
// forge then read each pull request's head.ref as `refs/pull/<n>/head`, and `pull.merged` named that as the head: the
|
||||
// controller asked mesh-delivery for a group of that name, found none, and cut #251's walk at once, before #271's,
|
||||
// which the group's order put first.
|
||||
test("a merged pull request whose branch was deleted is announced with its branch, not its pull ref", async () => {
|
||||
const { GiteaClient, headBranchOf } = await import("../client.ts");
|
||||
const group = "feat/341-a-merge-check-judges-the-merge";
|
||||
// As the forge answered for #251 after its merge.
|
||||
const merged = {
|
||||
number: 251, title: "A verdict goes out of date when its trunk moves", state: "closed", merged: true,
|
||||
merged_at: "2026-10-12T16:11:29Z", merge_commit_sha: "23ea0f431faf91aa83e7b0297e0a9e442180f0c7",
|
||||
head: { label: group, ref: "refs/pull/251/head", sha: "41b6e10077efdef694b519879fcbaeaf69d472b1" },
|
||||
base: { label: "main", ref: "main" }, html_url: "http://fake/novox/mesh-catalog/pulls/251",
|
||||
};
|
||||
const server = createServer((req, res) => {
|
||||
const url = req.url ?? "";
|
||||
const send = (v: unknown) => res.end(JSON.stringify(v));
|
||||
if (url.startsWith("/api/v1/repos/novox/mesh-catalog/pulls/251")) return send(merged);
|
||||
if (url.startsWith("/api/v1/repos/novox/mesh-catalog/pulls?")) return send([merged]);
|
||||
res.statusCode = 404;
|
||||
res.end("{}");
|
||||
});
|
||||
await new Promise<void>((r) => server.listen(0, r));
|
||||
const client = new GiteaClient(`http://127.0.0.1:${(server.address() as any).port}`, "t");
|
||||
try {
|
||||
// What the announcer reads each pass, and what `pull.merged` carries as its head.
|
||||
const listed = await client.listPullRequests("novox", "mesh-catalog", { state: "closed", sort: "recentupdate", limit: "20" });
|
||||
assert.deepEqual(listed.map((p) => p.head), [group], "the pass names the group's branch, never refs/pull/251/head");
|
||||
assert.equal((await client.getPullRequest("novox", "mesh-catalog", 251)).head, group);
|
||||
} finally {
|
||||
server.close();
|
||||
}
|
||||
// While the branch exists the forge says it as ref, and that is what is named; a label alone is the branch too.
|
||||
assert.equal(headBranchOf({ label: "feat/x", ref: "feat/x" }), "feat/x");
|
||||
assert.equal(headBranchOf({ label: "feat/x" }), "feat/x");
|
||||
assert.equal(headBranchOf({ ref: "feat/x" }), "feat/x");
|
||||
// A branch whose own name only looks like a pull ref's start is still a branch.
|
||||
assert.equal(headBranchOf({ label: "y", ref: "refs/pulled" }), "refs/pulled");
|
||||
// No label to read: the pull ref is all the forge said, and it is kept rather than made up.
|
||||
assert.equal(headBranchOf({ ref: "refs/pull/7/head" }), "refs/pull/7/head");
|
||||
assert.equal(headBranchOf(undefined), undefined);
|
||||
});
|
||||
|
||||
@@ -7,11 +7,14 @@
|
||||
import { test, after, beforeEach } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { createServer, type IncomingMessage, type ServerResponse } from "node:http";
|
||||
import { readFileSync } from "node:fs";
|
||||
import { mkdtempSync, readFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
|
||||
import { GiteaClient } from "../dist/client.js";
|
||||
import { ConfiguredToken } from "../dist/token.js";
|
||||
import { getGiteaTools } from "../dist/tools/index.js";
|
||||
import { mergeTool } from "../dist/merge.js";
|
||||
import { commitStatusTool } from "../dist/delivery.js";
|
||||
import { GateWriter } from "../dist/gate.js";
|
||||
import { judgeApiCall, judgeProtectionChange, ruleCovers, routedPath } from "../dist/guard.js";
|
||||
@@ -21,7 +24,11 @@ interface Sent { method: string; path: string; body: any }
|
||||
const state = {
|
||||
sent: [] as Sent[],
|
||||
asked: [] as string[],
|
||||
statuses: [] as { context: string; state: string }[],
|
||||
statuses: [] as { context: string; state: string; description?: string; updated_at?: string }[],
|
||||
// The trunk's head (novox/hq ADR 0323): what the merge tool compares the verdict's trunk commit with.
|
||||
trunk: "",
|
||||
// What the tools said on the bus.
|
||||
emitted: [] as { type: string; body: any }[],
|
||||
rules: [] as any[],
|
||||
lastCommit: "2026-01-01T00:00:00Z",
|
||||
// The pull request's head; the head the merge commit joined (the same, unless a test says otherwise) — the forge's
|
||||
@@ -36,13 +43,21 @@ const state = {
|
||||
commitsFail: 0,
|
||||
};
|
||||
const REVIEWED = "abc1234def5678901234567890abcdef12345678";
|
||||
const TRUNK = "7a8b9c0d1e2f3a4b5c6d7e8f9a0b1c2d3e4f5a6b";
|
||||
const MOVED = "3c4d5e6f7a8b9c0d1e2f3a4b5c6d7e8f9a0b1c2d";
|
||||
// A verdict judged against a trunk commit, as both statuses say it (novox/hq ADR 0323 §3).
|
||||
const judgedAt = (trunk: string, state = "success") => [
|
||||
{ context: "mesh/merge-gate", state, description: `head ${REVIEWED.slice(0, 12)} merged into trunk ${trunk.slice(0, 12)} — pass: x` },
|
||||
{ context: "mesh/repo-check", state, description: `head ${REVIEWED.slice(0, 12)} merged into trunk ${trunk.slice(0, 12)} — pass: y` }];
|
||||
const protectedMain = () => ({ rule_name: "main", enable_push: false, enable_status_check: true,
|
||||
status_check_contexts: ["mesh/merge-gate", "mesh/repo-check"], block_admin_merge_override: true });
|
||||
|
||||
beforeEach(() => {
|
||||
state.sent = [];
|
||||
state.asked = [];
|
||||
state.statuses = [{ context: "mesh/merge-gate", state: "success" }, { context: "mesh/repo-check", state: "success" }];
|
||||
state.statuses = judgedAt(TRUNK);
|
||||
state.trunk = TRUNK;
|
||||
state.emitted = [];
|
||||
state.rules = [protectedMain()];
|
||||
state.head = REVIEWED;
|
||||
state.landed = "";
|
||||
@@ -73,6 +88,15 @@ const server = createServer(async (req, res) => {
|
||||
const repo = "/repos/novox/mesh-catalog";
|
||||
if (method === "GET" && path === repo) return json(res, 200, { full_name: "novox/mesh-catalog", default_branch: "main" });
|
||||
if (method === "GET" && path === `${repo}/branch_protections`) return json(res, 200, state.rules);
|
||||
if (method === "GET" && path === `${repo}/branches/main`) return json(res, 200, { name: "main", commit: { id: state.trunk } });
|
||||
if (method === "GET" && path === `${repo}/pulls/7/files`) {
|
||||
return json(res, 200, url.searchParams.get("page") === "1" ? [{ filename: "modules/gitea/x.ts", status: "modified" }] : []);
|
||||
}
|
||||
if (method === "POST" && path === `${repo}/statuses/${state.head}`) {
|
||||
// Kept as the forge keeps them: the newest of a context first.
|
||||
state.statuses = [{ ...body, updated_at: new Date().toISOString() }, ...state.statuses.filter((s) => s.context !== body.context)];
|
||||
return json(res, 201, body);
|
||||
}
|
||||
if (method === "GET" && path === "/user") return json(res, 200, { login: "mesh-admin" });
|
||||
if (method === "GET" && path === "/repos/mesh-admin/scratch/branch_protections") return json(res, 200, []);
|
||||
if (method === "GET" && /^\/repos\/[^/]+\/[^/]+\/commits$/.test(path)) {
|
||||
@@ -102,11 +126,19 @@ after(() => new Promise<void>((r) => server.close(() => r())));
|
||||
const { port } = server.address() as { port: number };
|
||||
const client = new GiteaClient(`http://127.0.0.1:${port}`, new ConfiguredToken("t"));
|
||||
// gitea_commit_status is served from the watcher's bundle, beside the merge check's one writer (novox/hq issue 456).
|
||||
const stateDir = mkdtempSync(join(tmpdir(), "gitea-tools-"));
|
||||
// The merge check's one writer, as the watcher's bundle holds it: what the merge tool's pending goes through.
|
||||
const writer = new GateWriter({ setCommitStatus: (o, r, sha, s) => client.setCommitStatus(o, r, sha, s),
|
||||
gateOf: async () => undefined, gateHistory: async () => [], headChecks: async () => ({}) });
|
||||
const throughTheWriter: string[] = [];
|
||||
const gates = { rechecked: (o: string, r: string, sha: string, s: any) => {
|
||||
throughTheWriter.push(`${sha.slice(0, 12)} ${s.context} ${s.state}`);
|
||||
return writer.rechecked(o, r, sha, s);
|
||||
} };
|
||||
const tools = [...getGiteaTools(client), commitStatusTool({
|
||||
setCommitStatus: (o, r, sha, s) => client.setCommitStatus(o, r, sha, s),
|
||||
gates: new GateWriter({ setCommitStatus: (o, r, sha, s) => client.setCommitStatus(o, r, sha, s),
|
||||
gateOf: async () => undefined, gateHistory: async () => [] }),
|
||||
})];
|
||||
setCommitStatus: (o, r, sha, s) => client.setCommitStatus(o, r, sha, s), gates: writer,
|
||||
}), mergeTool(client, { gates, stateDir, emit: async (type, body) => { state.emitted.push({ type, body }); },
|
||||
now: () => new Date(2026, 9, 12, 15, 1) })];
|
||||
const tool = (name: string) => tools.find((t) => t.name === name)!;
|
||||
const writes = () => state.sent.map((s) => `${s.method} ${s.path}`);
|
||||
|
||||
@@ -189,15 +221,15 @@ test("gitea_branch_protection_set only strengthens", async () => {
|
||||
test("gitea_merge_pull_request merges only a head whose required statuses all succeeded, and only that head", async () => {
|
||||
const merge = () => tool("gitea_merge_pull_request").run({ owner: "novox", repo: "mesh-catalog", number: 7,
|
||||
head: REVIEWED });
|
||||
state.statuses = [{ context: "mesh/merge-gate", state: "success" }, { context: "mesh/repo-check", state: "failure" }];
|
||||
state.statuses = [judgedAt(TRUNK)[0], judgedAt(TRUNK, "failure")[1]];
|
||||
await assert.rejects(merge(), /not merged: .*mesh\/repo-check \(failure\)/);
|
||||
state.statuses = [{ context: "mesh/merge-gate", state: "success" }];
|
||||
state.statuses = [judgedAt(TRUNK)[0]];
|
||||
await assert.rejects(merge(), /mesh\/repo-check \(missing\)/);
|
||||
await assert.rejects(tool("gitea_merge_pull_request").run({ owner: "novox", repo: "mesh-catalog", number: 7,
|
||||
head: REVIEWED, method: "manually-merged" }), /not a merge method/);
|
||||
assert.deepEqual(writes(), []);
|
||||
// The operator said yes and the checks passed: the flow that stays.
|
||||
state.statuses = [{ context: "mesh/merge-gate", state: "success" }, { context: "mesh/repo-check", state: "success" }];
|
||||
state.statuses = judgedAt(TRUNK);
|
||||
await merge();
|
||||
const sent = state.sent.find((s) => s.path.endsWith("/pulls/7/merge"))!;
|
||||
assert.equal(sent.body.force_merge, false);
|
||||
@@ -205,6 +237,75 @@ test("gitea_merge_pull_request merges only a head whose required statuses all su
|
||||
});
|
||||
|
||||
|
||||
// **The merge tool refuses a verdict whose trunk commit is not the trunk's head, and asks the check itself** (novox/hq
|
||||
// ADR 0323 §5, issue 341): node-engine #66 merged at 00:54 on a verdict of 00:49 that judged its head alone, after #65
|
||||
// had moved the trunk. Both commits are named; one check is asked, the announcement a new head makes, and not a second
|
||||
// while it is on its way; the refusal says how long the last check of the repository took; and once the verdict judges
|
||||
// the trunk's head and passes, the merge goes through — asked again, never by itself.
|
||||
test("gitea_merge_pull_request refuses a verdict judged against an earlier trunk, and asks the check once", async () => {
|
||||
const merge = () => tool("gitea_merge_pull_request").run({ owner: "novox", repo: "mesh-catalog", number: 7, head: REVIEWED });
|
||||
const { keepTook, tookFile } = await import("../dist/trunk.js");
|
||||
keepTook(tookFile(stateDir), "novox/mesh-catalog", { took: "4m31s", at: "2026-10-12T14:49:00Z" });
|
||||
|
||||
state.trunk = MOVED; // another merge moved the trunk after the verdict
|
||||
await assert.rejects(merge(), (err: Error) => {
|
||||
assert.match(err.message, /is not merged: its verdict was judged against trunk 7a8b9c0d1e2f, and main is now 3c4d5e6f7a8b/);
|
||||
assert.match(err.message, /check of head abc1234def56 merged into trunk 3c4d5e6f7a8b was asked/);
|
||||
assert.match(err.message, /the last check of novox\/mesh-catalog took 4m31s/);
|
||||
assert.match(err.message, /never merges by itself/);
|
||||
return true;
|
||||
});
|
||||
assert.equal(state.merged, false);
|
||||
assert.equal(state.emitted.length, 1, "one check asked");
|
||||
assert.equal(state.emitted[0].type, "pull.updated");
|
||||
assert.equal(state.emitted[0].body.head_sha, REVIEWED);
|
||||
assert.equal(state.emitted[0].body.base, "main");
|
||||
assert.deepEqual(state.emitted[0].body.paths, ["modules/gitea/x.ts"]);
|
||||
// The announcement names the trunk's head, for a verdict no build judges (ADR 0323 §3).
|
||||
assert.equal(state.emitted[0].body.base_sha, MOVED);
|
||||
// Its pending went through the merge check's one writer (novox/hq issue 456), not around it.
|
||||
assert.deepEqual(throughTheWriter.splice(0), ["abc1234def56 mesh/merge-gate pending", "abc1234def56 mesh/repo-check pending"]);
|
||||
const gate = state.statuses.find((s) => s.context === "mesh/merge-gate")!;
|
||||
assert.equal(gate.state, "pending");
|
||||
assert.match(gate.description!, /^waiting in the build queue since 15:01: asked by the merge tool against trunk 3c4d5e6f7a8b/);
|
||||
assert.equal(state.statuses.find((s) => s.context === "mesh/repo-check")!.state, "pending");
|
||||
|
||||
// Asked again while that check is on its way: refused, and nothing is asked a second time.
|
||||
await assert.rejects(merge(), /already asked and not yet answered .*not asked again; the last check of novox\/mesh-catalog took 4m31s.*mesh-delivery\.recheck/);
|
||||
assert.equal(state.emitted.length, 1, "no second check while one is on its way");
|
||||
state.statuses = [{ ...gate, description: "building on ace since 15:02" }, ...state.statuses.slice(1)];
|
||||
await assert.rejects(merge(), /not asked again/);
|
||||
assert.equal(state.emitted.length, 1);
|
||||
|
||||
// A verdict put out of date by the trunk's move is asked too, once.
|
||||
state.statuses = judgedAt(TRUNK).map((s) => ({ ...s, state: "pending",
|
||||
description: "out of date: judged against trunk 7a8b9c0d1e2f, the trunk is now 3c4d5e6f7a8b; judged again on a recheck or when it is merged" }));
|
||||
await assert.rejects(merge(), /was judged against trunk 7a8b9c0d1e2f, and main is now 3c4d5e6f7a8b\. The check .* was asked/);
|
||||
assert.equal(state.emitted.length, 2);
|
||||
|
||||
// A verdict that names no trunk commit — a build seat from before the rule — is not taken as the trunk's head.
|
||||
state.statuses = [{ context: "mesh/merge-gate", state: "success", description: "pass: x" },
|
||||
{ context: "mesh/repo-check", state: "success", description: "pass: y" }];
|
||||
await assert.rejects(merge(), /names no trunk commit it was judged against, and main is now 3c4d5e6f7a8b/);
|
||||
assert.equal(state.emitted.length, 3);
|
||||
assert.equal(state.merged, false);
|
||||
|
||||
// An ask that cannot be announced puts the statuses back to out of date, so the next ask asks again.
|
||||
state.statuses = judgedAt(TRUNK);
|
||||
const failing = mergeTool(client, { gates, stateDir, emit: async () => { throw new Error("the bus is away"); } });
|
||||
await assert.rejects(failing.run({ owner: "novox", repo: "mesh-catalog", number: 7, head: REVIEWED }),
|
||||
/its check could not be asked: the bus is away\. Nothing was merged/);
|
||||
assert.match(state.statuses.find((s) => s.context === "mesh/merge-gate")!.description!, /^out of date: /);
|
||||
await assert.rejects(merge(), /was asked/);
|
||||
assert.equal(state.emitted.length, 4, "asked again after the announcement failed");
|
||||
|
||||
// The check judged the merge into the trunk's head and passed: asked again at the reviewed head, it merges.
|
||||
state.statuses = judgedAt(MOVED);
|
||||
const done = (await merge()) as Record<string, unknown>;
|
||||
assert.equal(done.merged, true);
|
||||
assert.equal(state.emitted.length, 4, "a merge asks no check");
|
||||
});
|
||||
|
||||
// A merge lands only the head that was reviewed (novox/hq issue 381): mesh-controller#197 was reviewed at one head,
|
||||
// force-pushed, checked green at the new one, and merged there, because the tool merged whatever the head was.
|
||||
test("gitea_merge_pull_request merges only the head it is named, and refuses one that is not the head", async () => {
|
||||
|
||||
@@ -0,0 +1,92 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { test } from "node:test";
|
||||
import { createServer } from "node:http";
|
||||
|
||||
// **Every open pull request's new head is announced, however many are open** (novox/hq issue 579). A pass asked the
|
||||
// forge for the 20 most recently updated open pull requests of a repository, so a new head on the 21st or later was
|
||||
// not announced and its merge check never started. A pass now pages through the open pull requests updated since its
|
||||
// last look — every page on a sweep or the first pass — and stops at the first page that reaches back past the floor.
|
||||
|
||||
const HOUR = 3600_000;
|
||||
const T0 = Date.parse("2026-10-12T12:00:00Z");
|
||||
const sha = (n: number, v: string) => `${v}${String(n).padStart(4, "0")}`.padEnd(40, "0");
|
||||
|
||||
/** A forge holding `count` open pull requests of novox/hq, #n updated n hours after T0 (#count the newest), paged and
|
||||
* sorted the way gitea answers `sort=recentupdate`: by update time, newest first. */
|
||||
function forge(count: number, heads: Record<number, string>) {
|
||||
const asked: URLSearchParams[] = [];
|
||||
const server = createServer((req, res) => {
|
||||
const url = new URL(req.url ?? "", "http://x");
|
||||
if (url.pathname !== "/api/v1/repos/novox/hq/pulls") {
|
||||
res.statusCode = 404;
|
||||
return res.end("{}");
|
||||
}
|
||||
asked.push(url.searchParams);
|
||||
const limit = Number(url.searchParams.get("limit") ?? 30);
|
||||
const page = Number(url.searchParams.get("page") ?? 1);
|
||||
const all = Array.from({ length: count }, (_, i) => count - i).map((n) => ({
|
||||
number: n, state: "open", base: { ref: "main" }, head: { ref: `b${n}`, sha: heads[n] ?? sha(n, "a") },
|
||||
updated_at: new Date(T0 + n * HOUR).toISOString(), html_url: `h/${n}`,
|
||||
}));
|
||||
res.end(JSON.stringify(all.slice((page - 1) * limit, page * limit)));
|
||||
});
|
||||
return { server, asked };
|
||||
}
|
||||
|
||||
async function serve(count: number, heads: Record<number, string>) {
|
||||
const { GiteaClient } = await import("../client.ts");
|
||||
const f = forge(count, heads);
|
||||
await new Promise<void>((r) => f.server.listen(0, r));
|
||||
const client = new GiteaClient(`http://127.0.0.1:${(f.server.address() as any).port}`, "t");
|
||||
return { client, ...f };
|
||||
}
|
||||
|
||||
test("a new head beyond the newest 20 open pull requests is announced, exactly once", async () => {
|
||||
const { headsToAnnounce } = await import("../pulls.ts");
|
||||
// 25 open, every head announced before; #3 — the 23rd most recently updated — got a new head.
|
||||
const announced: Record<string, string> = {};
|
||||
for (let n = 1; n <= 25; n++) announced[`novox/hq#${n}`] = sha(n, "a");
|
||||
const { client, server, asked } = await serve(25, { 3: sha(3, "b") });
|
||||
try {
|
||||
// The fault: the newest 20 alone do not hold it.
|
||||
const newest = await client.listPullRequests("novox", "hq", { state: "open", sort: "recentupdate", limit: "20" });
|
||||
assert.deepEqual(headsToAnnounce("novox/hq", newest, announced), [], "the newest 20 miss #3's new head");
|
||||
asked.length = 0;
|
||||
const said: string[] = [];
|
||||
for (let pass = 0; pass < 2; pass++) {
|
||||
// The sweep, as index.ts's pollPulls asks: no floor, every page.
|
||||
const open = await client.listOpenPullsSince("novox", "hq", "");
|
||||
for (const p of headsToAnnounce("novox/hq", open, announced)) {
|
||||
said.push(`#${p.number}@${p.head_sha}`);
|
||||
announced[`novox/hq#${p.number}`] = String(p.head_sha);
|
||||
}
|
||||
}
|
||||
assert.deepEqual(said, [`#3@${sha(3, "b")}`], "the new head past the newest 20, announced once over two passes");
|
||||
assert.ok(asked.every((q) => q.get("state") === "open" && q.get("sort") === "recentupdate"), "asked newest first");
|
||||
} finally {
|
||||
server.close();
|
||||
}
|
||||
});
|
||||
|
||||
test("a pass pages until it reaches back past its floor, and no further", async () => {
|
||||
// 130 open pull requests, #n updated n hours after T0: page 1 holds #130–#81, page 2 #80–#31, page 3 #30–#1.
|
||||
const { client, server, asked } = await serve(130, {});
|
||||
try {
|
||||
// A floor between #60 and #61: page 2 reaches back past it, so page 3 is never asked.
|
||||
const floor = new Date(T0 + 60.5 * HOUR).toISOString();
|
||||
const open = await client.listOpenPullsSince("novox", "hq", floor);
|
||||
assert.deepEqual(asked.map((q) => q.get("page")), ["1", "2"]);
|
||||
assert.equal(open.length, 100);
|
||||
assert.ok(open.some((p) => p.number === 61), "every one updated since the floor is read");
|
||||
// A floor no page reaches past: the last short page ends it.
|
||||
asked.length = 0;
|
||||
assert.equal((await client.listOpenPullsSince("novox", "hq", "")).length, 130);
|
||||
assert.deepEqual(asked.map((q) => q.get("page")), ["1", "2", "3"]);
|
||||
// Every page holds fewer than 50 updated since a recent floor: one page.
|
||||
asked.length = 0;
|
||||
await client.listOpenPullsSince("novox", "hq", new Date(T0 + 129 * HOUR).toISOString());
|
||||
assert.deepEqual(asked.map((q) => q.get("page")), ["1"]);
|
||||
} finally {
|
||||
server.close();
|
||||
}
|
||||
});
|
||||
@@ -0,0 +1,41 @@
|
||||
// A password, a token or an API key is the vault's, never a setting (novox/hq issue 342): a value for it in
|
||||
// the settings file the mesh merges is ignored, the vault's is used, and that is said once, by the key alone.
|
||||
// Run against the compiled module, the way the runtime loads it.
|
||||
|
||||
import { test, mock } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { mkdtempSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
|
||||
import { GiteaClient } from "../dist/client.js";
|
||||
import { ConfiguredToken, MintedToken } from "../dist/token.js";
|
||||
|
||||
const SETTING = "one-a-setting-carried";
|
||||
const VAULT = "the-vault-made-this";
|
||||
|
||||
function delivered(settings: Record<string, string>): Record<string, string> {
|
||||
const dir = mkdtempSync(join(tmpdir(), "gitea-342-"));
|
||||
writeFileSync(join(dir, "config.json"), JSON.stringify(settings));
|
||||
writeFileSync(join(dir, "vault.secret"), VAULT + "\n");
|
||||
return { dir, config: join(dir, "config.json"), vault: join(dir, "vault.secret") };
|
||||
}
|
||||
|
||||
test("the vault's token is used, a setting of it is ignored and said once, without its value", () => {
|
||||
const said: string[] = [];
|
||||
const err = mock.method(console, "error", (...args: unknown[]) => { said.push(args.join(" ")); });
|
||||
try {
|
||||
const at = delivered({ token: SETTING });
|
||||
const env = { MESH_GITEA_CONFIG_FILE: at.config, MESH_GITEA_URL: "http://127.0.0.1:1", MESH_GITEA_ADMIN_USER: "mesh-admin", MESH_GITEA_ADMIN_PASSWORD_FILE: at.vault, MESH_GITEA_STATE_DIR: at.dir };
|
||||
const first = GiteaClient.fromEnv(env);
|
||||
const second = GiteaClient.fromEnv(env);
|
||||
// No token from the setting: the module mints its own with the vault's admin account.
|
||||
assert.ok((first as any).tokens instanceof MintedToken);
|
||||
assert.ok((second as any).tokens instanceof MintedToken);
|
||||
assert.ok(!((first as any).tokens instanceof ConfiguredToken));
|
||||
assert.deepEqual(said, ["the setting token is ignored: it comes from the vault"]);
|
||||
assert.ok(!said.join("\n").includes(SETTING), "the setting's value is never said");
|
||||
} finally {
|
||||
err.mock.restore();
|
||||
}
|
||||
});
|
||||
@@ -414,7 +414,8 @@ test("the tools register once there is a way to a token, and the first call mint
|
||||
[
|
||||
"gitea_list_repos", "gitea_create_repo", "gitea_delete_repo",
|
||||
"gitea_list_issues", "gitea_get_issue", "gitea_create_issue", "gitea_close_issue", "gitea_add_comment",
|
||||
"gitea_list_pull_requests", "gitea_get_pull_request", "gitea_create_pull_request", "gitea_merge_pull_request",
|
||||
// gitea_merge_pull_request is served from the watcher's bundle, beside the merge check's one writer (ADR 0323).
|
||||
"gitea_list_pull_requests", "gitea_get_pull_request", "gitea_create_pull_request",
|
||||
"gitea_close_pull_request",
|
||||
"gitea_reopen_pull_request",
|
||||
"gitea_update_pull_request",
|
||||
|
||||
@@ -0,0 +1,175 @@
|
||||
// **A merge check's verdict holds for the trunk commit it judged** (novox/hq ADR 0323, issue 341): both statuses name
|
||||
// the head and the trunk commit; a trunk's move puts the merge check of every open pull request into that trunk — and
|
||||
// of none into another — back to pending with the reason, and runs nothing; a verdict that lands judged against a trunk
|
||||
// that has moved since is said out of date.
|
||||
|
||||
import { test } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
|
||||
import { checkOnItsWay, lateVerdictComment, outOfDateStatuses, outOfDateWords, trunkNamed, verdictAtTrunk, weighTrunks } from "../dist/trunk.js";
|
||||
import { GateWriter } from "../dist/gate.js";
|
||||
import { statusesFor, commentFor } from "../dist/pulls.js";
|
||||
|
||||
const HEAD = "dfb1415aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa";
|
||||
const TRUNK = "a2c2c35bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb";
|
||||
const MOVED = "d28ca95ccccccccccccccccccccccccccccccccc";
|
||||
|
||||
test("both statuses say the head and the trunk commit the verdict judged, first, whatever the summary's length", () => {
|
||||
const c = { owner: "novox", repo: "mesh-host", number: 66, commit: HEAD, verdict: "pass", summary: "x", id: "build-1",
|
||||
trunk: TRUNK.slice(0, 12), beside: { "mesh-catalog": "0123456789ab", "mesh-lab": "ba9876543210" },
|
||||
gate: { verdict: "pass", summary: "4 of 4 compose ".repeat(20) },
|
||||
"repo-check": { verdict: "fail", summary: "its merge-check.sh failed: internal/store/store.go:145:2: Declared redeclared" } };
|
||||
const [gate, repo] = statusesFor(c);
|
||||
for (const s of [gate, repo]) {
|
||||
assert.ok(s.description.startsWith("head dfb1415aaaaa merged into trunk a2c2c35bbbbb — "), s.description);
|
||||
assert.ok(s.description.length <= 140, `${s.description.length} characters`);
|
||||
assert.equal(trunkNamed(s.description), "a2c2c35bbbbb");
|
||||
}
|
||||
assert.equal(gate.state, "success");
|
||||
assert.equal(repo.state, "failure");
|
||||
const comment = commentFor(c)!;
|
||||
assert.match(comment, /Judged: head `dfb1415aaaaa` merged into trunk `a2c2c35bbbbb`; read beside it: mesh-catalog `0123456789ab`, mesh-lab `ba9876543210`/);
|
||||
|
||||
// A verdict of a build seat from before the rule names no trunk, and is said as before.
|
||||
const [old] = statusesFor({ ...c, trunk: undefined, gate: { verdict: "pass", summary: "every machine composes" } });
|
||||
assert.equal(old.description, "pass: every machine composes");
|
||||
assert.equal(trunkNamed(old.description), undefined);
|
||||
});
|
||||
|
||||
test("a trunk's move puts an open head's merge check out of date, naming both commits, unless a check is on its way", () => {
|
||||
const verdict = (trunk: string, state = "success") => ({
|
||||
"mesh/merge-gate": { state, description: `head ${HEAD.slice(0, 12)} merged into trunk ${trunk.slice(0, 12)} — pass: x` },
|
||||
"mesh/repo-check": { state, description: `head ${HEAD.slice(0, 12)} merged into trunk ${trunk.slice(0, 12)} — pass: y` } });
|
||||
const put = outOfDateStatuses(verdict(TRUNK), MOVED);
|
||||
assert.deepEqual(put.map((s) => [s.context, s.state]), [["mesh/merge-gate", "pending"], ["mesh/repo-check", "pending"]]);
|
||||
assert.equal(put[0].description, "out of date: judged against trunk a2c2c35bbbbb, the trunk is now d28ca95ccccc; " +
|
||||
"judged again on a recheck or when it is merged");
|
||||
assert.equal(put[0].description, outOfDateWords(TRUNK, MOVED));
|
||||
assert.equal(trunkNamed(put[0].description), "a2c2c35bbbbb", "an out-of-date status still names what it judged");
|
||||
// A failure is out of date too: it judged another merge.
|
||||
assert.equal(outOfDateStatuses(verdict(TRUNK, "failure"), MOVED).length, 2);
|
||||
// Already judged against the trunk's head; a check on its way or already out of date; no check yet: left alone.
|
||||
assert.deepEqual(outOfDateStatuses(verdict(MOVED), MOVED), []);
|
||||
assert.deepEqual(outOfDateStatuses({ "mesh/merge-gate": { state: "pending", description: "building on ace since 14:02" } }, MOVED), []);
|
||||
assert.deepEqual(outOfDateStatuses({ "mesh/merge-gate": { state: "pending", description: put[0].description } }, MOVED), []);
|
||||
assert.deepEqual(outOfDateStatuses({}, MOVED), []);
|
||||
// A head with no repository check of its own gets no pending one a verdict would never set.
|
||||
assert.deepEqual(outOfDateStatuses({ "mesh/merge-gate": verdict(TRUNK)["mesh/merge-gate"] }, MOVED).map((s) => s.context),
|
||||
["mesh/merge-gate"]);
|
||||
// A verdict naming no trunk, of a build seat from before the rule, judged an earlier trunk.
|
||||
assert.match(outOfDateStatuses({ "mesh/merge-gate": { state: "success", description: "pass: x" } }, MOVED)[0].description,
|
||||
/judged against a trunk it does not name, the trunk is now d28ca95ccccc/);
|
||||
|
||||
assert.ok(checkOnItsWay({ state: "pending", description: "waiting in the build queue since 14:00" }));
|
||||
assert.ok(checkOnItsWay({ state: "pending", description: "checking again: a person asked" }));
|
||||
assert.ok(!checkOnItsWay({ state: "pending", description: put[0].description }));
|
||||
assert.deepEqual(verdictAtTrunk(verdict(MOVED), MOVED), { current: true, onItsWay: false, judged: "d28ca95ccccc" });
|
||||
assert.equal(verdictAtTrunk(verdict(TRUNK), MOVED).current, false);
|
||||
});
|
||||
|
||||
/** A forge holding open pull requests into two branches, each head's statuses, and what was written. */
|
||||
function aForge(pulls: { number: number; base: string; head: string }[], heads: Record<string, string>) {
|
||||
const statuses = new Map<string, Record<string, { state: string; description?: string }>>();
|
||||
const writes: string[] = [];
|
||||
const verdict = (trunk: string) => ({
|
||||
"mesh/merge-gate": { state: "success", description: `head x merged into trunk ${trunk.slice(0, 12)} — pass` },
|
||||
"mesh/repo-check": { state: "success", description: `head x merged into trunk ${trunk.slice(0, 12)} — pass` } });
|
||||
for (const p of pulls) statuses.set(p.head, verdict(heads[p.base]));
|
||||
const forge = {
|
||||
listOpenPulls: async () => pulls.map((p) => ({ number: p.number, title: "t", state: "open", merged: false, base: p.base,
|
||||
head: "feat", head_sha: p.head, html_url: `http://forge/pulls/${p.number}` })),
|
||||
branchHead: async (_o: string, _r: string, branch: string) => heads[branch],
|
||||
setCommitStatus: async (_o: string, _r: string, sha: string, s: { context: string; state: string; description: string }) => {
|
||||
writes.push(`${sha} ${s.context} ${s.state}`);
|
||||
statuses.set(sha, { ...statuses.get(sha), [s.context]: { state: s.state, description: s.description } });
|
||||
},
|
||||
gateOf: async () => undefined,
|
||||
gateHistory: async () => [],
|
||||
headChecks: async (_o: string, _r: string, sha: string) => statuses.get(sha) ?? {},
|
||||
};
|
||||
return { forge, writes, statuses };
|
||||
}
|
||||
|
||||
test("a push to a trunk puts every open pull request into it out of date — twenty-five of them — and none into another", async () => {
|
||||
const heads: Record<string, string> = { main: TRUNK, release: "eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee" };
|
||||
const pulls = Array.from({ length: 25 }, (_, i) => ({ number: i + 1, base: "main", head: `${String(i).padStart(2, "0")}`.repeat(20) }));
|
||||
pulls.push({ number: 99, base: "release", head: "9".repeat(40) });
|
||||
const { forge, writes, statuses } = aForge(pulls, heads);
|
||||
const gates = new GateWriter(forge);
|
||||
const trunks: Record<string, string> = {};
|
||||
|
||||
// First sight: each trunk is taken as it is, and nothing is put out of date.
|
||||
assert.equal(await weighTrunks(forge, gates, trunks, "novox", "mesh-host"), true);
|
||||
assert.deepEqual(writes, []);
|
||||
assert.equal(trunks["novox/mesh-host:main"], TRUNK);
|
||||
// Nothing moved: nothing read past the trunks, nothing written.
|
||||
assert.equal(await weighTrunks(forge, gates, trunks, "novox", "mesh-host"), false);
|
||||
|
||||
heads.main = MOVED;
|
||||
assert.equal(await weighTrunks(forge, gates, trunks, "novox", "mesh-host"), true);
|
||||
assert.equal(writes.length, 50, "both statuses of each of the 25 pull requests into main");
|
||||
for (const p of pulls.filter((p) => p.base === "main")) {
|
||||
const s = statuses.get(p.head)!;
|
||||
assert.equal(s["mesh/merge-gate"].state, "pending");
|
||||
assert.match(s["mesh/merge-gate"].description!, /judged against trunk a2c2c35bbbbb, the trunk is now d28ca95ccccc/);
|
||||
assert.equal(s["mesh/repo-check"].state, "pending");
|
||||
}
|
||||
assert.equal(statuses.get("9".repeat(40))!["mesh/merge-gate"].state, "success", "a pull request into another branch is left");
|
||||
assert.equal(trunks["novox/mesh-host:main"], MOVED);
|
||||
|
||||
// Weighed again, nothing more: the move was taken once, and the out-of-date statuses are left as they are.
|
||||
writes.length = 0;
|
||||
heads.main = "f".repeat(40);
|
||||
await weighTrunks(forge, gates, trunks, "novox", "mesh-host");
|
||||
assert.deepEqual(writes, [], "a check already out of date is not written again");
|
||||
});
|
||||
|
||||
test("a trunk that cannot be read is weighed again on the next pass, never recorded half", async () => {
|
||||
const heads: Record<string, string> = { main: TRUNK };
|
||||
const { forge } = aForge([{ number: 1, base: "main", head: "1".repeat(40) }], heads);
|
||||
const trunks: Record<string, string> = { "novox/mesh-host:main": TRUNK };
|
||||
heads.main = MOVED;
|
||||
const failing = { ...forge, setCommitStatus: async () => { throw new Error("the forge is away"); } };
|
||||
assert.equal(await weighTrunks(failing, new GateWriter(failing), trunks, "novox", "mesh-host"), false);
|
||||
assert.equal(trunks["novox/mesh-host:main"], TRUNK, "not recorded: the next pass weighs it again");
|
||||
});
|
||||
|
||||
test("the trunk's move takes its turn after a verdict in flight, and weighs what the forge holds then", async () => {
|
||||
const heads: Record<string, string> = { main: TRUNK };
|
||||
const { forge, statuses } = aForge([{ number: 1, base: "main", head: HEAD }], heads);
|
||||
const gates = new GateWriter(forge);
|
||||
// A verdict judged against the trunk's new head, in flight when the move is weighed: it lands first, and is left.
|
||||
let release!: () => void;
|
||||
const held = new Promise<void>((r) => (release = r));
|
||||
const verdictLanded = gates.verdict("novox", "mesh-host", HEAD, async () => {
|
||||
await held;
|
||||
statuses.set(HEAD, { "mesh/merge-gate": { state: "success", description: `head x merged into trunk ${MOVED.slice(0, 12)} — pass` } });
|
||||
});
|
||||
const moved = gates.outOfDate("novox", "mesh-host", HEAD, (checks) => outOfDateStatuses(checks, MOVED));
|
||||
release();
|
||||
await verdictLanded;
|
||||
assert.deepEqual(await moved, [], "a verdict judged against the trunk's head is not put out of date");
|
||||
assert.equal(statuses.get(HEAD)!["mesh/merge-gate"].state, "success");
|
||||
});
|
||||
|
||||
test("a verdict judged against a trunk newer than the pass's first read is not put out of date", async () => {
|
||||
const NEWER = "ffffffffffffffffffffffffffffffffffffffff";
|
||||
const heads: Record<string, string> = { main: TRUNK };
|
||||
const { forge, statuses } = aForge([{ number: 1, base: "main", head: HEAD }], heads);
|
||||
const trunks: Record<string, string> = { "novox/mesh-host:main": TRUNK };
|
||||
// The pass reads main at MOVED; by the head's turn the trunk moved again, and the head's check answered for it.
|
||||
let reads = 0;
|
||||
const moving = { ...forge, branchHead: async () => (reads++ === 0 ? MOVED : NEWER) };
|
||||
statuses.set(HEAD, { "mesh/merge-gate": { state: "success", description: `head x merged into trunk ${NEWER.slice(0, 12)} — pass` } });
|
||||
await weighTrunks(moving, new GateWriter(moving), trunks, "novox", "mesh-host");
|
||||
assert.equal(statuses.get(HEAD)!["mesh/merge-gate"].state, "success", "a fresh verdict was put out of date");
|
||||
});
|
||||
|
||||
test("a verdict that landed out of date keeps its account on the pull request", () => {
|
||||
const words = outOfDateWords(TRUNK, MOVED);
|
||||
const own = lateVerdictComment({ commit: HEAD, trunk: TRUNK.slice(0, 12), verdict: "fail" }, "**Merge check** at `dfb1415a`\n\nthe report", words);
|
||||
assert.match(own, /the report/);
|
||||
assert.match(own, /\*\*Out of date when it landed\*\*: out of date: judged against trunk a2c2c35bbbbb/);
|
||||
const bare = lateVerdictComment({ commit: HEAD, trunk: TRUNK.slice(0, 12), verdict: "pass" }, null, words);
|
||||
assert.match(bare, /head dfb1415aaaaa merged into trunk a2c2c35bbbbb, pass/);
|
||||
});
|
||||
@@ -63,6 +63,8 @@ export function getGiteaTools(gitea: GiteaClient): ToolDefinition[] {
|
||||
return namesChecked(allGiteaTools(gitea));
|
||||
}
|
||||
|
||||
// gitea_merge_pull_request is served from the watcher's bundle (merge.ts, index.ts), beside the merge check's one
|
||||
// writer (novox/hq issue 456, ADR 0323).
|
||||
function allGiteaTools(gitea: GiteaClient): ToolDefinition[] {
|
||||
return [
|
||||
// ---- Repositories ----
|
||||
@@ -263,111 +265,6 @@ function allGiteaTools(gitea: GiteaClient): ToolDefinition[] {
|
||||
}),
|
||||
}),
|
||||
},
|
||||
{
|
||||
name: "gitea_merge_pull_request",
|
||||
description: "Merge a pull request at the head you name — the commit you reviewed — when that is still its head " +
|
||||
"and every status its base branch's protection requires succeeded there (mesh/merge-gate, mesh/repo-check), " +
|
||||
"optionally deleting the source branch afterwards. Refused otherwise, with nothing merged: no tool merges past " +
|
||||
"a check, and none merges a head that moved after it was reviewed.",
|
||||
input: {
|
||||
owner: { type: "string", description: "the repository owner" },
|
||||
repo: { type: "string", description: "the repository name" },
|
||||
number: { type: "number", description: "the PR number" },
|
||||
head: { type: "string", description: "the commit you reviewed, as its sha or at least its first 12 characters: " +
|
||||
"refused unless it is the pull request's head" },
|
||||
method: { type: "string", description: "merge (default) | fast-forward-only | squash | rebase | rebase-merge. " +
|
||||
"Only merge and fast-forward-only can be read back to prove the head you named is what landed" },
|
||||
delete_branch: { type: "boolean", description: "delete the source branch after merge (default true)" },
|
||||
},
|
||||
run: async (args) => {
|
||||
const owner = String(args.owner);
|
||||
const repo = String(args.repo);
|
||||
const number = Number(args.number);
|
||||
const method = args.method ? String(args.method) : "merge";
|
||||
if (!MERGE_METHODS.has(method)) throw new Refused(`${method} is not a merge method: ${[...MERGE_METHODS].join(", ")}`);
|
||||
const deleteBranch = args.delete_branch === undefined ? true : Boolean(args.delete_branch);
|
||||
// **Only a head whose required statuses all succeeded** (novox/hq ADR 0266): read here rather than left
|
||||
// to the forge, whose administrator — the account this tool's token is minted for — may merge past a
|
||||
// rule that lets it. A rule that cannot be read is no rule passed.
|
||||
// **Only the head that was reviewed** (novox/hq issue 381): mesh-controller#197 was reviewed at 065cfa0d,
|
||||
// force-pushed to 3ced96f, checked green there, and merged at 3ced96f, because this tool merged whatever the
|
||||
// head was and nothing told it which head the review had approved. The caller names it; a head that moved
|
||||
// since is refused, and the forge is handed the same head, so a push between this read and the merge is
|
||||
// refused by the forge too.
|
||||
const named = args.head === undefined ? "" : String(args.head).trim().toLowerCase();
|
||||
if (!named) {
|
||||
throw new Refused(`${owner}/${repo}#${number} is not merged: name the head you reviewed (head), so that only ` +
|
||||
`that commit can land (novox/hq issue 381). Nothing was merged`);
|
||||
}
|
||||
// Twelve characters, not seven: an author who force-pushes can make a commit whose first seven match the
|
||||
// reviewed one in seconds, and the review would then merge it. Twelve cannot be matched that way.
|
||||
if (!REVIEWED_HEAD.test(named)) {
|
||||
throw new Refused(`${owner}/${repo}#${number} is not merged: head ${named} does not name one commit — give its ` +
|
||||
`sha, or at least its first 12 characters. Nothing was merged`);
|
||||
}
|
||||
const pull = await gitea.getPullRequest(owner, repo, number);
|
||||
if (pull.merged) throw new Refused(`${owner}/${repo}#${number} is already merged`);
|
||||
if (!pull.head_sha || !pull.base) throw new Refused(`${owner}/${repo}#${number} names no head commit or base branch`);
|
||||
if (!pull.head_sha.toLowerCase().startsWith(named)) {
|
||||
throw new Refused(`${owner}/${repo}#${number} is not merged: its head is ${pull.head_sha.slice(0, 12)}, not the ` +
|
||||
`${named} you named. If you reviewed an earlier head, review the one it has now, then name it. Nothing was merged`);
|
||||
}
|
||||
const rule = await gitea.branchProtection(owner, repo, pull.base);
|
||||
// Only into a branch whose protection requires the checks: a pull request whose base was moved to a
|
||||
// branch nothing guards would merge past every check (novox/hq ADR 0266).
|
||||
if (!rule || !rule.enable_status_check || !(rule.status_check_contexts ?? []).filter(Boolean).length) {
|
||||
throw new Refused(`${owner}/${repo}#${number} merges into ${pull.base}, which no protection rule requiring ` +
|
||||
`the checks guards: a tool merges only into such a branch (novox/hq ADR 0266). Nothing was merged`);
|
||||
}
|
||||
const statuses = newestByContext((await gitea.combinedStatus(owner, repo, pull.head_sha)).statuses);
|
||||
const verdict = mergeableUnder(pull.base, rule, statuses);
|
||||
if (verdict.mergeable !== true) {
|
||||
throw new Refused(`${owner}/${repo}#${number} is not merged: ${verdict.says}. A pull request merges once ` +
|
||||
`its checks succeed; no tool merges past them (novox/hq ADR 0266)`);
|
||||
}
|
||||
await gitea.mergePullRequest(owner, repo, number, method, deleteBranch, pull.head_sha);
|
||||
// The merge commit only exists now; answered so the caller can follow what is built from it.
|
||||
// pull.merged is the events entrypoint's to announce (index.ts), once, within its poll.
|
||||
// From here the merge has happened. Nothing below may read as if it had not: a read-back that fails or cannot
|
||||
// be done is said beside merged: true, and only a proven other head is an error — one that says the merge happened.
|
||||
const answer: Record<string, unknown> = { merged: true, number, method, head: pull.head_sha, deleted_branch: deleteBranch };
|
||||
let landed: string | undefined;
|
||||
try {
|
||||
const merged = await gitea.getPullRequest(owner, repo, number);
|
||||
answer.merge_commit_sha = merged.merge_commit_sha;
|
||||
// What landed, read back from the commit the merge made rather than from the forge's record of the pull
|
||||
// request, which is the very thing a race would leave behind. A merge commit's second parent is the head it
|
||||
// joined; a fast-forward lands the head itself. A squash or a rebase makes new commits that name no head.
|
||||
if (!merged.merge_commit_sha) {
|
||||
answer.read_back = "the forge named no merge commit, so what landed cannot be read back";
|
||||
} else if (method === "merge") {
|
||||
const parents = await gitea.commitParents(owner, repo, merged.merge_commit_sha);
|
||||
if (parents.length === 2) landed = parents[1];
|
||||
else answer.read_back = `the merge commit has ${parents.length} parent(s), not two: what landed cannot be read back`;
|
||||
} else if (method === "fast-forward-only") {
|
||||
// That the forge names the head itself as a fast-forward's merge commit is assumed, not seen on this forge:
|
||||
// a different sha is said, not raised as another head having landed.
|
||||
if (merged.merge_commit_sha === pull.head_sha) landed = merged.merge_commit_sha;
|
||||
else answer.read_back = `the forge named ${merged.merge_commit_sha.slice(0, 12)} as this fast-forward's commit, ` +
|
||||
`not the head: that the head landed could not be confirmed`;
|
||||
} else {
|
||||
answer.read_back = `a ${method} makes new commits that name no head: what landed cannot be read back`;
|
||||
}
|
||||
} catch (err) {
|
||||
answer.read_back = `the merge happened, but reading back what landed failed: ${(err as Error).message}`;
|
||||
}
|
||||
// Loud, not a field beside merged: true that a caller reading only that would miss. The merge cannot be taken
|
||||
// back by this tool; what must not happen is that it reads as the merge that was asked for.
|
||||
if (landed !== undefined && landed !== pull.head_sha) {
|
||||
throw new Error(`${owner}/${repo}#${number} WAS merged (${answer.merge_commit_sha}), but the forge merged ` +
|
||||
`${landed.slice(0, 12)}, not the ${pull.head_sha.slice(0, 12)} it was handed and you reviewed: a commit nobody ` +
|
||||
`reviewed has landed. Read it before anything is built from it (novox/hq issue 381)`);
|
||||
}
|
||||
if (landed !== undefined) answer.landed = landed;
|
||||
return answer;
|
||||
},
|
||||
},
|
||||
|
||||
{
|
||||
name: "gitea_close_pull_request",
|
||||
description: "Close a pull request without merging it — one whose work landed elsewhere, or was abandoned.",
|
||||
|
||||
@@ -0,0 +1,250 @@
|
||||
// **A merge check's verdict holds for the trunk commit it judged** (novox/hq ADR 0323, issue 341).
|
||||
//
|
||||
// The build seat checks a pull request's head merged into its trunk's head at that moment — the verdict's trunk
|
||||
// commit — and the verdict names both: "head 1a2b3c4d5e6f merged into trunk 7a8b9c0d1e2f", in both statuses'
|
||||
// descriptions. When the trunk moves, every open pull request's verdict into it is out of date:
|
||||
//
|
||||
// - **the forge's holder** (index.ts) hears the move on its own — it reads each trunk's head on every pass that
|
||||
// looks at the repository, never through mesh-delivery (ADR 0239 §36) — and sets mesh/merge-gate and
|
||||
// mesh/repo-check on the head of every open pull request into that trunk back to pending, with the reason.
|
||||
// Nothing is run for it: a recheck, a new head, or the merge tool asks the check again;
|
||||
// - **the merge tool** (tools/index.ts) compares the trunk commit the head's verdict names with the trunk's head at
|
||||
// the moment it is asked, merges nothing when they differ, asks the head's check once — the same announcement a
|
||||
// new head makes — and says how long the last check of that repository took. It never merges by itself after.
|
||||
//
|
||||
// Pure functions and two small files of the module's own state, so this is tested without a forge.
|
||||
|
||||
import { existsSync, mkdirSync, readFileSync, renameSync, writeFileSync } from "node:fs";
|
||||
import { dirname } from "node:path";
|
||||
import type { CommitStatus, GiteaClient, GiteaPull, HeadCheck } from "./client.js";
|
||||
import { CHECK_CONTEXT, REPO_CHECK_CONTEXT } from "./pulls.js";
|
||||
|
||||
/** A commit as a verdict names it: its first twelve characters. */
|
||||
export function twelve(sha: string | undefined): string {
|
||||
return String(sha ?? "").slice(0, 12).toLowerCase();
|
||||
}
|
||||
|
||||
/** Whether two commits are one, however either is abbreviated (seven characters at least). */
|
||||
export function sameCommit(a: string | undefined, b: string | undefined): boolean {
|
||||
const x = String(a ?? "").toLowerCase(), y = String(b ?? "").toLowerCase();
|
||||
if (x.length < 7 || y.length < 7) return false;
|
||||
return x.length <= y.length ? y.startsWith(x) : x.startsWith(y);
|
||||
}
|
||||
|
||||
/** What a verdict judged, as both statuses' descriptions begin (ADR 0323 §3). */
|
||||
export function judgedWords(head: string, trunk: string): string {
|
||||
return `head ${twelve(head)} merged into trunk ${twelve(trunk)}`;
|
||||
}
|
||||
|
||||
/** The trunk commit a merge check's status names: the one its verdict judged ("merged into trunk …"), or, for a
|
||||
* verdict put out of date, the one it had judged ("judged against trunk …"). Undefined when it names none, as a
|
||||
* verdict of a build seat from before ADR 0323 does. */
|
||||
export function trunkNamed(description: string | undefined): string | undefined {
|
||||
const m = /(?:merged into|judged against) trunk ([0-9a-f]{7,64})\b/i.exec(description ?? "");
|
||||
return m ? m[1].toLowerCase() : undefined;
|
||||
}
|
||||
|
||||
/** A verdict out of date, as its statuses say it (ADR 0323 §4). */
|
||||
export function outOfDateWords(judged: string | undefined, now: string): string {
|
||||
const was = judged ? `judged against trunk ${twelve(judged)}` : "judged against a trunk it does not name";
|
||||
return `out of date: ${was}, the trunk is now ${twelve(now)}; judged again on a recheck or when it is merged`;
|
||||
}
|
||||
|
||||
/** Whether a pending gate says a check of the head is asked and not yet answered: waiting in the build queue,
|
||||
* building on a machine (gate.ts), or asked again by a recheck (mesh-delivery). Such a check is not asked again. */
|
||||
export function checkOnItsWay(gate: HeadCheck | undefined): boolean {
|
||||
if (!gate || gate.state !== "pending") return false;
|
||||
const d = gate.description ?? "";
|
||||
return d.startsWith("waiting in the build queue") || d.startsWith("building on ") || d.startsWith("checking again");
|
||||
}
|
||||
|
||||
/**
|
||||
* The statuses a trunk's move puts back to pending on one open pull request's head, from what the forge holds on it
|
||||
* now: none when it has no merge check yet, when a check is pending (on its way, or already out of date — its
|
||||
* verdict is weighed against the trunk when it lands), or when its verdict judged the trunk's head. Otherwise the
|
||||
* gate, and the repository's own check when the head has one — a pending repo-check a verdict will not set would
|
||||
* stand for ever (novox/hq issue 305's rule).
|
||||
*/
|
||||
export function outOfDateStatuses(checks: Record<string, HeadCheck>, trunkHead: string): CommitStatus[] {
|
||||
const gate = checks[CHECK_CONTEXT];
|
||||
if (!gate || gate.state === "pending") return [];
|
||||
const judged = trunkNamed(gate.description);
|
||||
if (judged && sameCommit(judged, trunkHead)) return [];
|
||||
const description = outOfDateWords(judged, trunkHead);
|
||||
const out: CommitStatus[] = [{ state: "pending", context: CHECK_CONTEXT, description }];
|
||||
if (checks[REPO_CHECK_CONTEXT]) out.push({ state: "pending", context: REPO_CHECK_CONTEXT, description });
|
||||
return out;
|
||||
}
|
||||
|
||||
/** Where the merge tool stands with a head's verdict and the trunk's head (ADR 0323 §5). */
|
||||
export interface VerdictAtTrunk {
|
||||
/** The verdict judged the trunk's head: the required statuses decide, as before. */
|
||||
current: boolean;
|
||||
/** A check of the head is asked and not yet answered: nothing is asked again. */
|
||||
onItsWay: boolean;
|
||||
/** The trunk commit the verdict names, when it names one. */
|
||||
judged?: string;
|
||||
}
|
||||
|
||||
export function verdictAtTrunk(checks: Record<string, HeadCheck>, trunkHead: string): VerdictAtTrunk {
|
||||
const gate = checks[CHECK_CONTEXT];
|
||||
const judged = trunkNamed(gate?.description);
|
||||
if (checkOnItsWay(gate)) return { current: false, onItsWay: true, judged };
|
||||
return { current: !!gate && gate.state !== "pending" && !!judged && sameCommit(judged, trunkHead), onItsWay: false, judged };
|
||||
}
|
||||
|
||||
/** The pending words the merge tool's ask of a check is said with: in the build queue from now, as an announced head's. */
|
||||
export function askedWords(at: Date, trunkHead: string): string {
|
||||
const two = (n: number) => String(n).padStart(2, "0");
|
||||
return `waiting in the build queue since ${two(at.getHours())}:${two(at.getMinutes())}: asked by the merge tool ` +
|
||||
`against trunk ${twelve(trunkHead)}`;
|
||||
}
|
||||
|
||||
/** The comment a verdict that landed out of date leaves (ADR 0323 §7: kept and shown): its own account when it has
|
||||
* one, else what it judged and said, and that it is out of date. */
|
||||
export function lateVerdictComment(c: { commit: string; trunk?: string; verdict: string }, account: string | null,
|
||||
outOfDate: string): string {
|
||||
const own = account ?? `**Merge check** at \`${c.commit.slice(0, 8)}\`: ${judgedWords(c.commit, c.trunk ?? "")}, ${c.verdict}`;
|
||||
return `${own}\n\n**Out of date when it landed**: ${outOfDate}`;
|
||||
}
|
||||
|
||||
// --- a trunk's move, weighed --------------------------------------------------------------------------------------
|
||||
|
||||
/** What weighing a trunk's move reads of the forge. */
|
||||
export interface TrunkForge {
|
||||
listOpenPulls(owner: string, repo: string): Promise<GiteaPull[]>;
|
||||
branchHead(owner: string, repo: string, branch: string): Promise<string>;
|
||||
}
|
||||
|
||||
/** What it writes with: the merge check's one writer (gate.ts), which decides in the head's own turn. */
|
||||
export interface OutOfDateWriter {
|
||||
outOfDate(owner: string, repo: string, sha: string,
|
||||
decide: (checks: Record<string, HeadCheck>) => CommitStatus[] | Promise<CommitStatus[]>, target?: string): Promise<CommitStatus[]>;
|
||||
}
|
||||
|
||||
/**
|
||||
* Every trunk of one repository that an open pull request merges into, read; for one that moved since it was last read
|
||||
* (`trunks`, by `owner/repo:branch`, which this keeps current), the merge check of every open pull request into it put
|
||||
* out of date (ADR 0323 §4). A trunk first seen is recorded as it is and nothing is put out of date. Bounded however
|
||||
* many are open: one listing of the open pull requests (paged), one read of each trunk, then per open pull request into
|
||||
* a trunk that moved one read and at most two writes, one at a time. A trunk that could not be weighed whole is not
|
||||
* recorded, so the next pass weighs it again. Answers whether the record changed.
|
||||
*/
|
||||
export async function weighTrunks(forge: TrunkForge, gates: OutOfDateWriter, trunks: Record<string, string>, owner: string,
|
||||
name: string): Promise<boolean> {
|
||||
let changed = false;
|
||||
try {
|
||||
const open = (await forge.listOpenPulls(owner, name)).filter((p) => p.state === "open" && !!p.head_sha && !!p.base);
|
||||
for (const base of new Set(open.map((p) => String(p.base)))) {
|
||||
const key = `${owner}/${name}:${base}`.toLowerCase();
|
||||
const head = await forge.branchHead(owner, name, base);
|
||||
const before = trunks[key];
|
||||
if (before && sameCommit(before, head)) continue;
|
||||
if (before) {
|
||||
let put = 0;
|
||||
for (const pull of open.filter((p) => p.base === base)) {
|
||||
// The trunk is read again in the head's own turn: a verdict judged against a head newer than the pass's first
|
||||
// read — the trunk moved again meanwhile, and a check answered — is not put out of date.
|
||||
const written = await gates.outOfDate(owner, name, String(pull.head_sha),
|
||||
async (checks) => outOfDateStatuses(checks, await forge.branchHead(owner, name, base)), pull.html_url || undefined);
|
||||
if (written.length) put++;
|
||||
}
|
||||
console.log(`[gitea] ${owner}/${name}'s ${base} moved from ${twelve(before)} to ${twelve(head)}: ${put} open pull ` +
|
||||
`request(s) into it put out of date, nothing run (novox/hq ADR 0323)`);
|
||||
}
|
||||
trunks[key] = head;
|
||||
changed = true;
|
||||
}
|
||||
} catch (err) {
|
||||
console.error(`[gitea] ${owner}/${name}: whether a trunk moved could not be weighed, so it is weighed again on the next ` +
|
||||
`pass — ${err instanceof Error ? err.message : err}`);
|
||||
}
|
||||
return changed;
|
||||
}
|
||||
|
||||
// --- how long the last check of a repository took -------------------------------------------------------------
|
||||
|
||||
/** How long one repository's last check took, and when it was heard: kept by the watcher, read by the merge tool. */
|
||||
export interface Took {
|
||||
took: string;
|
||||
at: string;
|
||||
head?: string;
|
||||
}
|
||||
|
||||
/** The file the watcher keeps each repository's last check's duration in, beside its other records. */
|
||||
export function tookFile(stateDir: string | undefined): string | null {
|
||||
return stateDir ? `${stateDir.replace(/\/+$/, "")}/checks-took.json` : null;
|
||||
}
|
||||
|
||||
function readTookAll(file: string | null): Record<string, Took> {
|
||||
if (!file || !existsSync(file)) return {};
|
||||
try {
|
||||
return JSON.parse(readFileSync(file, "utf8")) as Record<string, Took>;
|
||||
} catch {
|
||||
return {};
|
||||
}
|
||||
}
|
||||
|
||||
/** Keep how long a repository's check took. A record that cannot be written is said, never fatal to the verdict. */
|
||||
export function keepTook(file: string | null, repository: string, took: Took): void {
|
||||
if (!file || !took.took) return;
|
||||
try {
|
||||
const all = readTookAll(file);
|
||||
all[repository.toLowerCase()] = took;
|
||||
mkdirSync(dirname(file), { recursive: true });
|
||||
writeFileSync(file + ".tmp", JSON.stringify(all));
|
||||
renameSync(file + ".tmp", file);
|
||||
} catch (err) {
|
||||
console.error(`[gitea] how long ${repository}'s check took could not be kept — ${err instanceof Error ? err.message : err}`);
|
||||
}
|
||||
}
|
||||
|
||||
/** How long a repository's last check took, in words for the merge tool's refusal. */
|
||||
export function tookWords(file: string | null, repository: string): string {
|
||||
const t = readTookAll(file)[repository.toLowerCase()];
|
||||
if (!t) return `how long a check of ${repository} takes is not known yet: no verdict of it was heard since this was kept`;
|
||||
return `the last check of ${repository} took ${t.took} (heard ${t.at})`;
|
||||
}
|
||||
|
||||
// --- the announcement of a head, shared by the watcher and the merge tool ---------------------------------------
|
||||
|
||||
/** `pull.updated` for one open pull request's head: what the controller asks the build seat to check (novox/hq to-be
|
||||
* 45 §9). The merge tool's ask is this same announcement (ADR 0323 §5). */
|
||||
export async function announcementOf(client: GiteaClient, owner: string, name: string, cloneURL: string | undefined,
|
||||
pull: GiteaPull): Promise<Record<string, unknown>> {
|
||||
const files = await client.listPullFiles(owner, name, pull.number);
|
||||
const head = String(pull.head_sha);
|
||||
// What the controller maps the change onto the mesh's module graph with (novox/hq ADR 0237 as amended): which
|
||||
// changed directories hold a module at the head — the merge's rule (issue 278) — and whether the head holds the
|
||||
// repository's own merge-check.sh. Not said when it could not be read; the controller then reads the change as
|
||||
// touching everything built from the repository.
|
||||
let moduleDirs: string[] | null = null;
|
||||
let mergeCheck: boolean | null = null;
|
||||
try {
|
||||
if (!files.truncated) moduleDirs = await client.moduleDirsAt(owner, name, head, files.paths);
|
||||
mergeCheck = await client.holdsFile(owner, name, head, "merge-check.sh");
|
||||
} catch (err) {
|
||||
console.error(`[gitea] ${owner}/${name}#${pull.number}: what the head holds could not be read — ${err instanceof Error ? err.message : err}`);
|
||||
}
|
||||
// The base branch's head as the head is announced: the trunk commit a verdict no build judges is said against
|
||||
// (novox/hq ADR 0323 §3). Not said when it cannot be read; that verdict then names none.
|
||||
const baseSHA = pull.base ? await client.branchHead(owner, name, String(pull.base)).catch(() => undefined) : undefined;
|
||||
return {
|
||||
owner,
|
||||
repo: name,
|
||||
number: pull.number,
|
||||
...(baseSHA ? { base_sha: baseSHA } : {}),
|
||||
title: pull.title,
|
||||
body: pull.body,
|
||||
base: pull.base,
|
||||
head: pull.head,
|
||||
head_sha: pull.head_sha,
|
||||
clone_url: cloneURL,
|
||||
html_url: pull.html_url,
|
||||
paths: files.paths,
|
||||
paths_truncated: files.truncated,
|
||||
removed: files.removed,
|
||||
...(moduleDirs ? { module_dirs: moduleDirs, module_dirs_said: true } : {}),
|
||||
...(mergeCheck !== null ? { merge_check: mergeCheck, merge_check_said: true } : {}),
|
||||
};
|
||||
}
|
||||
@@ -44,6 +44,17 @@ function meshConfig(file?: string): Record<string, string> {
|
||||
catch { return {}; }
|
||||
}
|
||||
|
||||
/**
|
||||
* A password, token or API key is the vault's, never a setting (novox/hq issue 342): a value for one of
|
||||
* them in the settings file is ignored, and that is said once — by its key, never with its value.
|
||||
*/
|
||||
const saidIgnored = new Set<string>();
|
||||
function ignoreSecretSetting(cfg: Record<string, unknown>, key: string): void {
|
||||
if (cfg[key] === undefined || saidIgnored.has(key)) return;
|
||||
saidIgnored.add(key);
|
||||
console.error(`the setting ${key} is ignored: it comes from the vault`);
|
||||
}
|
||||
|
||||
export class GrafanaClient {
|
||||
readonly baseUrl: string;
|
||||
private readonly authHeader: string;
|
||||
@@ -54,21 +65,24 @@ export class GrafanaClient {
|
||||
}
|
||||
|
||||
/**
|
||||
* Build from the module's resolved environment. Auth is a service-account/API token
|
||||
* (MESH_GRAFANA_TOKEN, sent as Bearer) when present, else HTTP basic with the admin password the
|
||||
* module keeps as its own secret (MESH_GRAFANA_PASSWORD, user MESH_GRAFANA_USER, default admin).
|
||||
* Throws when neither is configured — the module then contributes nothing rather than failing.
|
||||
* Build from the module's resolved environment. Auth is HTTP basic with the vault's admin password,
|
||||
* which the runtime config carries as `password` (kept protected, so no setting replaces it) or
|
||||
* MESH_GRAFANA_PASSWORD (user MESH_GRAFANA_USER, default admin); else a service-account/API token from
|
||||
* the environment (MESH_GRAFANA_TOKEN, sent as Bearer). A `token` in the settings file is never used: a
|
||||
* credential is the vault's, never a setting (hq issue 342). Throws when neither is configured — the
|
||||
* module then contributes nothing rather than failing.
|
||||
*/
|
||||
static fromEnv(env: NodeJS.ProcessEnv = process.env): GrafanaClient {
|
||||
const cfg = meshConfig(env.MESH_GRAFANA_CONFIG_FILE);
|
||||
const url = cfg.url ?? env.MESH_GRAFANA_URL ?? `http://127.0.0.1:${env.GRAFANA_PORT ?? "3000"}`;
|
||||
const token = cfg.token ?? env.MESH_GRAFANA_TOKEN;
|
||||
if (token) return new GrafanaClient(url, `Bearer ${token}`);
|
||||
ignoreSecretSetting(cfg, "token");
|
||||
const password = cfg.password ?? env.MESH_GRAFANA_PASSWORD;
|
||||
if (password) {
|
||||
const user = cfg.user ?? env.MESH_GRAFANA_USER ?? "admin";
|
||||
return new GrafanaClient(url, `Basic ${Buffer.from(`${user}:${password}`).toString("base64")}`);
|
||||
}
|
||||
const token = env.MESH_GRAFANA_TOKEN;
|
||||
if (token) return new GrafanaClient(url, `Bearer ${token}`);
|
||||
throw new Error("no Grafana auth — set MESH_GRAFANA_TOKEN or MESH_GRAFANA_PASSWORD");
|
||||
}
|
||||
|
||||
|
||||
@@ -71,20 +71,36 @@
|
||||
"mode": "0644",
|
||||
"content": "GF_SERVER_ROOT_URL=https://${bound:route:name}\nGF_AUTH_GENERIC_OAUTH_ENABLED=true\nGF_AUTH_GENERIC_OAUTH_NAME=Keycloak\nGF_AUTH_GENERIC_OAUTH_CLIENT_ID=${bound:oidc-client:as}\nGF_AUTH_GENERIC_OAUTH_CLIENT_SECRET__FILE=/run/secrets/oidc-client\nGF_AUTH_GENERIC_OAUTH_SCOPES=openid email profile roles\nGF_AUTH_GENERIC_OAUTH_AUTH_URL=${bound:oidc-client:issuer}${bound:oidc-client:authorization-path}\nGF_AUTH_GENERIC_OAUTH_TOKEN_URL=${bound:oidc-client:issuer}${bound:oidc-client:token-path}\nGF_AUTH_GENERIC_OAUTH_API_URL=${bound:oidc-client:issuer}${bound:oidc-client:userinfo-path}\nGF_AUTH_GENERIC_OAUTH_ROLE_ATTRIBUTE_PATH=contains(roles[*], 'admin') && 'Admin' || contains(realm_access.roles[*], 'admin') && 'Admin' || 'Viewer'\nGF_AUTH_GENERIC_OAUTH_USE_PKCE=true\nGF_AUTH_GENERIC_OAUTH_ALLOW_SIGN_UP=true\nGF_AUTH_GENERIC_OAUTH_ALLOW_ASSIGN_GRAFANA_ADMIN=true\n"
|
||||
},
|
||||
{
|
||||
"id": "influxdb-datasources",
|
||||
"type": "directory",
|
||||
"path": "${dir:state}/datasources",
|
||||
"mode": "0755",
|
||||
"owner": "472:472"
|
||||
},
|
||||
{
|
||||
"id": "influxdb-credential",
|
||||
"type": "directory",
|
||||
"path": "${dir:state}/influxdb",
|
||||
"mode": "0700",
|
||||
"owner": "472:472"
|
||||
},
|
||||
{
|
||||
"id": "influxdb-secret",
|
||||
"type": "file",
|
||||
"path": "${dir:state}/influxdb-api.secret",
|
||||
"path": "${dir:influxdb-credential}/influxdb-api",
|
||||
"mode": "0400",
|
||||
"owner": "472:472",
|
||||
"content": "${secret:influxdb-api}"
|
||||
"content": "${secret:influxdb-api}",
|
||||
"only-when-bound": "influxdb-api"
|
||||
},
|
||||
{
|
||||
"id": "influxdb-datasource",
|
||||
"type": "file",
|
||||
"path": "${dir:state}/datasource-influxdb.yaml",
|
||||
"path": "${dir:influxdb-datasources}/mesh-influxdb.yaml",
|
||||
"mode": "0644",
|
||||
"content": "apiVersion: 1\n# Written by the mesh from grafana's influxdb-api binding; grafana reads it at start. Its own name and\n# uid, so a data source somebody made in the UI is never overwritten, and read-only in the UI because\n# the mesh resets it. The password is read from the file the mesh delivers, never written here.\ndatasources:\n - name: InfluxDB (mesh)\n uid: mesh-influxdb-api\n type: influxdb\n access: proxy\n url: ${bound:influxdb-api:scheme}://${bound:influxdb-api:at}:${bound:influxdb-api:port}\n user: ${bound:influxdb-api:as}\n isDefault: false\n editable: false\n jsonData:\n dbName: ${bound:influxdb-api:bucket}\n httpMode: POST\n secureJsonData:\n password: $__file{/run/secrets/influxdb-api}\n"
|
||||
"content": "apiVersion: 1\n# Written by the mesh from grafana's influxdb-api binding, only while it is bound (novox/hq ADR 0330):\n# without influxdb, grafana runs with no mesh data source. grafana reads it at start. Its own name and\n# uid, so a data source somebody made in the UI is never overwritten, and read-only in the UI because\n# the mesh resets it. The password is read from the file the mesh delivers, never written here.\ndatasources:\n - name: InfluxDB (mesh)\n uid: mesh-influxdb-api\n type: influxdb\n access: proxy\n url: ${bound:influxdb-api:scheme}://${bound:influxdb-api:at}:${bound:influxdb-api:port}\n user: ${bound:influxdb-api:as}\n isDefault: false\n editable: false\n jsonData:\n dbName: ${bound:influxdb-api:bucket}\n httpMode: POST\n secureJsonData:\n password: $__file{/run/secrets/influxdb/influxdb-api}\n",
|
||||
"only-when-bound": "influxdb-api"
|
||||
},
|
||||
{
|
||||
"id": "server",
|
||||
@@ -103,8 +119,8 @@
|
||||
"${dir:data}:/var/lib/grafana",
|
||||
"${dir:state}/admin.secret:/run/secrets/admin:ro",
|
||||
"${dir:state}/oidc-client.secret:/run/secrets/oidc-client:ro",
|
||||
"${dir:state}/influxdb-api.secret:/run/secrets/influxdb-api:ro",
|
||||
"${dir:state}/datasource-influxdb.yaml:/etc/grafana/provisioning/datasources/mesh-influxdb.yaml:ro"
|
||||
"${dir:influxdb-credential}:/run/secrets/influxdb:ro",
|
||||
"${dir:influxdb-datasources}:/etc/grafana/provisioning/datasources:ro"
|
||||
],
|
||||
"env": {
|
||||
"GF_SECURITY_ADMIN_PASSWORD__FILE": "/run/secrets/admin"
|
||||
@@ -125,13 +141,17 @@
|
||||
"path": "${dir:mesh-state}/config.json",
|
||||
"mode": "0600",
|
||||
"content": "{\n \"user\": \"admin\",\n \"password\": \"${secret:admin}\"\n}\n",
|
||||
"merge": "json"
|
||||
"merge": "json",
|
||||
"protected": ["password", "token"]
|
||||
}
|
||||
],
|
||||
"requires": [
|
||||
"route",
|
||||
"oidc-client",
|
||||
"influxdb-api"
|
||||
{
|
||||
"name": "influxdb-api",
|
||||
"optional": true
|
||||
}
|
||||
],
|
||||
"contributes": {
|
||||
"route": {
|
||||
|
||||
@@ -0,0 +1,39 @@
|
||||
// grafana runs without a data source, so its influxdb-api is optional (novox/hq ADR 0330, issue 166):
|
||||
// a mesh without influxdb can assign grafana, and influxdb is never installed for it. While absent, the
|
||||
// data source and its credential are not delivered, and grafana mounts their directories empty.
|
||||
|
||||
import { test } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { readFileSync } from "node:fs";
|
||||
|
||||
type Requirement = string | { name: string; optional?: boolean };
|
||||
type Resource = { id: string; type: string; path?: string; content?: string; volumes?: string[]; "only-when-bound"?: string };
|
||||
const manifest = JSON.parse(readFileSync(new URL("../module.json", import.meta.url), "utf8")) as {
|
||||
requires: Requirement[];
|
||||
resources: Resource[];
|
||||
};
|
||||
|
||||
const optional = (name: string) =>
|
||||
manifest.requires.some((r) => typeof r !== "string" && r.name === name && r.optional === true);
|
||||
|
||||
test("influxdb-api is optional, and route and oidc-client stay hard", () => {
|
||||
assert.ok(optional("influxdb-api"));
|
||||
assert.ok(manifest.requires.includes("route"));
|
||||
assert.ok(manifest.requires.includes("oidc-client"));
|
||||
});
|
||||
|
||||
test("every file naming influxdb-api is delivered only while it is bound", () => {
|
||||
for (const r of manifest.resources) {
|
||||
if (r.type === "file" && /\$\{(bound:influxdb-api:|secret:influxdb-api\})/.test(r.content ?? "")) {
|
||||
assert.equal(r["only-when-bound"], "influxdb-api", `${r.id} names influxdb-api and is delivered while it is absent`);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
test("the container mounts the data source's directories, never a file that may be absent", () => {
|
||||
// A bind mount of a missing file makes a directory in its place, which the file could then never be written over.
|
||||
const server = manifest.resources.find((r) => r.id === "server");
|
||||
for (const v of server?.volumes ?? []) {
|
||||
assert.ok(!/influxdb.*\.(yaml|secret)|influxdb-api:/.test(v.split(":")[0]), `${v} mounts an only-when-bound file`);
|
||||
}
|
||||
});
|
||||
@@ -0,0 +1,40 @@
|
||||
// A password, a token or an API key is the vault's, never a setting (novox/hq issue 342): grafana logs in
|
||||
// with the vault's admin password, a `token` in the settings file is ignored, and that is said once, by the
|
||||
// key alone. The runtime config keeps both keys protected, so the controller refuses a layer setting either.
|
||||
|
||||
import { test, mock } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { mkdtempSync, readFileSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
|
||||
import { GrafanaClient } from "../dist/client.js";
|
||||
|
||||
const SETTING = "one-a-setting-carried";
|
||||
const VAULT = "the-vault-made-this";
|
||||
|
||||
test("the vault's admin password is used, a token setting is ignored and said once, without its value", () => {
|
||||
const said: string[] = [];
|
||||
const err = mock.method(console, "error", (...args: unknown[]) => { said.push(args.join(" ")); });
|
||||
try {
|
||||
const dir = mkdtempSync(join(tmpdir(), "grafana-342-"));
|
||||
// What the runtime config holds once the vault filled it, with a token a layer carried beside it.
|
||||
writeFileSync(join(dir, "config.json"), JSON.stringify({ user: "admin", password: VAULT, token: SETTING }));
|
||||
const env = { MESH_GRAFANA_CONFIG_FILE: join(dir, "config.json") };
|
||||
const basic = "Basic " + Buffer.from(`admin:${VAULT}`).toString("base64");
|
||||
assert.equal((GrafanaClient.fromEnv(env) as any).authHeader, basic);
|
||||
assert.equal((GrafanaClient.fromEnv(env) as any).authHeader, basic);
|
||||
assert.deepEqual(said, ["the setting token is ignored: it comes from the vault"]);
|
||||
assert.ok(!said.join("\n").includes(SETTING), "the setting's value is never said");
|
||||
} finally {
|
||||
err.mock.restore();
|
||||
}
|
||||
});
|
||||
|
||||
test("the runtime config keeps password and token protected", () => {
|
||||
const manifest = JSON.parse(readFileSync(new URL("../module.json", import.meta.url), "utf8")) as {
|
||||
resources: { id: string; protected?: string[] }[];
|
||||
};
|
||||
const runtime = manifest.resources.find((r) => r.id === "runtime-config");
|
||||
assert.deepEqual(runtime?.protected, ["password", "token"]);
|
||||
});
|
||||
@@ -27,6 +27,17 @@ function meshConfig(file?: string): Record<string, string> {
|
||||
catch { return {}; }
|
||||
}
|
||||
|
||||
/**
|
||||
* A password, token or API key is the vault's, never a setting (novox/hq issue 342): a value for one of
|
||||
* them in the settings file is ignored, and that is said once — by its key, never with its value.
|
||||
*/
|
||||
const saidIgnored = new Set<string>();
|
||||
function ignoreSecretSetting(cfg: Record<string, unknown>, key: string): void {
|
||||
if (cfg[key] === undefined || saidIgnored.has(key)) return;
|
||||
saidIgnored.add(key);
|
||||
console.error(`the setting ${key} is ignored: it comes from the vault`);
|
||||
}
|
||||
|
||||
/** Read a secret the mesh mounted at a file path (an own-secret delivered by `secret accept`);
|
||||
* absent or unreadable yields undefined so callers fall back rather than crash. */
|
||||
function readSecret(file?: string): string | undefined {
|
||||
@@ -53,7 +64,8 @@ export class HomeAssistantClient {
|
||||
static fromEnv(env: NodeJS.ProcessEnv = process.env): HomeAssistantClient {
|
||||
const cfg = meshConfig(env.MESH_HOMEASSISTANT_CONFIG_FILE);
|
||||
const url = cfg.url ?? env.MESH_HOMEASSISTANT_URL ?? `http://127.0.0.1:${env.HOMEASSISTANT_PORT ?? "8123"}`;
|
||||
const token = cfg.token ?? readSecret(env.MESH_HOMEASSISTANT_TOKEN_FILE) ?? env.MESH_HOMEASSISTANT_TOKEN;
|
||||
ignoreSecretSetting(cfg, "token");
|
||||
const token = readSecret(env.MESH_HOMEASSISTANT_TOKEN_FILE) ?? env.MESH_HOMEASSISTANT_TOKEN;
|
||||
if (!token) throw new Error("no Home Assistant token — set MESH_HOMEASSISTANT_TOKEN");
|
||||
return new HomeAssistantClient(url, token);
|
||||
}
|
||||
|
||||
@@ -123,11 +123,23 @@
|
||||
}
|
||||
],
|
||||
"requires": [
|
||||
"lidarr-api",
|
||||
"mqtt-topic",
|
||||
"radarr-api",
|
||||
{
|
||||
"name": "lidarr-api",
|
||||
"optional": true
|
||||
},
|
||||
{
|
||||
"name": "mqtt-topic",
|
||||
"optional": true
|
||||
},
|
||||
{
|
||||
"name": "radarr-api",
|
||||
"optional": true
|
||||
},
|
||||
"route",
|
||||
"sonarr-api"
|
||||
{
|
||||
"name": "sonarr-api",
|
||||
"optional": true
|
||||
}
|
||||
],
|
||||
"contributes": {
|
||||
"mqtt-topic": {
|
||||
|
||||
@@ -7,7 +7,7 @@
|
||||
"scripts": {
|
||||
"build": "tsc client.ts index.ts tools/index.ts provisions/hass.ts provisions/probe.ts provisions/connections.ts provisions/mesh.ts provisions/index.ts --module NodeNext --moduleResolution NodeNext --target ES2022 --outDir dist",
|
||||
"typecheck": "tsc -p tsconfig.json",
|
||||
"test": "node --test --experimental-strip-types 'test/*.test.ts'"
|
||||
"test": "npm run build && node --test --experimental-strip-types 'test/*.test.ts'"
|
||||
},
|
||||
"dependencies": {
|
||||
"@novox/mesh-sdk": "^0.1.0"
|
||||
|
||||
@@ -0,0 +1,20 @@
|
||||
// Home Assistant runs without the Servarr apps and without a broker, so those requirements are optional
|
||||
// (novox/hq ADR 0330, issue 166): a mesh without lidarr or an MQTT broker can assign home-assistant, and
|
||||
// nothing is installed for it. Its provisions step already reads a binding that is not there as absent.
|
||||
|
||||
import { test } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { readFileSync } from "node:fs";
|
||||
|
||||
type Requirement = string | { name: string; optional?: boolean };
|
||||
const manifest = JSON.parse(readFileSync(new URL("../module.json", import.meta.url), "utf8")) as { requires: Requirement[] };
|
||||
|
||||
test("the Servarr APIs and mqtt-topic are optional, and route stays hard", () => {
|
||||
for (const name of ["sonarr-api", "radarr-api", "lidarr-api", "mqtt-topic"]) {
|
||||
assert.ok(
|
||||
manifest.requires.some((r) => typeof r !== "string" && r.name === name && r.optional === true),
|
||||
`${name} is not optional`,
|
||||
);
|
||||
}
|
||||
assert.ok(manifest.requires.includes("route"));
|
||||
});
|
||||
@@ -0,0 +1,38 @@
|
||||
// A password, a token or an API key is the vault's, never a setting (novox/hq issue 342): a value for it in
|
||||
// the settings file the mesh merges is ignored, the vault's is used, and that is said once, by the key alone.
|
||||
// Run against the compiled module, the way the runtime loads it.
|
||||
|
||||
import { test, mock } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { mkdtempSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
|
||||
import { HomeAssistantClient } from "../dist/client.js";
|
||||
|
||||
const SETTING = "one-a-setting-carried";
|
||||
const VAULT = "the-vault-made-this";
|
||||
|
||||
function delivered(settings: Record<string, string>): Record<string, string> {
|
||||
const dir = mkdtempSync(join(tmpdir(), "home-assistant-342-"));
|
||||
writeFileSync(join(dir, "config.json"), JSON.stringify(settings));
|
||||
writeFileSync(join(dir, "vault.secret"), VAULT + "\n");
|
||||
return { dir, config: join(dir, "config.json"), vault: join(dir, "vault.secret") };
|
||||
}
|
||||
|
||||
test("the vault's token is used, a setting of it is ignored and said once, without its value", () => {
|
||||
const said: string[] = [];
|
||||
const err = mock.method(console, "error", (...args: unknown[]) => { said.push(args.join(" ")); });
|
||||
try {
|
||||
const at = delivered({ token: SETTING });
|
||||
const env = { MESH_HOMEASSISTANT_CONFIG_FILE: at.config, MESH_HOMEASSISTANT_TOKEN_FILE: at.vault };
|
||||
const first = HomeAssistantClient.fromEnv(env);
|
||||
const second = HomeAssistantClient.fromEnv(env);
|
||||
assert.equal((first as any).token, VAULT);
|
||||
assert.equal((second as any).token, VAULT);
|
||||
assert.deepEqual(said, ["the setting token is ignored: it comes from the vault"]);
|
||||
assert.ok(!said.join("\n").includes(SETTING), "the setting's value is never said");
|
||||
} finally {
|
||||
err.mock.restore();
|
||||
}
|
||||
});
|
||||
@@ -42,6 +42,24 @@ function meshConfig(file?: string): Record<string, string> {
|
||||
catch { return {}; }
|
||||
}
|
||||
|
||||
/**
|
||||
* A password, token or API key is the vault's, never a setting (novox/hq issue 342): a value for one of
|
||||
* them in the settings file is ignored, and that is said once — by its key, never with its value.
|
||||
*/
|
||||
const saidIgnored = new Set<string>();
|
||||
function ignoreSecretSetting(cfg: Record<string, unknown>, key: string): void {
|
||||
if (cfg[key] === undefined || saidIgnored.has(key)) return;
|
||||
saidIgnored.add(key);
|
||||
console.error(`the setting ${key} is ignored: it comes from the vault`);
|
||||
}
|
||||
|
||||
/** A secret delivered as a file, trimmed; undefined when there is none. */
|
||||
function readSecret(path: string | undefined): string | undefined {
|
||||
if (!path) return undefined;
|
||||
try { return readFileSync(path, "utf8").trim() || undefined; }
|
||||
catch { return undefined; }
|
||||
}
|
||||
|
||||
export class IcecastClient {
|
||||
readonly baseUrl: string;
|
||||
private readonly authHeader?: string;
|
||||
@@ -57,7 +75,9 @@ export class IcecastClient {
|
||||
static fromEnv(env: NodeJS.ProcessEnv = process.env): IcecastClient {
|
||||
const cfg = meshConfig(env.MESH_ICECAST_CONFIG_FILE);
|
||||
const url = cfg.url ?? (env.MESH_ICECAST_URL ?? `http://127.0.0.1:${env.ICECAST_PORT ?? "8000"}`);
|
||||
return new IcecastClient(url, cfg.user ?? env.MESH_ICECAST_ADMIN_USER, cfg.password ?? env.MESH_ICECAST_ADMIN_PASSWORD);
|
||||
ignoreSecretSetting(cfg, "password");
|
||||
const password = readSecret(env.MESH_ICECAST_ADMIN_PASSWORD_FILE) ?? env.MESH_ICECAST_ADMIN_PASSWORD;
|
||||
return new IcecastClient(url, cfg.user ?? env.MESH_ICECAST_ADMIN_USER ?? "admin", password);
|
||||
}
|
||||
|
||||
async getStatus(): Promise<IcecastStatus> {
|
||||
|
||||
@@ -120,7 +120,8 @@
|
||||
],
|
||||
"env": {
|
||||
"MESH_ICECAST_URL": "http://127.0.0.1:${port:8000}",
|
||||
"MESH_ICECAST_CONFIG_FILE": "${dir:mesh-state}/config.json"
|
||||
"MESH_ICECAST_CONFIG_FILE": "${dir:mesh-state}/config.json",
|
||||
"MESH_ICECAST_ADMIN_PASSWORD_FILE": "${dir:state}/admin.secret"
|
||||
}
|
||||
}
|
||||
]
|
||||
|
||||
@@ -0,0 +1,39 @@
|
||||
// A password, a token or an API key is the vault's, never a setting (novox/hq issue 342): a value for it in
|
||||
// the settings file the mesh merges is ignored, the vault's is used, and that is said once, by the key alone.
|
||||
// Run against the compiled module, the way the runtime loads it.
|
||||
|
||||
import { test, mock } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { mkdtempSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
|
||||
import { IcecastClient } from "../dist/client.js";
|
||||
|
||||
const SETTING = "one-a-setting-carried";
|
||||
const VAULT = "the-vault-made-this";
|
||||
|
||||
function delivered(settings: Record<string, string>): Record<string, string> {
|
||||
const dir = mkdtempSync(join(tmpdir(), "icecast-342-"));
|
||||
writeFileSync(join(dir, "config.json"), JSON.stringify(settings));
|
||||
writeFileSync(join(dir, "vault.secret"), VAULT + "\n");
|
||||
return { dir, config: join(dir, "config.json"), vault: join(dir, "vault.secret") };
|
||||
}
|
||||
|
||||
test("the vault's password is used, a setting of it is ignored and said once, without its value", () => {
|
||||
const said: string[] = [];
|
||||
const err = mock.method(console, "error", (...args: unknown[]) => { said.push(args.join(" ")); });
|
||||
try {
|
||||
const at = delivered({ password: SETTING });
|
||||
const env = { MESH_ICECAST_CONFIG_FILE: at.config, MESH_ICECAST_ADMIN_PASSWORD_FILE: at.vault };
|
||||
const first = IcecastClient.fromEnv(env);
|
||||
const second = IcecastClient.fromEnv(env);
|
||||
const basic = "Basic " + Buffer.from(`admin:${VAULT}`).toString("base64");
|
||||
assert.equal((first as any).authHeader, basic);
|
||||
assert.equal((second as any).authHeader, basic);
|
||||
assert.deepEqual(said, ["the setting password is ignored: it comes from the vault"]);
|
||||
assert.ok(!said.join("\n").includes(SETTING), "the setting's value is never said");
|
||||
} finally {
|
||||
err.mock.restore();
|
||||
}
|
||||
});
|
||||
@@ -43,6 +43,17 @@ function meshConfig(file?: string): Record<string, string> {
|
||||
catch { return {}; }
|
||||
}
|
||||
|
||||
/**
|
||||
* A password, token or API key is the vault's, never a setting (novox/hq issue 342): a value for one of
|
||||
* them in the settings file is ignored, and that is said once — by its key, never with its value.
|
||||
*/
|
||||
const saidIgnored = new Set<string>();
|
||||
function ignoreSecretSetting(cfg: Record<string, unknown>, key: string): void {
|
||||
if (cfg[key] === undefined || saidIgnored.has(key)) return;
|
||||
saidIgnored.add(key);
|
||||
console.error(`the setting ${key} is ignored: it comes from the vault`);
|
||||
}
|
||||
|
||||
/** A secret delivered as a file, trimmed; undefined when there is none, so the caller can fall back. */
|
||||
function tokenFromFile(file?: string): string | undefined {
|
||||
if (!file) return undefined;
|
||||
@@ -71,7 +82,8 @@ export class InfluxDBClient {
|
||||
const url = cfg.url ?? env.MESH_INFLUXDB_URL ?? `http://127.0.0.1:${env.INFLUXDB_PORT ?? "8086"}`;
|
||||
// The token reaches the process as a file (novox/hq ADR 0086); the environment variable stays
|
||||
// only for a workstation running the tools by hand.
|
||||
const token = cfg.token ?? tokenFromFile(env.MESH_INFLUXDB_TOKEN_FILE) ?? env.MESH_INFLUXDB_TOKEN;
|
||||
ignoreSecretSetting(cfg, "token");
|
||||
const token = tokenFromFile(env.MESH_INFLUXDB_TOKEN_FILE) ?? env.MESH_INFLUXDB_TOKEN;
|
||||
if (!token) throw new Error("no InfluxDB token — set MESH_INFLUXDB_TOKEN_FILE");
|
||||
const org = cfg.org ?? env.MESH_INFLUXDB_ORG ?? "mesh";
|
||||
return new InfluxDBClient(url, token, org);
|
||||
|
||||
@@ -0,0 +1,38 @@
|
||||
// A password, a token or an API key is the vault's, never a setting (novox/hq issue 342): a value for it in
|
||||
// the settings file the mesh merges is ignored, the vault's is used, and that is said once, by the key alone.
|
||||
// Run against the compiled module, the way the runtime loads it.
|
||||
|
||||
import { test, mock } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { mkdtempSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
|
||||
import { InfluxDBClient } from "../dist/client.js";
|
||||
|
||||
const SETTING = "one-a-setting-carried";
|
||||
const VAULT = "the-vault-made-this";
|
||||
|
||||
function delivered(settings: Record<string, string>): Record<string, string> {
|
||||
const dir = mkdtempSync(join(tmpdir(), "influxdb-342-"));
|
||||
writeFileSync(join(dir, "config.json"), JSON.stringify(settings));
|
||||
writeFileSync(join(dir, "vault.secret"), VAULT + "\n");
|
||||
return { dir, config: join(dir, "config.json"), vault: join(dir, "vault.secret") };
|
||||
}
|
||||
|
||||
test("the vault's token is used, a setting of it is ignored and said once, without its value", () => {
|
||||
const said: string[] = [];
|
||||
const err = mock.method(console, "error", (...args: unknown[]) => { said.push(args.join(" ")); });
|
||||
try {
|
||||
const at = delivered({ token: SETTING });
|
||||
const env = { MESH_INFLUXDB_CONFIG_FILE: at.config, MESH_INFLUXDB_TOKEN_FILE: at.vault };
|
||||
const first = InfluxDBClient.fromEnv(env);
|
||||
const second = InfluxDBClient.fromEnv(env);
|
||||
assert.equal((first as any).token, VAULT);
|
||||
assert.equal((second as any).token, VAULT);
|
||||
assert.deepEqual(said, ["the setting token is ignored: it comes from the vault"]);
|
||||
assert.ok(!said.join("\n").includes(SETTING), "the setting's value is never said");
|
||||
} finally {
|
||||
err.mock.restore();
|
||||
}
|
||||
});
|
||||
@@ -162,7 +162,13 @@
|
||||
"path": "${dir:mesh-state}/config.json",
|
||||
"mode": "0600",
|
||||
"content": "{}\n",
|
||||
"merge": "json"
|
||||
"merge": "json",
|
||||
"takes-settings": [
|
||||
"url",
|
||||
"user",
|
||||
"realm",
|
||||
"issuer"
|
||||
]
|
||||
}
|
||||
],
|
||||
"build": {
|
||||
|
||||
@@ -112,7 +112,7 @@ for p in postgres-provisioner objectstore-provisioner route-proxy; do
|
||||
(cd ${shellQuote(dir)}/mesh-controller && CGO_ENABLED=0 go build -o ${shellQuote(bin)}/$p ./examples/$p)
|
||||
done
|
||||
export MESH_LAB_HOST_BINARY=${shellQuote(join(dir, "mesh-host", "mesh-host"))}
|
||||
export MESH_LAB_BUNDLE=${shellQuote(join(dir, "mesh-host", "examples", "foundation-first-node-nats.lock"))}
|
||||
export MESH_LAB_BUNDLE=${shellQuote(join(dir, "mesh-host", "examples", "foundation-first-node.lock"))}
|
||||
export MESH_LAB_MODULES=${shellQuote(join(dir, "mesh-controller", "examples", "modules"))}
|
||||
export MESH_LAB_BUILDER=${shellQuote(join(dir, "mesh-controller", "build", "mesh-builder"))}
|
||||
export MESH_LAB_BOOTSTRAP_BINARY=${shellQuote(join(dir, "mesh-host", "mesh-bootstrap"))}
|
||||
|
||||
@@ -114,7 +114,8 @@
|
||||
"path": "${dir:mesh-state}/config.json",
|
||||
"mode": "0600",
|
||||
"content": "{\n \"password\": \"${secret:server-password}\"\n}\n",
|
||||
"merge": "json"
|
||||
"merge": "json",
|
||||
"protected": ["password"]
|
||||
}
|
||||
],
|
||||
"build": {
|
||||
|
||||
+13
-1
@@ -52,6 +52,17 @@ function meshConfig(file?: string): Record<string, string> {
|
||||
catch { return {}; }
|
||||
}
|
||||
|
||||
/**
|
||||
* A password, token or API key is the vault's, never a setting (novox/hq issue 342): a value for one of
|
||||
* them in the settings file is ignored, and that is said once — by its key, never with its value.
|
||||
*/
|
||||
const saidIgnored = new Set<string>();
|
||||
function ignoreSecretSetting(cfg: Record<string, unknown>, key: string): void {
|
||||
if (cfg[key] === undefined || saidIgnored.has(key)) return;
|
||||
saidIgnored.add(key);
|
||||
console.error(`the setting ${key} is ignored: it comes from the vault`);
|
||||
}
|
||||
|
||||
/** Read a secret from the file the mesh mounted it at (an own-secret), if the pointing env is set. */
|
||||
function readSecret(path: string | undefined): string | undefined {
|
||||
if (!path) return undefined;
|
||||
@@ -81,7 +92,8 @@ export class MailuClient {
|
||||
static fromEnv(env: NodeJS.ProcessEnv = process.env): MailuClient {
|
||||
const cfg = meshConfig(env.MESH_MAILU_CONFIG_FILE);
|
||||
const url = cfg.url ?? env.MESH_MAILU_URL;
|
||||
const apiKey = cfg.apiKey ?? readSecret(env.MESH_MAILU_API_KEY_FILE) ?? env.MESH_MAILU_API_KEY;
|
||||
ignoreSecretSetting(cfg, "apiKey");
|
||||
const apiKey = readSecret(env.MESH_MAILU_API_KEY_FILE) ?? env.MESH_MAILU_API_KEY;
|
||||
if (!url || !apiKey) {
|
||||
throw new Error("Mailu is not configured — set MESH_MAILU_URL and the API token own-secret");
|
||||
}
|
||||
|
||||
@@ -0,0 +1,38 @@
|
||||
// A password, a token or an API key is the vault's, never a setting (novox/hq issue 342): a value for it in
|
||||
// the settings file the mesh merges is ignored, the vault's is used, and that is said once, by the key alone.
|
||||
// Run against the compiled module, the way the runtime loads it.
|
||||
|
||||
import { test, mock } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { mkdtempSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
|
||||
import { MailuClient } from "../dist/client.js";
|
||||
|
||||
const SETTING = "one-a-setting-carried";
|
||||
const VAULT = "the-vault-made-this";
|
||||
|
||||
function delivered(settings: Record<string, string>): Record<string, string> {
|
||||
const dir = mkdtempSync(join(tmpdir(), "mailu-342-"));
|
||||
writeFileSync(join(dir, "config.json"), JSON.stringify(settings));
|
||||
writeFileSync(join(dir, "vault.secret"), VAULT + "\n");
|
||||
return { dir, config: join(dir, "config.json"), vault: join(dir, "vault.secret") };
|
||||
}
|
||||
|
||||
test("the vault's apiKey is used, a setting of it is ignored and said once, without its value", () => {
|
||||
const said: string[] = [];
|
||||
const err = mock.method(console, "error", (...args: unknown[]) => { said.push(args.join(" ")); });
|
||||
try {
|
||||
const at = delivered({ apiKey: SETTING });
|
||||
const env = { MESH_MAILU_CONFIG_FILE: at.config, MESH_MAILU_URL: "http://127.0.0.1:1/api/v1", MESH_MAILU_API_KEY_FILE: at.vault };
|
||||
const first = MailuClient.fromEnv(env);
|
||||
const second = MailuClient.fromEnv(env);
|
||||
assert.equal((first as any).apiKey, VAULT);
|
||||
assert.equal((second as any).apiKey, VAULT);
|
||||
assert.deepEqual(said, ["the setting apiKey is ignored: it comes from the vault"]);
|
||||
assert.ok(!said.join("\n").includes(SETTING), "the setting's value is never said");
|
||||
} finally {
|
||||
err.mock.restore();
|
||||
}
|
||||
});
|
||||
@@ -51,6 +51,10 @@ type ChecksAnswer struct {
|
||||
Statuses []CheckStatus `json:"statuses"`
|
||||
// Verdict is the merge check's verdict whole, as this owner heard it for this commit.
|
||||
Verdict *Verdict `json:"verdict,omitempty"`
|
||||
// Judged is what the verdict judged: the head merged into the trunk commit (novox/hq ADR 0323 §3); OutOfDate
|
||||
// why it no longer counts, when its trunk has moved since.
|
||||
Judged string `json:"judged,omitempty"`
|
||||
OutOfDate string `json:"out_of_date,omitempty"`
|
||||
// Merge is whether the branch's protection lets the commit merge; Mergeable nil when it is not known.
|
||||
Merge ForgeMerge `json:"merge"`
|
||||
// Forge is why the forge could not be read, when it could not.
|
||||
@@ -65,6 +69,11 @@ type ChecksOf struct {
|
||||
Since string `json:"since"`
|
||||
Waits string `json:"waits,omitempty"`
|
||||
Group string `json:"group,omitempty"`
|
||||
// Class, Opened and MergedAt are what the issue tracker measures a fix's time by (novox/hq ADR 0313): the
|
||||
// class of what it moves, when its pull request was first heard, and when it merged.
|
||||
Class string `json:"class,omitempty"`
|
||||
Opened string `json:"opened,omitempty"`
|
||||
MergedAt string `json:"merged_at,omitempty"`
|
||||
}
|
||||
|
||||
// Checks is the `checks` verb: a repository and a pull request's number, or a repository and a commit.
|
||||
@@ -104,9 +113,16 @@ func (h *Holder) Checks(repository string, number int, commit string) (*ChecksAn
|
||||
}
|
||||
if d != nil {
|
||||
a.Delivery = &ChecksOf{ID: d.ID, State: d.State, Since: d.Since.UTC().Format(time.RFC3339), Waits: d.waits,
|
||||
Group: d.Group}
|
||||
Group: d.Group, Class: fixClass(&d.Delivery)}
|
||||
if !d.opened.IsZero() {
|
||||
a.Delivery.Opened = d.opened.UTC().Format(time.RFC3339)
|
||||
}
|
||||
if !d.MergedAt.IsZero() {
|
||||
a.Delivery.MergedAt = d.MergedAt.UTC().Format(time.RFC3339)
|
||||
}
|
||||
if d.Check != nil && sameCommit(d.Commit, a.Commit) {
|
||||
a.Verdict = d.Check
|
||||
a.Judged, a.OutOfDate = d.Check.Judged(), d.outOfDate()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -139,6 +155,7 @@ type checksDelivery struct {
|
||||
Delivery
|
||||
waits string
|
||||
groupSaid string
|
||||
opened time.Time
|
||||
}
|
||||
|
||||
// deliveryOf is the delivery of a commit — its head, or the commit it merged as — else, for a pull request,
|
||||
@@ -167,7 +184,7 @@ func (h *Holder) deliveryOf(repository string, number int, commit string) *check
|
||||
if found == nil {
|
||||
return nil
|
||||
}
|
||||
c := &checksDelivery{Delivery: *found, waits: h.waitsFor(found)}
|
||||
c := &checksDelivery{Delivery: *found, waits: h.waitsFor(found), opened: h.openedOf(found)}
|
||||
if found.Check != nil {
|
||||
v := *found.Check
|
||||
c.Check = &v
|
||||
@@ -261,6 +278,12 @@ func (a *ChecksAnswer) says() string {
|
||||
default:
|
||||
merge = "NOT mergeable — " + merge
|
||||
}
|
||||
if a.Judged != "" {
|
||||
what += " (" + a.Judged + ")"
|
||||
}
|
||||
if a.OutOfDate != "" {
|
||||
merge += "; its verdict is " + a.OutOfDate
|
||||
}
|
||||
return what + ": " + statuses + "; " + merge
|
||||
}
|
||||
|
||||
|
||||
@@ -451,7 +451,7 @@ func TestAMemberHeldForItsComposedCheckIsLiftedWhenItPasses(t *testing.T) {
|
||||
State: Ready, Since: t0, Created: t0.Add(-time.Hour), Plan: planMoving(b, "b"), Check: &Verdict{Gate: "pass"}}
|
||||
g := &Group{ID: "feat/k", Members: []string{aID, bID}, Order: []string{aID, bID}, Closed: true,
|
||||
OrderOf: map[string]string{aID: aHead, bID: bHead},
|
||||
Check: &GroupCheck{Asked: "check-1", At: t0, Commits: c.commits}}
|
||||
Check: &GroupCheck{Asked: "check-old", At: t0, Commits: c.commits}}
|
||||
for _, d := range []*Delivery{da, db} {
|
||||
_ = store.PutDelivery(d)
|
||||
}
|
||||
@@ -463,7 +463,7 @@ func TestAMemberHeldForItsComposedCheckIsLiftedWhenItPasses(t *testing.T) {
|
||||
if s := w.state(aID); s != Held {
|
||||
t.Fatalf("before its composed verdict it is %s", s)
|
||||
}
|
||||
w.h.Checked(CheckedEvent{Group: "feat/k", ID: "check-1", Verdict: c.verdict, Summary: "composed: " + c.verdict})
|
||||
w.h.Checked(CheckedEvent{Group: "feat/k", ID: "check-old", Verdict: c.verdict, Summary: "composed: " + c.verdict})
|
||||
w.settleAll()
|
||||
if s := w.state(aID); s != c.want {
|
||||
t.Fatalf("its composed check %s, and it is %s (%s), want %s", c.name, s, w.delivery(aID).HeldWhy, c.want)
|
||||
@@ -471,6 +471,10 @@ func TestAMemberHeldForItsComposedCheckIsLiftedWhenItPasses(t *testing.T) {
|
||||
if c.want == Delivering && !containsString(ctl.delivers, "plan-a") {
|
||||
t.Fatalf("lifted, its walk was not let go: %v", ctl.delivers)
|
||||
}
|
||||
// A composed check of other heads is asked again for these heads (novox/hq ADR 0323 §6).
|
||||
if c.name == "passed for other heads" && !containsString(ctl.checks, "feat/k:"+aID+","+bID) {
|
||||
t.Fatalf("the composed check was not asked again for the heads that merged: %v", ctl.checks)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -14,8 +14,10 @@ import (
|
||||
// the forge's holder, asked through its tools). Owed, never lost: kept with the delivery before anything is
|
||||
// tried, and tried again until done.
|
||||
|
||||
// transitionEvent is what `mesh-delivery.transition` says: no secret, no address.
|
||||
func transitionEvent(d *Delivery, t Transition) map[string]any {
|
||||
// transitionEvent is what `mesh-delivery.transition` says: no secret, no address. opened is when its pull
|
||||
// request was first heard (zero when not known); with its class and its merge's moment it is what the issue
|
||||
// tracker measures a fix's time by (novox/hq ADR 0313 §2, §4).
|
||||
func transitionEvent(d *Delivery, t Transition, opened time.Time) map[string]any {
|
||||
e := map[string]any{"id": d.ID, "repository": d.Repository, "commit": d.Commit, "from": t.From, "to": t.To,
|
||||
"event": t.Event, "why": t.Why, "at": t.At}
|
||||
if t.By != "" {
|
||||
@@ -36,9 +38,57 @@ func transitionEvent(d *Delivery, t Transition) map[string]any {
|
||||
if d.Plan != nil {
|
||||
e["plan"] = d.Plan.Summary
|
||||
}
|
||||
if c := fixClass(d); c != "" {
|
||||
e["class"] = c
|
||||
}
|
||||
if !opened.IsZero() {
|
||||
e["opened"] = opened.UTC()
|
||||
}
|
||||
if !d.MergedAt.IsZero() {
|
||||
e["merged_at"] = d.MergedAt.UTC()
|
||||
}
|
||||
return e
|
||||
}
|
||||
|
||||
// fixClass is the class of what a delivery moves (novox/hq ADR 0313 §2): its plan's, as the planner said it;
|
||||
// none for a plan that moves no module, from a planner that did not say a class; else its walk's class once its
|
||||
// delivery time is known; empty when nothing says it.
|
||||
func fixClass(d *Delivery) string {
|
||||
switch {
|
||||
case d.Plan != nil && d.Plan.Class != "":
|
||||
return d.Plan.Class
|
||||
case d.Plan != nil && len(d.Plan.Moved) == 0 && len(d.Plan.Dependents) == 0 && len(d.Plan.New) == 0:
|
||||
return classNone
|
||||
case d.Time != nil:
|
||||
return d.Time.Class
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// classNone is the class of a delivery that moves no module: nothing for a walk to move, and no fix budget.
|
||||
const classNone = "none"
|
||||
|
||||
// openedOf is when a delivery's pull request was first heard: the earliest head of it this owner holds, never
|
||||
// one made from the forge's word at its merge; zero when none is held or it has no pull request. Under the lock.
|
||||
func (h *Holder) openedOf(d *Delivery) time.Time {
|
||||
if d.Number <= 0 {
|
||||
return time.Time{}
|
||||
}
|
||||
var first time.Time
|
||||
for _, c := range h.deliveries {
|
||||
if c.FromForge || c.Number != d.Number || !strings.EqualFold(c.Repository, d.Repository) || c.Created.IsZero() {
|
||||
continue
|
||||
}
|
||||
if first.IsZero() || c.Created.Before(first) {
|
||||
first = c.Created
|
||||
}
|
||||
}
|
||||
if first.IsZero() && !d.FromForge {
|
||||
first = d.Created
|
||||
}
|
||||
return first
|
||||
}
|
||||
|
||||
// noteLine is one transition as the commit's note keeps it.
|
||||
func noteLine(d *Delivery, t Transition) string {
|
||||
by := ""
|
||||
@@ -104,6 +154,9 @@ func statusOf(d *Delivery) (string, string) {
|
||||
case Superseded:
|
||||
return "warning", clip("superseded: "+why, 140)
|
||||
case Stopped:
|
||||
if d.coveredStop() {
|
||||
return "success", clip(d.coveredStopSaid(), 140) // its change reached the machines (novox/hq issue 480)
|
||||
}
|
||||
return "error", clip("stopped: "+why, 140)
|
||||
case Retired:
|
||||
return "success", "history: merged before this owner delivered anything, never walked"
|
||||
@@ -124,6 +177,9 @@ const viewMarker = "<!-- mesh-delivery:view -->"
|
||||
func ViewBody(d *Delivery, g *Group) string {
|
||||
var b strings.Builder
|
||||
fmt.Fprintf(&b, "%s\n**Delivery** `%s` — **%s** since %s\n\n", viewMarker, d.ID, d.State, d.Since.UTC().Format(time.RFC3339))
|
||||
if d.coveredStop() {
|
||||
fmt.Fprintf(&b, "**Covered** after its stop: a later merge that contains it was delivered: %s\n\n", d.CoveredBy)
|
||||
}
|
||||
if d.HeldWhy != "" && d.State == Held {
|
||||
fmt.Fprintf(&b, "Held for a person: %s — `mesh-delivery.release` with why lets it go on.\n\n", d.HeldWhy)
|
||||
}
|
||||
@@ -278,6 +334,16 @@ func (h *Holder) Flush() {
|
||||
d.Owed = owed(d.Owed)
|
||||
h.keep(d)
|
||||
}
|
||||
// Counted for delivery_status, so a window's writes can be judged against its tries (novox/hq issue 407).
|
||||
h.tried.last = h.Now()
|
||||
switch {
|
||||
case err == nil:
|
||||
h.tried.done++
|
||||
case dropped != "":
|
||||
h.tried.givenUp++
|
||||
default:
|
||||
h.tried.failed++
|
||||
}
|
||||
if dropped != "" {
|
||||
h.refuse(fmt.Errorf("%s: its owed %s given up — %s: %v", j.id, j.e.Kind, dropped, err))
|
||||
}
|
||||
|
||||
@@ -67,6 +67,16 @@ type Holder struct {
|
||||
// leaveRefused is, by walk, what the controller answered when it refused both to leave a stopped merge out and
|
||||
// to stop the walk whole.
|
||||
leaveRefused map[string]string
|
||||
// tried counts the owed effects Flush tried since this owner started, by how each ended (novox/hq issue 407).
|
||||
tried owedTries
|
||||
}
|
||||
|
||||
// owedTries is how many owed effects were tried since this owner started, and when the last was: three numbers
|
||||
// and a time, however many were tried. Not kept: a restarted owner counts from its start, and says since when, so a
|
||||
// window's count is the difference of two readings in the same run (novox/hq issue 407).
|
||||
type owedTries struct {
|
||||
done, failed, givenUp int
|
||||
last time.Time
|
||||
}
|
||||
|
||||
// unmatched is a walk no delivery is known for yet: its merge not heard, or merged before this owner held.
|
||||
@@ -162,9 +172,10 @@ func (h *Holder) refuse(err error) {
|
||||
// moved records what a delivery's transitions owe outside, and keeps it.
|
||||
func (h *Holder) moved(d *Delivery, ts []Transition) {
|
||||
now := h.Now()
|
||||
opened := h.openedOf(d)
|
||||
for _, t := range ts {
|
||||
h.Logf("[mesh-delivery] %s: %s → %s (%s): %s", d.ID, orNothing(t.From), t.To, t.Event, t.Why)
|
||||
d.Owed = append(d.Owed, Effect{Kind: EffectEmit, Line: string(mustJSON(transitionEvent(d, t))), Since: now})
|
||||
d.Owed = append(d.Owed, Effect{Kind: EffectEmit, Line: string(mustJSON(transitionEvent(d, t, opened))), Since: now})
|
||||
line := noteLine(d, t)
|
||||
d.Owed = append(d.Owed, Effect{Kind: EffectNote, Commit: d.Commit, Line: line, Since: now})
|
||||
if d.MergedAs != "" && !sameCommit(d.MergedAs, d.Commit) && afterTheTrunk(t.To) {
|
||||
@@ -269,6 +280,8 @@ type PullEvent struct {
|
||||
ModuleDirs []string `json:"module_dirs,omitempty"`
|
||||
ModuleDirsSaid bool `json:"module_dirs_said,omitempty"`
|
||||
HeadChecks map[string]string `json:"head_checks,omitempty"`
|
||||
// BaseSHA is the base branch's head when the head was announced (novox/hq ADR 0323).
|
||||
BaseSHA string `json:"base_sha,omitempty"`
|
||||
}
|
||||
|
||||
func (p PullEvent) repository() string { return strings.ToLower(p.Owner + "/" + p.Repo) }
|
||||
@@ -286,10 +299,22 @@ func (h *Holder) PullUpdated(p PullEvent) {
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
id := IDOf(p.repository(), p.HeadSHA)
|
||||
if d := h.deliveries[id]; d != nil {
|
||||
return // announced again: nothing new
|
||||
}
|
||||
now := h.Now()
|
||||
if d := h.deliveries[id]; d != nil {
|
||||
// Announced again: nothing new — but for a delivery whose verdict is out of date, the announcement is its
|
||||
// check asked again against the trunk's head (the merge tool's ask, novox/hq ADR 0323 §5), and its bound
|
||||
// runs from now (§7). The base head it was announced with is kept, for a check this owner asks after it.
|
||||
if p.BaseSHA != "" && d.BaseSHA != p.BaseSHA {
|
||||
d.BaseSHA = p.BaseSHA
|
||||
h.keep(d)
|
||||
}
|
||||
if d.State == Proposed && d.outOfDate() != "" && d.AskedAgain == nil {
|
||||
d.AskedAgain = &now
|
||||
h.keep(d)
|
||||
}
|
||||
return
|
||||
}
|
||||
replaced := ""
|
||||
for _, old := range h.deliveries {
|
||||
if old.Repository == p.repository() && old.Number == p.Number && !sameCommit(old.Commit, p.HeadSHA) &&
|
||||
!old.State.Final() && old.MergedAs == "" {
|
||||
@@ -297,13 +322,14 @@ func (h *Holder) PullUpdated(p PullEvent) {
|
||||
h.settleAndKeep(old)
|
||||
if old.Group != "" {
|
||||
h.leaveGroup(old)
|
||||
replaced = old.ID
|
||||
}
|
||||
}
|
||||
}
|
||||
d := &Delivery{ID: id, Repository: p.repository(), Commit: p.HeadSHA, Number: p.Number, Title: p.Title,
|
||||
Base: p.Base, Branch: p.Head, HTMLURL: p.HTMLURL, CloneURL: p.CloneURL, Paths: p.Paths,
|
||||
PathsTruncated: p.PathsTruncated, Removed: p.Removed, ModuleDirs: p.ModuleDirs, ModuleDirsSaid: p.ModuleDirsSaid,
|
||||
After: AfterIn(p.Body), Created: now}
|
||||
After: AfterIn(p.Body), Created: now, BaseSHA: p.BaseSHA}
|
||||
t, err := Apply(d, EvAnnounced, false, Facts{Now: now, Why: fmt.Sprintf("%s#%d's head announced", d.Repository, d.Number)})
|
||||
if err != nil {
|
||||
h.refuse(err)
|
||||
@@ -317,7 +343,45 @@ func (h *Holder) PullUpdated(p PullEvent) {
|
||||
ts = append(ts, settle(d, Facts{Now: now})...)
|
||||
}
|
||||
h.moved(d, ts)
|
||||
h.joinGroup(d)
|
||||
if !h.takePlaceOf(replaced, d) {
|
||||
h.joinGroup(d)
|
||||
}
|
||||
}
|
||||
|
||||
// takePlaceOf puts a member's new head — a rebase, a push — in its group in the place of the head it superseded,
|
||||
// when the group is closed: a member of it already reached the trunk, and a closed group takes no new member. The
|
||||
// new head has its own check, and the group's composed check runs again for the new heads rather than holding the
|
||||
// group for a person (novox/hq ADR 0323 §6, issue 406). An open group takes it as joinGroup does.
|
||||
func (h *Holder) takePlaceOf(old string, d *Delivery) bool {
|
||||
if old == "" {
|
||||
return false
|
||||
}
|
||||
for _, g := range h.groups {
|
||||
_, ordered := g.OrderOf[old]
|
||||
if !g.Closed || !ordered && !containsString(g.Order, old) {
|
||||
continue
|
||||
}
|
||||
if ordered {
|
||||
delete(g.OrderOf, old)
|
||||
g.OrderOf[d.ID] = d.Commit
|
||||
}
|
||||
if !containsString(g.Members, d.ID) {
|
||||
g.Members = append(g.Members, d.ID)
|
||||
sort.Strings(g.Members)
|
||||
}
|
||||
for i, id := range g.Order {
|
||||
if id == old {
|
||||
g.Order[i] = d.ID
|
||||
}
|
||||
}
|
||||
d.Group = g.ID
|
||||
g.Updated = h.Now()
|
||||
h.keep(d)
|
||||
h.keepGroup(g)
|
||||
h.Logf("[mesh-delivery] group %s: %s takes the place of %s, which it superseded", g.ID, d.ID, old)
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// decidedOnHead is the verdict the forge's statuses on a head hold, when the gate decided one: passed, warned
|
||||
@@ -458,6 +522,47 @@ func (h *Holder) PullMerged(p PullEvent) {
|
||||
}
|
||||
h.moved(d, ts)
|
||||
h.closeGroupOf(d)
|
||||
h.trunkMoved(d, p.Base, p.MergeCommit, now)
|
||||
}
|
||||
|
||||
// trunkMoved takes a merge into a trunk as that trunk's move (novox/hq ADR 0323 §7): every other open delivery into
|
||||
// it keeps the trunk's new head, and a ready one whose verdict names the commit before goes back to proposed. A
|
||||
// group's composed check goes out of date when the merge is not one of its members and a member still open merges
|
||||
// into that trunk (§6); the group then reads checking. Nothing is asked: a recheck or the merge tool judges again.
|
||||
func (h *Holder) trunkMoved(merged *Delivery, base, commit string, now time.Time) {
|
||||
if base == "" || commit == "" {
|
||||
return
|
||||
}
|
||||
for _, o := range h.deliveries {
|
||||
if o == merged || o.Repository != merged.Repository || o.Base != base || o.MergedAs != "" || o.State.Final() ||
|
||||
sameCommit(o.TrunkNow, commit) {
|
||||
continue
|
||||
}
|
||||
// A check asked again against the trunk before this move judged that trunk: it is out of date too.
|
||||
o.TrunkNow, o.AskedAgain = commit, nil
|
||||
h.settleAndKeep(o)
|
||||
}
|
||||
for _, g := range h.groups {
|
||||
if g.ID == merged.Group || g.Check == nil || g.Check.Verdict == "" || g.Check.OutOfDate != "" {
|
||||
continue
|
||||
}
|
||||
for _, m := range h.membersOf(g) {
|
||||
if m.Repository != merged.Repository || m.Base != base || m.MergedAs != "" || m.State.Final() {
|
||||
continue
|
||||
}
|
||||
judged := g.Check.Trunks[m.ID]
|
||||
if judged == "" {
|
||||
judged = "?"
|
||||
}
|
||||
g.Check.OutOfDate = fmt.Sprintf("composed check out of date: %s was judged against trunk %s, the trunk is now %s "+
|
||||
"(%s merged)", m.ID, shortOf(judged, 12), shortOf(commit, 12), merged.ID)
|
||||
g.Check.OutOfDateAt = now
|
||||
g.Updated = now
|
||||
h.keepGroup(g)
|
||||
h.Logf("[mesh-delivery] group %s: %s", g.ID, g.Check.OutOfDate)
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// closeGroupOf closes a delivery's group once it reached the trunk: a delivering group takes no new member.
|
||||
@@ -529,7 +634,13 @@ type CheckedEvent struct {
|
||||
Owner string `json:"owner"`
|
||||
Repo string `json:"repo"`
|
||||
Commit string `json:"commit"`
|
||||
Trunk string `json:"trunk,omitempty"`
|
||||
} `json:"members,omitempty"`
|
||||
// Trunk is the trunk commit the head was merged into and judged as, Beside the other repositories' commits the
|
||||
// check read, and Took how long it ran (novox/hq ADR 0323 §3).
|
||||
Trunk string `json:"trunk,omitempty"`
|
||||
Beside map[string]string `json:"beside,omitempty"`
|
||||
Took string `json:"took,omitempty"`
|
||||
}
|
||||
|
||||
// Checked takes a verdict: a head's own, or a group's composed check.
|
||||
@@ -544,6 +655,18 @@ func (h *Holder) Checked(c CheckedEvent) {
|
||||
return
|
||||
}
|
||||
g.Check.Verdict, g.Check.Summary = c.Verdict, c.Summary
|
||||
// Each member's head and the trunk commit it was merged into (novox/hq ADR 0323 §6).
|
||||
for _, m := range c.Members {
|
||||
for id := range g.Check.Commits {
|
||||
if d := h.deliveries[id]; d != nil && strings.EqualFold(d.Repository, m.Owner+"/"+m.Repo) &&
|
||||
sameCommit(d.Commit, m.Commit) && m.Trunk != "" {
|
||||
if g.Check.Trunks == nil {
|
||||
g.Check.Trunks = map[string]string{}
|
||||
}
|
||||
g.Check.Trunks[id] = m.Trunk
|
||||
}
|
||||
}
|
||||
}
|
||||
g.Updated = now
|
||||
h.keepGroup(g)
|
||||
h.Logf("[mesh-delivery] group %s composed: %s — %s", g.ID, c.Verdict, c.Summary)
|
||||
@@ -564,11 +687,23 @@ func (h *Holder) Checked(c CheckedEvent) {
|
||||
}
|
||||
h.moved(d, []Transition{t})
|
||||
}
|
||||
if d.State == Rejected && c.Trunk != "" && d.Check != nil && !sameCommit(d.Check.Trunk, c.Trunk) {
|
||||
// Judged again against another trunk commit — the merge tool's ask of a verdict out of date, or a check asked
|
||||
// elsewhere (novox/hq ADR 0323 §5, §7): taken as the recheck it is.
|
||||
t, err := Apply(d, EvRecheck, true, Facts{Now: now, By: byOwner, Why: fmt.Sprintf("judged again against trunk %s "+
|
||||
"(it was rejected against trunk %s)", shortOf(c.Trunk, 12), shortOf(d.Check.Trunk, 12))})
|
||||
if err != nil {
|
||||
h.refuse(err)
|
||||
return
|
||||
}
|
||||
h.moved(d, []Transition{t})
|
||||
}
|
||||
if d.State != Proposed {
|
||||
h.Logf("[mesh-delivery] %s is %s: a verdict for it now is history (%s)", d.ID, d.State, c.Verdict)
|
||||
return
|
||||
}
|
||||
v := &Verdict{ID: c.ID, On: c.On, At: now, Gate: c.Verdict, Summary: c.Summary, Report: c.Report}
|
||||
v := &Verdict{ID: c.ID, On: c.On, At: now, Gate: c.Verdict, Summary: c.Summary, Report: c.Report,
|
||||
Head: shortOf(c.Commit, 12), Trunk: c.Trunk, Beside: c.Beside, Took: c.Took}
|
||||
if c.Gate != nil {
|
||||
v.Gate, v.Summary = c.Gate.Verdict, c.Gate.Summary
|
||||
v.Modules, v.Dependents = c.Gate.Modules, c.Gate.Dependents
|
||||
@@ -579,10 +714,14 @@ func (h *Holder) Checked(c CheckedEvent) {
|
||||
if v.Gate == "" {
|
||||
v.Gate = "error"
|
||||
}
|
||||
d.Check = v
|
||||
d.Check, d.AskedAgain = v, nil
|
||||
if c.Plan != nil {
|
||||
d.Plan = c.Plan
|
||||
}
|
||||
if why := d.outOfDate(); why != "" {
|
||||
// Kept and shown, and it does not count: the delivery stays proposed (ADR 0323 §7).
|
||||
h.Logf("[mesh-delivery] %s: its verdict %s is %s", d.ID, c.ID, why)
|
||||
}
|
||||
h.settleAndKeep(d)
|
||||
}
|
||||
|
||||
@@ -953,12 +1092,16 @@ func (h *Holder) plan() []func() {
|
||||
// grace, it is stopped saying so — never left waiting for a start that cannot come.
|
||||
for _, d := range h.deliveries {
|
||||
ended := d.endedUnstarted()
|
||||
if d.coveredStop() && !d.CoverTold {
|
||||
h.tellCovered(d, now) // covered before this holder told it, or before a restart (novox/hq issue 480)
|
||||
continue
|
||||
}
|
||||
if strandedUncovered(d) {
|
||||
// Stopped and final, but still read for a later merge that delivered its modules: until one did,
|
||||
// `stalled` keeps listing it (review of novox/hq issue 474).
|
||||
var ask []string
|
||||
if d.CoveredBy, ask = h.coverOf(d); d.CoveredBy != "" {
|
||||
h.keep(d)
|
||||
h.tellCovered(d, now)
|
||||
}
|
||||
for _, in := range ask {
|
||||
if a := h.askContains(d, in, now); a != nil {
|
||||
@@ -1103,6 +1246,25 @@ func (h *Holder) laterOnItsWay(d *Delivery) string {
|
||||
return ""
|
||||
}
|
||||
|
||||
// tellCovered tells the forge that a stop is covered since, once: its status says covered and by which merge, its view
|
||||
// says so, and its note keeps the line on its head and its merge commit (novox/hq issue 480). No transition: the stop
|
||||
// stays final, and the bus's transition event is not said again.
|
||||
func (h *Holder) tellCovered(d *Delivery, now time.Time) {
|
||||
d.CoverTold = true
|
||||
line := oneLine(fmt.Sprintf("%s delivery %s: stopped, %s", now.UTC().Format(time.RFC3339), d.ID, d.coveredStopSaid()))
|
||||
d.Owed = append(d.Owed, Effect{Kind: EffectNote, Commit: d.Commit, Line: line, Since: now})
|
||||
if d.MergedAs != "" && !sameCommit(d.MergedAs, d.Commit) {
|
||||
d.Owed = append(d.Owed, Effect{Kind: EffectNote, Commit: d.MergedAs, Line: line, Since: now})
|
||||
}
|
||||
if d.shownOnItsPullRequest() {
|
||||
d.owe(Effect{Kind: EffectView, Since: now})
|
||||
state, desc := statusOf(d)
|
||||
d.owe(Effect{Kind: EffectStatus, Commit: d.Commit, Context: "mesh/delivery", State: state, Line: desc, Since: now})
|
||||
}
|
||||
h.Logf("[mesh-delivery] %s: stopped, %s", d.ID, d.coveredStopSaid())
|
||||
h.keep(d)
|
||||
}
|
||||
|
||||
// strandedUncovered is whether a delivery stopped because its ended walk delivered nothing is still uncovered: a
|
||||
// merged change that never reached the machines, which `stalled` keeps listing until a later delivered merge
|
||||
// carries its modules (review of novox/hq issue 474).
|
||||
@@ -1408,14 +1570,19 @@ func (h *Holder) planGroup(g *Group, members []*Delivery, now time.Time) []func(
|
||||
})
|
||||
return asks
|
||||
}
|
||||
// The composed check, once every member is ready and the order holds.
|
||||
// The composed check, once every member is ready and the order holds. Asked again for new heads, even once the
|
||||
// group is closed: a member rebased or pushed is a new head, and the group waits for its composed check rather
|
||||
// than being held for a person (novox/hq ADR 0323 §6, issue 406). One out of date is asked again once a member
|
||||
// was judged again after it went out of date: a recheck, or the merge tool's ask.
|
||||
allReady := true
|
||||
for _, d := range members {
|
||||
if d.State != Ready {
|
||||
// In a closed group a member on the trunk counts — merged and on its way, or already delivered (reached).
|
||||
if d.State != Ready && !(g.Closed && d.MergedAs != "" && (!d.State.Final() || d.State.Reached())) {
|
||||
allReady = false
|
||||
}
|
||||
}
|
||||
if allReady && len(g.Cycle) == 0 && !g.Closed && (g.Check == nil || !sameMembers(g.Check.Commits, commits)) {
|
||||
again := g.Check != nil && (!sameMembers(g.Check.Commits, commits) || g.Check.OutOfDate != "" && rejudgedSince(members, g.Check.OutOfDateAt))
|
||||
if allReady && len(g.Cycle) == 0 && (g.Check == nil && !g.Closed || again) {
|
||||
ms := h.inOrder(g, members)
|
||||
id := g.ID
|
||||
asks = append(asks, func() {
|
||||
@@ -1537,7 +1704,7 @@ func (h *Holder) turnsOf(g *Group, members []*Delivery, waits map[string]carryWa
|
||||
}
|
||||
// Composed for these very heads: a verdict on another set of heads is no verdict on these.
|
||||
composed := g.Check != nil && (g.Check.Verdict == "pass" || g.Check.Verdict == "warning") &&
|
||||
sameMembers(g.Check.Commits, commitsOf(members))
|
||||
sameMembers(g.Check.Commits, commitsOf(members)) && g.Check.OutOfDate == ""
|
||||
// A member's turn registered, the members after it that the same walk carries take theirs too: one walk is let
|
||||
// go once, for all of them (ADR 0276 decision 5). A member on another walk, or not at its turn, waits.
|
||||
var asks []func()
|
||||
@@ -1623,9 +1790,11 @@ func (h *Holder) turnsOf(g *Group, members []*Delivery, waits map[string]carryWa
|
||||
}
|
||||
return asks
|
||||
case d.State == Published:
|
||||
// Asked and not answered is not "did not pass": it waits, published, for the answer (issue 316).
|
||||
if !composed && d.HeldWhy == "" && g.Check != nil && g.Check.Verdict == "" &&
|
||||
sameMembers(g.Check.Commits, commitsOf(members)) {
|
||||
// Asked and not answered is not "did not pass": it waits, published, for the answer (issue 316). Nor is a
|
||||
// composed check of other heads — a member rebased or pushed — or one out of date: it runs again for these
|
||||
// heads, and the member waits for it (novox/hq ADR 0323 §6, issue 406).
|
||||
if !composed && d.HeldWhy == "" && g.Check != nil && (g.Check.Verdict == "" || g.Check.OutOfDate != "" ||
|
||||
!sameMembers(g.Check.Commits, commitsOf(members))) {
|
||||
return asks
|
||||
}
|
||||
if !composed && d.HeldWhy == "" {
|
||||
@@ -2146,12 +2315,23 @@ func GroupState(g *Group, members []*Delivery) (string, string) {
|
||||
return "delivered", "every member is delivered"
|
||||
case g.Closed && n > 0 && ended == n:
|
||||
return "stopped", supersededWhy + strings.Join(uncovered, ", ")
|
||||
case g.Closed && g.Check != nil && (g.Check.OutOfDate != "" || g.Check.Verdict == "" ||
|
||||
!sameMembers(g.Check.Commits, commitsOf(members))):
|
||||
// A member rebased or pushed once the group was delivering, or its composed check out of date: it is composed
|
||||
// again for these heads, and the group is checking until then, never held (novox/hq ADR 0323 §6, issue 406).
|
||||
if g.Check.OutOfDate != "" {
|
||||
return "checking", g.Check.OutOfDate
|
||||
}
|
||||
return "checking", "composed again for its members' new heads"
|
||||
case g.Closed:
|
||||
return "delivering", fmt.Sprintf("%d of %d delivered", reached, n)
|
||||
case len(g.Cycle) > 0:
|
||||
return "rejected", cycleSaid(g)
|
||||
case count[Rejected] > 0:
|
||||
return "rejected", "a member's own check failed"
|
||||
case g.Check != nil && g.Check.OutOfDate != "":
|
||||
// Out of date counts as outstanding (ADR 0239 §18, novox/hq ADR 0323 §7): checking, saying why.
|
||||
return "checking", g.Check.OutOfDate
|
||||
case g.Check != nil && g.Check.Verdict != "" && g.Check.Verdict != "pass" && g.Check.Verdict != "warning":
|
||||
return "rejected", "composed together: " + g.Check.Summary
|
||||
case count[Ready] == n && g.Check != nil && sameMembers(g.Check.Commits, commitsOf(members)) &&
|
||||
@@ -2217,3 +2397,13 @@ func cycleSaid(g *Group) string {
|
||||
}
|
||||
return why
|
||||
}
|
||||
|
||||
// rejudgedSince is whether a member was judged again after a moment: a verdict of its own heard since.
|
||||
func rejudgedSince(members []*Delivery, at time.Time) bool {
|
||||
for _, d := range members {
|
||||
if d.Check != nil && d.Check.At.After(at) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
@@ -0,0 +1,463 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// **A verdict holds for the trunk commit it judged** (novox/hq ADR 0323, issue 341). The check judges the head
|
||||
// merged into a named trunk commit; when another merge moves that trunk, a ready delivery goes back to proposed,
|
||||
// a rejected one stays rejected, and nothing is asked. A verdict for an old trunk is kept and shown and does not
|
||||
// count, so the table reaches a fixed point. A group's composed check goes out of date on a merge that is not one of
|
||||
// its members, and runs again for new heads instead of holding the group.
|
||||
|
||||
const (
|
||||
trunk0 = "a2c2c35000000000000000000000000000000000"
|
||||
merge1 = "d28ca95000000000000000000000000000000000"
|
||||
)
|
||||
|
||||
// judged is a verdict of a head merged into a trunk commit.
|
||||
func judged(repo string, number int, sha, gate, trunk string) CheckedEvent {
|
||||
c := verdict(repo, number, sha, gate)
|
||||
c.Trunk = shortOf(trunk, 12)
|
||||
c.ID = "check-of-" + sha + "-at-" + shortOf(trunk, 8)
|
||||
c.Took = "4m12s"
|
||||
c.Beside = map[string]string{"mesh-catalog": "0123456789ab"}
|
||||
plan := *c.Plan
|
||||
plan.Head, plan.Trunk = sha, c.Trunk
|
||||
c.Plan = &plan
|
||||
return c
|
||||
}
|
||||
|
||||
func TestATrunkMoveSendsAReadyDeliveryBackToProposed(t *testing.T) {
|
||||
const host = "novox/mesh-host"
|
||||
const h64, h66, h67 = "6464646464646464", "dfb1415dfb1415df", "6767676767676767"
|
||||
w := newWorld(t)
|
||||
for _, p := range []PullEvent{pr(host, 64, h64, "a"), pr(host, 66, h66, "b"), pr(host, 67, h67, "c")} {
|
||||
w.h.PullUpdated(p)
|
||||
}
|
||||
w.h.Checked(judged(host, 64, h64, "pass", trunk0))
|
||||
w.h.Checked(judged(host, 66, h66, "pass", trunk0))
|
||||
w.h.Checked(judged(host, 67, h67, "fail", trunk0))
|
||||
id66, id67 := IDOf(host, h66), IDOf(host, h67)
|
||||
if s := w.state(id66); s != Ready {
|
||||
t.Fatalf("#66 checked green against the trunk is %s", s)
|
||||
}
|
||||
asked := len(w.ctl.checks)
|
||||
|
||||
// #64 merges: the trunk moves to its merge commit.
|
||||
w.h.PullMerged(merged(host, 64, h64, merge1))
|
||||
w.settleAll()
|
||||
d := w.delivery(id66)
|
||||
if d.State != Proposed {
|
||||
t.Fatalf("#66's verdict names trunk %s, the trunk is now %s, and it is %s", trunk0[:12], merge1[:12], d.State)
|
||||
}
|
||||
last := d.Transitions[len(d.Transitions)-1]
|
||||
if last.Event != EvTrunkMoved || !strings.Contains(last.Why, trunk0[:12]) || !strings.Contains(last.Why, merge1[:12]) {
|
||||
t.Fatalf("its transition was %s, %q: not trunk moved with both commits", last.Event, last.Why)
|
||||
}
|
||||
if d.Check == nil || d.Check.Trunk != trunk0[:12] {
|
||||
t.Fatalf("its out-of-date verdict was not kept to be shown: %+v", d.Check)
|
||||
}
|
||||
if s := w.state(id67); s != Rejected {
|
||||
t.Fatalf("a rejected delivery went to %s on the trunk's move", s)
|
||||
}
|
||||
if len(w.ctl.checks) != asked {
|
||||
t.Fatalf("the trunk's move asked a check: %v", w.ctl.checks[asked:])
|
||||
}
|
||||
|
||||
// Out of date and not asked: no bound, so stalled does not list it, however long it waits.
|
||||
w.later(5 * time.Hour)
|
||||
w.settleAll()
|
||||
for _, l := range w.h.Stalled() {
|
||||
if l.ID == id66 {
|
||||
t.Fatalf("stalled lists a proposed delivery whose check is out of date and not asked: %+v", l)
|
||||
}
|
||||
}
|
||||
if len(w.ctl.checks) != asked {
|
||||
t.Fatalf("an out-of-date check was asked by this owner: %v", w.ctl.checks[asked:])
|
||||
}
|
||||
|
||||
// A late verdict for the old trunk is kept, does not count, and moves nothing: the table's fixed point.
|
||||
before := len(w.delivery(id66).Transitions)
|
||||
late := judged(host, 66, h66, "pass", trunk0)
|
||||
late.ID = "check-late"
|
||||
w.h.Checked(late)
|
||||
w.settleAll()
|
||||
d = w.delivery(id66)
|
||||
if d.State != Proposed || len(d.Transitions) != before || d.Check == nil || d.Check.ID != "check-late" {
|
||||
t.Fatalf("a late verdict for an old trunk left it %s after %d transition(s), verdict %+v", d.State,
|
||||
len(d.Transitions)-before, d.Check)
|
||||
}
|
||||
if ts := settle(d, Facts{Now: w.now}); len(ts) != 0 {
|
||||
t.Fatalf("the table took %d more transition(s) on a verdict that does not count", len(ts))
|
||||
}
|
||||
|
||||
// The check asked again judges the head merged into the trunk's head: it counts.
|
||||
w.h.Checked(judged(host, 66, h66, "pass", merge1))
|
||||
w.settleAll()
|
||||
if s := w.state(id66); s != Ready {
|
||||
t.Fatalf("a verdict judged against the trunk's head left it %s", s)
|
||||
}
|
||||
}
|
||||
|
||||
// A rejected delivery judged again against another trunk — the merge tool's ask — is taken as its recheck.
|
||||
func TestARejectedDeliveryJudgedAgainstANewTrunkIsRecheckedByTheVerdict(t *testing.T) {
|
||||
const repo = "novox/mesh-lab"
|
||||
const head = "1a2b3c4d5e6f7a8b"
|
||||
w := newWorld(t)
|
||||
w.h.PullUpdated(pr(repo, 9, head, "x"))
|
||||
w.h.Checked(judged(repo, 9, head, "fail", trunk0))
|
||||
id := IDOf(repo, head)
|
||||
if s := w.state(id); s != Rejected {
|
||||
t.Fatalf("it is %s", s)
|
||||
}
|
||||
w.h.PullMerged(merged(repo, 8, "8888888888888888", merge1))
|
||||
w.h.Checked(judged(repo, 9, head, "pass", merge1))
|
||||
w.settleAll()
|
||||
d := w.delivery(id)
|
||||
if d.State != Ready || d.Check.Trunk != merge1[:12] {
|
||||
t.Fatalf("judged green against the trunk's head it is %s with %+v", d.State, d.Check)
|
||||
}
|
||||
}
|
||||
|
||||
// **A verdict names the trunk commit it judged** in `checks` and `show`, and the kept plan records it (ADR 0323 §3).
|
||||
func TestChecksAndShowNameTheHeadAndTheTrunkCommit(t *testing.T) {
|
||||
const repo = "novox/mesh-lab"
|
||||
const head = "1a2b3c4d5e6f7a8b"
|
||||
w := newWorld(t)
|
||||
w.h.PullUpdated(pr(repo, 9, head, "x"))
|
||||
w.h.Checked(judged(repo, 9, head, "pass", trunk0))
|
||||
id := IDOf(repo, head)
|
||||
|
||||
a, err := w.h.Checks(repo, 0, head)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := "head " + head[:12] + " merged into trunk " + trunk0[:12]
|
||||
if a.Judged != want || a.Verdict == nil || a.Verdict.Head != head[:12] || a.Verdict.Trunk != trunk0[:12] {
|
||||
t.Fatalf("checks says %q, verdict %+v; want %q", a.Judged, a.Verdict, want)
|
||||
}
|
||||
if !strings.Contains(a.Says, want) {
|
||||
t.Fatalf("checks' words do not say what was judged: %s", a.Says)
|
||||
}
|
||||
if a.Verdict.Beside["mesh-catalog"] != "0123456789ab" || a.Verdict.Took != "4m12s" {
|
||||
t.Fatalf("checks does not name the other repositories' commits or how long it took: %+v", a.Verdict)
|
||||
}
|
||||
shown, err := w.h.Show(id)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
d := shown.(map[string]any)["delivery"].(*Delivery)
|
||||
if d.Check.Head != head[:12] || d.Check.Trunk != trunk0[:12] {
|
||||
t.Fatalf("show gives the verdict %+v", d.Check)
|
||||
}
|
||||
if d.Plan == nil || d.Plan.Trunk != trunk0[:12] || d.Plan.Head != head {
|
||||
t.Fatalf("the kept delivery plan records %+v", d.Plan)
|
||||
}
|
||||
|
||||
w.h.PullMerged(merged(repo, 8, "8888888888888888", merge1))
|
||||
if a, _ = w.h.Checks(repo, 0, head); !strings.Contains(a.OutOfDate, merge1[:12]) {
|
||||
t.Fatalf("checks does not say the verdict is out of date: %+v", a)
|
||||
}
|
||||
}
|
||||
|
||||
// **A delivery group's composed check follows the same rule** (ADR 0323 §6, §7): a merge that is not one of its
|
||||
// members into a member's trunk puts it out of date and the group reads checking; its own members merging in its
|
||||
// order do not; and it runs again once a member is judged again.
|
||||
func TestAGroupsComposedCheckGoesOutOfDateOnAMergeThatIsNotItsMember(t *testing.T) {
|
||||
const ctl, cat = "novox/mesh-controller", "novox/mesh-catalog"
|
||||
const hc, hk = "c1c1c1c1c1c1c1c1", "k1k1k1k1k1k1k1k1"
|
||||
idc, idk := IDOf(ctl, hc), IDOf(cat, hk)
|
||||
start := func(t *testing.T) *world {
|
||||
w := newWorld(t)
|
||||
w.h.PullUpdated(pr(ctl, 1, hc, "feat/g"))
|
||||
w.h.PullUpdated(pr(cat, 2, hk, "feat/g"))
|
||||
w.h.Checked(judged(ctl, 1, hc, "pass", trunk0))
|
||||
w.h.Checked(judged(cat, 2, hk, "pass", trunk0))
|
||||
w.settleAll()
|
||||
g := w.h.groups["feat/g"]
|
||||
if g == nil || g.Check == nil {
|
||||
t.Fatalf("no composed check was asked: %+v", g)
|
||||
}
|
||||
w.h.Checked(CheckedEvent{Group: "feat/g", ID: g.Check.Asked, Verdict: "pass", Summary: "composed",
|
||||
Members: []struct {
|
||||
Owner string `json:"owner"`
|
||||
Repo string `json:"repo"`
|
||||
Commit string `json:"commit"`
|
||||
Trunk string `json:"trunk,omitempty"`
|
||||
}{{Owner: "novox", Repo: "mesh-controller", Commit: hc, Trunk: trunk0[:12]},
|
||||
{Owner: "novox", Repo: "mesh-catalog", Commit: hk, Trunk: trunk0[:12]}}})
|
||||
w.settleAll()
|
||||
if state, _ := GroupState(g, w.h.membersOf(g)); state != "ready" {
|
||||
t.Fatalf("the group composed green is %s", state)
|
||||
}
|
||||
if g.Check.Trunks[idc] != trunk0[:12] {
|
||||
t.Fatalf("the composed check names its members' trunks %v", g.Check.Trunks)
|
||||
}
|
||||
return w
|
||||
}
|
||||
|
||||
t.Run("a merge that is not a member", func(t *testing.T) {
|
||||
w := start(t)
|
||||
asked := len(w.ctl.checks)
|
||||
w.h.PullMerged(merged(ctl, 5, "5555555555555555", merge1))
|
||||
w.settleAll()
|
||||
g := w.h.groups["feat/g"]
|
||||
state, why := GroupState(g, w.h.membersOf(g))
|
||||
if state != "checking" || g.Check.OutOfDate == "" {
|
||||
t.Fatalf("the group is %s (%s) after a merge that is not its member moved a member's trunk", state, why)
|
||||
}
|
||||
for _, head := range []string{ctl + "@" + hc, cat + "@" + hk} {
|
||||
s := w.forge.statuses[head+" "+ctxGroup]
|
||||
if !strings.HasPrefix(s, "pending ") || !strings.Contains(s, "out of date") {
|
||||
t.Fatalf("mesh/delivery-group on %s says %q, not pending with the reason", head, s)
|
||||
}
|
||||
}
|
||||
if len(w.ctl.checks) != asked {
|
||||
t.Fatalf("the trunk's move asked a check: %v", w.ctl.checks[asked:])
|
||||
}
|
||||
// The controller's member judged again against the trunk's head: the composed check runs again.
|
||||
w.later(time.Minute)
|
||||
w.h.Checked(judged(ctl, 1, hc, "pass", merge1))
|
||||
w.settleAll()
|
||||
if len(w.ctl.checks) == asked || !strings.HasPrefix(w.ctl.checks[len(w.ctl.checks)-1], "feat/g:") {
|
||||
t.Fatalf("the composed check was not asked again once a member was judged again: %v", w.ctl.checks)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("its own member merging", func(t *testing.T) {
|
||||
w := start(t)
|
||||
mc := merged(ctl, 1, hc, merge1)
|
||||
mc.Head = "feat/g"
|
||||
w.h.PullMerged(mc)
|
||||
w.settleAll()
|
||||
g := w.h.groups["feat/g"]
|
||||
if g.Check.OutOfDate != "" {
|
||||
t.Fatalf("the group's own member merging put its composed check out of date: %s", g.Check.OutOfDate)
|
||||
}
|
||||
if s := w.state(idk); s != Ready {
|
||||
t.Fatalf("the catalogue's member, in another repository, is %s", s)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// **After a member is rebased or pushed, the composed check runs again for the new heads** and the group reads
|
||||
// checking, not held (ADR 0323 §6, issue 406's second symptom): its first member merged, closing the group, and then
|
||||
// its second was rebased just before its own merge.
|
||||
func TestARebasedMemberIsComposedAgainNotHeld(t *testing.T) {
|
||||
const ctl, cat = "novox/mesh-controller", "novox/mesh-catalog"
|
||||
const hc, hk, hc2 = "c1c1c1c1c1c1c1c1", "k1k1k1k1k1k1k1k1", "c2c2c2c2c2c2c2c2"
|
||||
idc, idk, idc2 := IDOf(ctl, hc), IDOf(cat, hk), IDOf(ctl, hc2)
|
||||
w := newWorld(t)
|
||||
w.h.PullUpdated(pr(ctl, 1, hc, "feat/g"))
|
||||
w.h.PullUpdated(pr(cat, 2, hk, "feat/g"))
|
||||
w.h.Checked(judged(ctl, 1, hc, "pass", trunk0))
|
||||
w.h.Checked(judged(cat, 2, hk, "pass", trunk0))
|
||||
w.settleAll()
|
||||
g := w.h.groups["feat/g"]
|
||||
if len(g.Order) != 2 || g.Order[0] != idk {
|
||||
t.Fatalf("the group's order is %v: the catalogue's member first is what this replays", g.Order)
|
||||
}
|
||||
w.h.Checked(CheckedEvent{Group: "feat/g", ID: g.Check.Asked, Verdict: "pass", Summary: "composed"})
|
||||
w.settleAll()
|
||||
|
||||
// The catalogue's member merges first: the group is closed.
|
||||
mk := merged(cat, 2, hk, merge1)
|
||||
mk.Head = "feat/g"
|
||||
w.h.PullMerged(mk)
|
||||
w.h.WalkMoved(w.ctl.put(aWalk("plan-cat", cat, merge1, true)))
|
||||
w.settleAll()
|
||||
if !g.Closed {
|
||||
t.Fatal("the group is not closed once a member merged")
|
||||
}
|
||||
|
||||
// The controller's member is rebased just before its merge: a new head, with its own check.
|
||||
w.h.PullUpdated(pr(ctl, 1, hc2, "feat/g"))
|
||||
w.settleAll()
|
||||
if d := w.delivery(idc2); d.Group != "feat/g" {
|
||||
t.Fatalf("the rebased head did not take its place in its closed group: %q", d.Group)
|
||||
}
|
||||
if containsString(g.Members, idc) || !containsString(g.Members, idc2) || len(g.Order) != 2 || g.Order[1] != idc2 {
|
||||
t.Fatalf("the group's members are %v in the order %v", g.Members, g.Order)
|
||||
}
|
||||
if state, why := GroupState(g, w.h.membersOf(g)); state != "checking" {
|
||||
t.Fatalf("the group reads %s (%s), not checking", state, why)
|
||||
}
|
||||
w.h.Checked(judged(ctl, 1, hc2, "pass", trunk0))
|
||||
w.settleAll()
|
||||
asked := w.ctl.checks[len(w.ctl.checks)-1]
|
||||
if !strings.HasPrefix(asked, "feat/g:") || !strings.Contains(asked, idc2) || !strings.Contains(asked, idk) {
|
||||
t.Fatalf("the composed check was not asked again for the new heads: %v", w.ctl.checks)
|
||||
}
|
||||
if state, why := GroupState(g, w.h.membersOf(g)); state != "checking" {
|
||||
t.Fatalf("the group reads %s (%s), not checking", state, why)
|
||||
}
|
||||
for _, id := range []string{idk, idc2} {
|
||||
if d := w.delivery(id); d.State == Held {
|
||||
t.Fatalf("%s is held for a person: %s", id, d.HeldWhy)
|
||||
}
|
||||
}
|
||||
// Its verdict for the new heads: the rebased member merges and takes its turn.
|
||||
w.h.Checked(CheckedEvent{Group: "feat/g", ID: g.Check.Asked, Verdict: "pass", Summary: "composed again"})
|
||||
mc := merged(ctl, 1, hc2, "e0e0e0e0e0e0e0e0")
|
||||
mc.Head = "feat/g"
|
||||
w.h.PullMerged(mc)
|
||||
w.h.WalkMoved(w.ctl.put(aWalk("plan-ctl", ctl, "e0e0e0e0e0e0e0e0", true)))
|
||||
w.settleAll()
|
||||
if d := w.delivery(idc2); d.State == Held {
|
||||
t.Fatalf("the rebased member, composed green for the new heads, is held: %s", d.HeldWhy)
|
||||
}
|
||||
}
|
||||
|
||||
// A head whose merge check the forge's holder put out of date is not a check asked and not answered: `stalled` does
|
||||
// not list it, however long it waits — rejected, or with no delivery here — until its check is asked (ADR 0323 §7).
|
||||
func TestAnOutOfDateCheckOnTheForgeIsNotSaidUnanswered(t *testing.T) {
|
||||
const repo = "novox/mesh-lab"
|
||||
const head = "1a2b3c4d5e6f7a8b9c0d"
|
||||
w := newWorld(t)
|
||||
w.h.PullUpdated(pr(repo, 9, head, "x"))
|
||||
w.h.Checked(judged(repo, 9, head, "fail", trunk0))
|
||||
words := outOfDateWords(trunk0, merge1)
|
||||
pending := func(description, at string) map[string]HeadCheck {
|
||||
return map[string]HeadCheck{ctxGate: {State: "pending", Description: description, UpdatedAt: at},
|
||||
ctxRepo: {State: "pending", Description: description, UpdatedAt: at}}
|
||||
}
|
||||
w.forge.open = []OpenPull{{Owner: "novox", Repo: "mesh-lab", Number: 9, Base: "main", HeadSHA: head,
|
||||
Required: []string{ctxGate, ctxRepo}, Statuses: []string{ctxGate, ctxRepo}, Checks: pending(words, "2026-10-06T12:01:00Z")}}
|
||||
w.h.SeeOpenPulls()
|
||||
w.later(3 * time.Hour)
|
||||
w.h.SeeOpenPulls()
|
||||
for _, l := range w.h.Stalled() {
|
||||
if strings.HasPrefix(l.ID, repo) {
|
||||
t.Fatalf("an out-of-date check nobody asked is said stalled: %+v", l)
|
||||
}
|
||||
}
|
||||
// Asked by the merge tool: pending anew, and its bound runs from then.
|
||||
w.forge.open[0].Checks = pending("waiting in the build queue since 15:01: asked by the merge tool", "2026-10-06T15:01:00Z")
|
||||
w.h.SeeOpenPulls()
|
||||
w.later(2 * time.Hour)
|
||||
w.h.SeeOpenPulls()
|
||||
said := false
|
||||
for _, l := range w.h.Stalled() {
|
||||
said = said || l.State == Unanswered
|
||||
}
|
||||
if !said {
|
||||
t.Fatal("a check the merge tool asked and that never answered is not said")
|
||||
}
|
||||
}
|
||||
|
||||
// The merge tool's ask announces an out-of-date head again: its one-hour bound runs from then, and a check that never
|
||||
// answers is said stalled (ADR 0323 §7).
|
||||
func TestAnOutOfDateDeliveryAskedAgainIsBoundFromTheAsk(t *testing.T) {
|
||||
const repo = "novox/mesh-lab"
|
||||
const head = "1a2b3c4d5e6f7a8b"
|
||||
w := newWorld(t)
|
||||
w.h.PullUpdated(pr(repo, 9, head, "x"))
|
||||
w.h.Checked(judged(repo, 9, head, "pass", trunk0))
|
||||
w.h.PullMerged(merged(repo, 8, "8888888888888888", merge1))
|
||||
id := IDOf(repo, head)
|
||||
w.later(3 * time.Hour)
|
||||
w.h.PullUpdated(pr(repo, 9, head, "x")) // the merge tool asked its check
|
||||
w.later(30 * time.Minute)
|
||||
for _, l := range w.h.Stalled() {
|
||||
if l.ID == id {
|
||||
t.Fatalf("listed half an hour after its check was asked again: %+v", l)
|
||||
}
|
||||
}
|
||||
w.later(time.Hour)
|
||||
said := false
|
||||
for _, l := range w.h.Stalled() {
|
||||
if l.ID == id {
|
||||
said = true
|
||||
if !strings.Contains(l.Says, "asked again") || !strings.Contains(l.H2, "operator") {
|
||||
t.Fatalf("its stalled line says %+v", l)
|
||||
}
|
||||
}
|
||||
}
|
||||
if !said {
|
||||
t.Fatal("a check asked again that never answered is not said")
|
||||
}
|
||||
// Its verdict judged against the trunk's head: ready, and nothing bounds it as proposed.
|
||||
w.h.Checked(judged(repo, 9, head, "pass", merge1))
|
||||
if d := w.delivery(id); d.State != Ready || d.AskedAgain != nil {
|
||||
t.Fatalf("judged again it is %s, asked again %v", d.State, d.AskedAgain)
|
||||
}
|
||||
}
|
||||
|
||||
// The same, with the first member already delivered when the second is rebased: a delivered member counts for the
|
||||
// composed check of the new heads, which is asked, and the rebased member is never held (review of ADR 0323 §6).
|
||||
func TestARebasedMemberAfterAnotherWasDeliveredIsComposedAgain(t *testing.T) {
|
||||
const ctl, cat = "novox/mesh-controller", "novox/mesh-catalog"
|
||||
const hc, hk, hc2 = "c1c1c1c1c1c1c1c1", "k1k1k1k1k1k1k1k1", "c2c2c2c2c2c2c2c2"
|
||||
idk, idc2 := IDOf(cat, hk), IDOf(ctl, hc2)
|
||||
w := newWorld(t)
|
||||
w.h.PullUpdated(pr(ctl, 1, hc, "feat/g"))
|
||||
w.h.PullUpdated(pr(cat, 2, hk, "feat/g"))
|
||||
w.h.Checked(judged(ctl, 1, hc, "pass", trunk0))
|
||||
w.h.Checked(judged(cat, 2, hk, "pass", trunk0))
|
||||
w.settleAll()
|
||||
g := w.h.groups["feat/g"]
|
||||
w.h.Checked(CheckedEvent{Group: "feat/g", ID: g.Check.Asked, Verdict: "pass", Summary: "composed"})
|
||||
w.settleAll()
|
||||
|
||||
// The catalogue's member merges and its walk is done: delivered.
|
||||
mk := merged(cat, 2, hk, merge1)
|
||||
mk.Head = "feat/g"
|
||||
w.h.PullMerged(mk)
|
||||
walk := w.ctl.put(aWalk("plan-cat", cat, merge1, true))
|
||||
w.h.WalkMoved(walk)
|
||||
w.settleAll()
|
||||
walk = w.ctl.walk("plan-cat")
|
||||
walk.State = walkDone
|
||||
w.h.WalkMoved(w.ctl.put(walk))
|
||||
w.settleAll()
|
||||
if s := w.state(idk); s != Delivered {
|
||||
t.Fatalf("the catalogue's member is %s, not delivered", s)
|
||||
}
|
||||
|
||||
// The controller's member is rebased, and judged.
|
||||
w.h.PullUpdated(pr(ctl, 1, hc2, "feat/g"))
|
||||
w.h.Checked(judged(ctl, 1, hc2, "pass", trunk0))
|
||||
w.settleAll()
|
||||
asked := w.ctl.checks[len(w.ctl.checks)-1]
|
||||
if !strings.HasPrefix(asked, "feat/g:") || !strings.Contains(asked, idc2) {
|
||||
t.Fatalf("the composed check was not asked again for the new heads: %v", w.ctl.checks)
|
||||
}
|
||||
if state, why := GroupState(g, w.h.membersOf(g)); state != "checking" {
|
||||
t.Fatalf("the group reads %s (%s), not checking", state, why)
|
||||
}
|
||||
w.h.Checked(CheckedEvent{Group: "feat/g", ID: g.Check.Asked, Verdict: "pass", Summary: "composed again"})
|
||||
mc := merged(ctl, 1, hc2, "e0e0e0e0e0e0e0e0")
|
||||
mc.Head = "feat/g"
|
||||
w.h.PullMerged(mc)
|
||||
w.h.WalkMoved(w.ctl.put(aWalk("plan-ctl", ctl, "e0e0e0e0e0e0e0e0", true)))
|
||||
w.settleAll()
|
||||
if d := w.delivery(idc2); d.State == Held {
|
||||
t.Fatalf("the rebased member is held: %s", d.HeldWhy)
|
||||
}
|
||||
}
|
||||
|
||||
// A check this owner asks — a recheck — sends the base head the forge last announced the head with, so the
|
||||
// controller's verdict for a head no build judges names its trunk commit (ADR 0323 §3, review of mesh-controller#271).
|
||||
func TestARecheckSendsTheAnnouncedBaseHead(t *testing.T) {
|
||||
const repo = "novox/mesh-tools-go"
|
||||
const head = "1a2b3c4d5e6f7a8b"
|
||||
w := newWorld(t)
|
||||
p := pr(repo, 3, head, "x")
|
||||
p.BaseSHA = trunk0
|
||||
w.h.PullUpdated(p)
|
||||
id := IDOf(repo, head)
|
||||
if m := MemberOf(w.delivery(id)); m.BaseSHA != trunk0 {
|
||||
t.Fatalf("the member sends base head %q", m.BaseSHA)
|
||||
}
|
||||
p.BaseSHA = merge1 // announced again, by the merge tool
|
||||
w.h.PullUpdated(p)
|
||||
if m := MemberOf(w.delivery(id)); m.BaseSHA != merge1 {
|
||||
t.Fatalf("announced again, the member sends base head %q", m.BaseSHA)
|
||||
}
|
||||
if raw := string(mustJSON(MemberOf(w.delivery(id)))); !strings.Contains(raw, `"base_sha":"`+merge1+`"`) {
|
||||
t.Fatalf("the member as the controller reads it: %s", raw)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,95 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// novox/hq issue 403, ADR 0313: the issue tracker measures a fix's time from the move to fixing to delivered and
|
||||
// splits it at the pull request's opening and its merge. A transition says what it reads them by: when the pull
|
||||
// request was first heard (its first head, not a later one), when it merged, and the class of what it moves —
|
||||
// core, leaf, or none for a change that moves no module. `checks` says the same for a fix the tracker hears of late.
|
||||
|
||||
// transitionsTo are the transition events said for a delivery that moved to a state.
|
||||
func transitionsTo(w *world, id string, to State) []map[string]any {
|
||||
var out []map[string]any
|
||||
for _, e := range w.said {
|
||||
if e["event-kind"] == "transition" && e["id"] == id && e["to"] == string(to) {
|
||||
out = append(out, e)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func TestATransitionSaysWhenItsPullRequestOpenedMergedAndItsClass(t *testing.T) {
|
||||
w := newWorld(t)
|
||||
repo := "novox/mesh-catalog"
|
||||
opened := w.now
|
||||
first := pr(repo, 9, "head9first", "fix/403-x", "modules/app/a.go")
|
||||
w.h.PullUpdated(first)
|
||||
w.later(3 * time.Minute)
|
||||
second := pr(repo, 9, "head9second", "fix/403-x", "modules/app/a.go")
|
||||
w.h.PullUpdated(second)
|
||||
c := verdictMoving(repo, 9, "head9second", "app")
|
||||
c.Plan.Class = "core"
|
||||
w.h.Checked(c)
|
||||
w.later(2 * time.Minute)
|
||||
mergedWhen := w.now
|
||||
w.h.PullMerged(mergedAt(second, "merge9aaaaaa", mergedWhen))
|
||||
walk := aWalk("plan-merge9", repo, "merge9aaaaaa", false)
|
||||
walk.Delivery = &WalkDelivery{Merges: []WalkMerge{{Repository: repo, Commit: "merge9aaaaaa"}}}
|
||||
w.h.WalkMoved(w.ctl.put(walk))
|
||||
w.settleAll()
|
||||
walk = w.ctl.walk(walk.ID)
|
||||
walk.State, walk.Revision = walkDone, walk.Revision+1
|
||||
walk.Modules = map[string]*WalkModule{"app": {State: "built"}}
|
||||
w.h.WalkMoved(w.ctl.put(walk))
|
||||
w.settleAll()
|
||||
|
||||
id := IDOf(repo, "head9second")
|
||||
if s := w.state(id); s != Delivered {
|
||||
t.Fatalf("the merge is %s, not delivered", s)
|
||||
}
|
||||
said := transitionsTo(w, id, Delivered)
|
||||
if len(said) == 0 {
|
||||
t.Fatalf("no transition to delivered was said: %v", w.said)
|
||||
}
|
||||
e := said[len(said)-1]
|
||||
if e["opened"] != opened.UTC().Format(time.RFC3339) {
|
||||
t.Errorf("opened %v, want the first head's %s (not the second head's)", e["opened"], opened.Format(time.RFC3339))
|
||||
}
|
||||
if e["merged_at"] != mergedWhen.UTC().Format(time.RFC3339) {
|
||||
t.Errorf("merged_at %v, want %s", e["merged_at"], mergedWhen.Format(time.RFC3339))
|
||||
}
|
||||
if e["class"] != "core" {
|
||||
t.Errorf("class %v, want the plan's core", e["class"])
|
||||
}
|
||||
|
||||
a, err := w.h.Checks(repo, 9, "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if a.Delivery == nil || a.Delivery.Opened != opened.UTC().Format(time.RFC3339) || a.Delivery.Class != "core" ||
|
||||
a.Delivery.MergedAt != mergedWhen.UTC().Format(time.RFC3339) {
|
||||
t.Errorf("checks says %+v", a.Delivery)
|
||||
}
|
||||
}
|
||||
|
||||
// A plan that moves no module is class none, so the tracker gives its fix no budget; a plan from a controller that
|
||||
// says no class falls back to its walk's class once known, and says nothing before.
|
||||
func TestAFixClassIsNoneWhenItMovesNothing(t *testing.T) {
|
||||
for _, c := range []struct {
|
||||
why string
|
||||
d Delivery
|
||||
want string
|
||||
}{
|
||||
{"the planner's class", Delivery{Plan: &DeliveryPlan{Moved: []string{"app"}, Class: "leaf"}}, "leaf"},
|
||||
{"a plan that moves nothing, no class said", Delivery{Plan: planMoving("novox/x")}, "none"},
|
||||
{"no class said, its walk's known", Delivery{Plan: planMoving("novox/x", "app"), Time: &DeliveryTime{Class: "core"}}, "core"},
|
||||
{"no class said, nothing known", Delivery{Plan: planMoving("novox/x", "app")}, ""},
|
||||
} {
|
||||
if got := fixClass(&c.d); got != c.want {
|
||||
t.Errorf("%s: %q, want %q", c.why, got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,105 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// triedSaid is delivery_status's count of owed effects tried (novox/hq issue 407), read as a test reads it.
|
||||
func triedSaid(t *testing.T, w *world) map[string]any {
|
||||
t.Helper()
|
||||
got, ok := w.h.Status("listening")["owed-tried"].(map[string]any)
|
||||
if !ok {
|
||||
t.Fatalf("delivery_status says no owed-tried: %v", w.h.Status("listening"))
|
||||
}
|
||||
return got
|
||||
}
|
||||
|
||||
func countOf(t *testing.T, m map[string]any, key string) int {
|
||||
t.Helper()
|
||||
n, ok := m[key].(int)
|
||||
if !ok {
|
||||
t.Fatalf("owed-tried has no count %q: %v", key, m)
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
// novox/hq issue 407: delivery_status says how many owed effects were tried since this owner started, how
|
||||
// many were done, failed to be tried again, or were given up, and when the last was tried — so a window's
|
||||
// writes can be checked against the tries made in it (issue 326's rule) by reading it at both ends.
|
||||
func TestDeliveryStatusCountsTheOwedEffectsTried(t *testing.T) {
|
||||
w := newWorld(t)
|
||||
start := w.now
|
||||
tried := triedSaid(t, w)
|
||||
if countOf(t, tried, "tried") != 0 || countOf(t, tried, "done") != 0 || countOf(t, tried, "failed") != 0 ||
|
||||
countOf(t, tried, "given-up") != 0 {
|
||||
t.Fatalf("a holder that tried nothing says %v", tried)
|
||||
}
|
||||
if tried["since"] != start.Format(time.RFC3339) {
|
||||
t.Fatalf("its count is since %v, the holder started %s", tried["since"], start.Format(time.RFC3339))
|
||||
}
|
||||
if _, there := tried["last"]; there {
|
||||
t.Fatalf("nothing tried, yet a last try is said: %v", tried)
|
||||
}
|
||||
|
||||
// A head seen: its event and its note are owed and done.
|
||||
w.h.PullUpdated(pr("novox/app", 7, head, "feat/x", "src/a.go"))
|
||||
w.settleAll()
|
||||
tried = triedSaid(t, w)
|
||||
done := countOf(t, tried, "done")
|
||||
if done < len(w.said)+len(w.forge.notesOn(head)) || done == 0 {
|
||||
t.Fatalf("said %d and noted %d, yet done counts %d", len(w.said), len(w.forge.notesOn(head)), done)
|
||||
}
|
||||
if countOf(t, tried, "tried") != done || countOf(t, tried, "failed") != 0 {
|
||||
t.Fatalf("every try succeeded, yet %v", tried)
|
||||
}
|
||||
if tried["last"] != w.now.Format(time.RFC3339) {
|
||||
t.Fatalf("last try said %v, tried at %s", tried["last"], w.now.Format(time.RFC3339))
|
||||
}
|
||||
|
||||
// The forge away: its effects fail and are counted failed; quiet passes try nothing more.
|
||||
w.forge.down = true
|
||||
w.later(time.Minute)
|
||||
w.h.Checked(verdict("novox/app", 7, head, "pass"))
|
||||
w.h.Flush()
|
||||
tried = triedSaid(t, w)
|
||||
failed := countOf(t, tried, "failed")
|
||||
if failed == 0 {
|
||||
t.Fatalf("the forge away failed nothing: %v", tried)
|
||||
}
|
||||
if countOf(t, tried, "tried") != countOf(t, tried, "done")+failed+countOf(t, tried, "given-up") {
|
||||
t.Fatalf("tries do not add up: %v", tried)
|
||||
}
|
||||
if tried["last"] != w.now.Format(time.RFC3339) {
|
||||
t.Fatalf("last try said %v, tried at %s", tried["last"], w.now.Format(time.RFC3339))
|
||||
}
|
||||
before := countOf(t, tried, "tried")
|
||||
w.h.Flush() // before each failure's next try: nothing is tried
|
||||
if n := countOf(t, triedSaid(t, w), "tried"); n != before {
|
||||
t.Fatalf("a pass before any next try tried %d", n-before)
|
||||
}
|
||||
|
||||
// The forge back, past every next try: the failed effects are done.
|
||||
w.forge.down = false
|
||||
w.later(time.Hour)
|
||||
w.h.Flush()
|
||||
tried = triedSaid(t, w)
|
||||
if countOf(t, tried, "tried") != before+failed || countOf(t, tried, "done") <= done {
|
||||
t.Fatalf("the forge back: tried %d (was %d, %d failed), %v", countOf(t, tried, "tried"), before, failed, tried)
|
||||
}
|
||||
}
|
||||
|
||||
// The count is this run's: a restarted holder says it counts since its own start, from nothing.
|
||||
func TestOwedTriedCountsSinceTheStart(t *testing.T) {
|
||||
w := newWorld(t)
|
||||
w.h.PullUpdated(pr("novox/app", 7, head, "feat/x", "src/a.go"))
|
||||
w.settleAll()
|
||||
if countOf(t, triedSaid(t, w), "tried") == 0 {
|
||||
t.Fatal("nothing counted before the restart")
|
||||
}
|
||||
again := newWorldOver(t, w.store, w.ctl, w.forge, w.now.Add(time.Minute))
|
||||
tried := triedSaid(t, again)
|
||||
if countOf(t, tried, "tried") != 0 || tried["since"] != again.now.Format(time.RFC3339) {
|
||||
t.Fatalf("a restarted holder says %v", tried)
|
||||
}
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user