Compare commits
69
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
62ec68355c | ||
|
|
6a5e6510ca | ||
|
|
3f97d2d50d | ||
|
|
6112233649 | ||
|
|
8c2d7e5f97 | ||
|
|
21ebbffed4 | ||
|
|
31120ed580 | ||
|
|
cf304d8629 | ||
|
|
12d7852082 | ||
|
|
c9848f4fb8 | ||
|
|
c070bea782 | ||
|
|
7443938fef | ||
|
|
aee97bf8ad | ||
|
|
c816cafc00 | ||
|
|
2353350d79 | ||
|
|
d4b8b24701 | ||
|
|
de6657c846 | ||
|
|
f75e1a91d9 | ||
|
|
ebfec39937 | ||
|
|
24d05701d3 | ||
|
|
a95b3fa053 | ||
|
|
c2cd2e1eb1 | ||
|
|
aa62de2e78 | ||
|
|
66713c5a1e | ||
|
|
b771794f1c | ||
|
|
6c63a17cfa | ||
|
|
ed66a2e873 | ||
|
|
cc73b38671 | ||
|
|
0993c31ad4 | ||
|
|
82475691b2 | ||
|
|
89046e6613 | ||
|
|
dc89995d96 | ||
|
|
6c10288f0b | ||
|
|
789b0a9474 | ||
|
|
eb97a3d9a2 | ||
|
|
8b2e2300d8 | ||
|
|
2f2daea415 | ||
|
|
3b48f5962a | ||
|
|
2b1678f315 | ||
|
|
ab739d02dd | ||
|
|
a60020b51a | ||
|
|
a8d84d1730 | ||
|
|
9e8521afb5 | ||
|
|
8c60604eb7 | ||
|
|
13cd710071 | ||
|
|
0914a33fba | ||
|
|
b5417c28d0 | ||
|
|
5f6a41146c | ||
|
|
c71c513027 | ||
|
|
2e32f47e45 | ||
|
|
9492278d93 | ||
|
|
5ce2d07c7f | ||
|
|
373f339750 | ||
|
|
48f0662fde | ||
|
|
7913b56d66 | ||
|
|
ab1811d003 | ||
|
|
c79337579d | ||
|
|
3a84044ece | ||
|
|
c51357f8af | ||
|
|
2833a049aa | ||
|
|
6f7f07635e | ||
|
|
0f76b6e232 | ||
|
|
3e16c055be | ||
|
|
0361c44a03 | ||
|
|
d1debdcce8 | ||
|
|
be53f5167a | ||
|
|
2130f99612 | ||
|
|
ed5690cee4 | ||
|
|
70c1110c53 |
@@ -0,0 +1,3 @@
|
|||||||
|
module git.novox.be/novox/mesh-catalog/checks/mailstore
|
||||||
|
|
||||||
|
go 1.22
|
||||||
@@ -0,0 +1,81 @@
|
|||||||
|
// Command mailstore refuses any manifest but modules/mesh-mail claiming the mail-store seat (novox/hq ADR 0378
|
||||||
|
// §3, issue 800 part A): the operator's mail is read, searched and sent through this one seat, and a second
|
||||||
|
// claimant would be a second path to the same mail that ADR 0378 never accounted for.
|
||||||
|
//
|
||||||
|
// go run . <catalogue root>
|
||||||
|
//
|
||||||
|
// Exits 1 on a finding.
|
||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"sort"
|
||||||
|
)
|
||||||
|
|
||||||
|
// MailStoreSeat is the seat only mesh-mail may claim.
|
||||||
|
const MailStoreSeat = "mail-store"
|
||||||
|
|
||||||
|
// MailStoreModule is the one module allowed to claim it.
|
||||||
|
const MailStoreModule = "mesh-mail"
|
||||||
|
|
||||||
|
// Finding is one manifest refused: a module other than mesh-mail claiming mail-store.
|
||||||
|
type Finding struct {
|
||||||
|
Module string
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check reads every manifest under root/modules and answers which ones it refuses.
|
||||||
|
func Check(root string) ([]Finding, error) {
|
||||||
|
manifests, err := filepath.Glob(filepath.Join(root, "modules", "*", "module.json"))
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
sort.Strings(manifests)
|
||||||
|
var out []Finding
|
||||||
|
for _, path := range manifests {
|
||||||
|
raw, err := os.ReadFile(path)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
var m struct {
|
||||||
|
Module string `json:"module"`
|
||||||
|
Claims []struct {
|
||||||
|
Name string `json:"name"`
|
||||||
|
} `json:"claims"`
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal(raw, &m); err != nil {
|
||||||
|
return nil, fmt.Errorf("%s: %w", path, err)
|
||||||
|
}
|
||||||
|
if m.Module == MailStoreModule {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
for _, c := range m.Claims {
|
||||||
|
if c.Name == MailStoreSeat {
|
||||||
|
out = append(out, Finding{Module: m.Module})
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func main() {
|
||||||
|
root := "."
|
||||||
|
if len(os.Args) > 1 {
|
||||||
|
root = os.Args[1]
|
||||||
|
}
|
||||||
|
found, err := Check(root)
|
||||||
|
if err != nil {
|
||||||
|
fmt.Fprintln(os.Stderr, err)
|
||||||
|
os.Exit(1)
|
||||||
|
}
|
||||||
|
for _, f := range found {
|
||||||
|
fmt.Printf("%s claims the mail-store seat: only %s may (novox/hq ADR 0378 §3)\n", f.Module, MailStoreModule)
|
||||||
|
}
|
||||||
|
if len(found) > 0 {
|
||||||
|
os.Exit(1)
|
||||||
|
}
|
||||||
|
fmt.Println("mail-store: claimed only by mesh-mail")
|
||||||
|
}
|
||||||
@@ -0,0 +1,43 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func manifest(t *testing.T, root, module, claims string) {
|
||||||
|
t.Helper()
|
||||||
|
dir := filepath.Join(root, "modules", module)
|
||||||
|
if err := os.MkdirAll(dir, 0o755); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
body := `{"module":"` + module + `","claims":[` + claims + `]}`
|
||||||
|
if err := os.WriteFile(filepath.Join(dir, "module.json"), []byte(body), 0o644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAnotherModuleClaimingMailStoreIsRefused(t *testing.T) {
|
||||||
|
root := t.TempDir()
|
||||||
|
manifest(t, root, "mesh-mail", `{"name":"mail-store","scope":"mesh"}`)
|
||||||
|
manifest(t, root, "betterbird", `{"name":"mail-store","scope":"mesh"}`)
|
||||||
|
manifest(t, root, "dunst", `{"name":"node-notifier","scope":"node"}`)
|
||||||
|
found, err := Check(root)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(found) != 1 || found[0].Module != "betterbird" {
|
||||||
|
t.Fatalf("refused %+v", found)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTheCatalogueHasNone(t *testing.T) {
|
||||||
|
found, err := Check("../..")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(found) != 0 {
|
||||||
|
t.Fatalf("%+v", found)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -23,9 +23,51 @@
|
|||||||
#
|
#
|
||||||
# By hand: MESH_CHECK_MODULES=gitea sh merge-check-typescript.sh, in the toolchain image (it reads
|
# By hand: MESH_CHECK_MODULES=gitea sh merge-check-typescript.sh, in the toolchain image (it reads
|
||||||
# /app/node_modules), or with TOOLCHAIN_MODULES pointing at a node_modules that holds the same.
|
# /app/node_modules), or with TOOLCHAIN_MODULES pointing at a node_modules that holds the same.
|
||||||
|
#
|
||||||
|
# **In an agent's session, outside the toolchain image** (novox/hq issue 742), the same node_modules is
|
||||||
|
# assembled first, in a throwaway directory under the check's own workspace, the way the image's `deps` stage
|
||||||
|
# assembles it: the toolchain's own package.json and lock (mesh-tools' node-tools/, read from the forge this
|
||||||
|
# clone came from, so the list is never copied here) installed with `npm ci`, then the SDK the mesh last
|
||||||
|
# published (`@novox/mesh-sdk@latest`), the @novox scope from the forge's package registry, read without a
|
||||||
|
# credential. Node and npm are the dev-tools module's; nothing is installed on the machine, no npm
|
||||||
|
# configuration or cache of the account's is read or written. Only when neither TOOLCHAIN_MODULES nor
|
||||||
|
# /app/node_modules is there: the forge's check, in the image, fetches nothing, as before. The forge is
|
||||||
|
# MESH_FORGE (default: the origin remote's), the owner MESH_FORGE_OWNER (default: the origin's owner).
|
||||||
set -eu
|
set -eu
|
||||||
|
|
||||||
|
work=$(mktemp -d)
|
||||||
|
trap 'rm -rf "$work"' EXIT
|
||||||
|
|
||||||
deps="${TOOLCHAIN_MODULES:-/app/node_modules}"
|
deps="${TOOLCHAIN_MODULES:-/app/node_modules}"
|
||||||
|
if [ -z "${TOOLCHAIN_MODULES:-}" ] && [ ! -f "$deps/typescript/bin/tsc" ]; then
|
||||||
|
origin=$(git remote get-url origin 2>/dev/null || true)
|
||||||
|
forge="${MESH_FORGE:-$(printf '%s\n' "$origin" | sed -n 's#^\(https\{0,1\}://[^/]*\)/.*#\1#p')}"
|
||||||
|
owner="${MESH_FORGE_OWNER:-$(printf '%s\n' "$origin" | sed -n 's#^https\{0,1\}://[^/]*/\([^/]*\)/.*#\1#p')}"
|
||||||
|
if [ -z "$forge" ] || [ -z "$owner" ]; then
|
||||||
|
echo "NOT CHECKED: no TypeScript toolchain at $deps, and no forge to assemble one from (origin is '$origin'; set MESH_FORGE and MESH_FORGE_OWNER, or TOOLCHAIN_MODULES)"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
echo "no TypeScript toolchain at $deps: assembling one from $forge/$owner/mesh-tools node-tools/ and $owner's package registry (novox/hq issue 742)"
|
||||||
|
tc="$work/toolchain"
|
||||||
|
mkdir -p "$tc"
|
||||||
|
printf '@%s:registry=%s/api/packages/%s/npm/\n' "$owner" "$forge" "$owner" >"$work/npmrc"
|
||||||
|
# Read with Node's own fetch (dev-tools declares Node, not curl); a status that is not 2xx fails.
|
||||||
|
if ! { node -e '
|
||||||
|
const [base, dir] = process.argv.slice(1);
|
||||||
|
Promise.all(["package.json", "package-lock.json"].map(async (f) => {
|
||||||
|
const r = await fetch(base + f);
|
||||||
|
if (!r.ok) throw new Error(base + f + ": " + r.status);
|
||||||
|
require("fs").writeFileSync(dir + "/" + f, await r.text());
|
||||||
|
})).catch((e) => { console.error(e.message); process.exit(1); });' \
|
||||||
|
"$forge/$owner/mesh-tools/raw/branch/main/node-tools/" "$tc" &&
|
||||||
|
(cd "$tc" && export npm_config_userconfig="$work/npmrc" npm_config_cache="$work/npm-cache" &&
|
||||||
|
npm ci --ignore-scripts --no-audit --no-fund --loglevel=error &&
|
||||||
|
npm install --ignore-scripts --no-audit --no-fund --no-save --loglevel=error "@$owner/mesh-sdk@latest"); }; then
|
||||||
|
echo "NOT CHECKED: the TypeScript toolchain could not be assembled from $forge (novox/hq issue 742)"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
deps="$tc/node_modules"
|
||||||
|
fi
|
||||||
tsc="$deps/typescript/bin/tsc"
|
tsc="$deps/typescript/bin/tsc"
|
||||||
if [ ! -f "$tsc" ]; then
|
if [ ! -f "$tsc" ]; then
|
||||||
echo "NOT CHECKED: no TypeScript compiler at $tsc — this part runs in the TypeScript toolchain"
|
echo "NOT CHECKED: no TypeScript compiler at $tsc — this part runs in the TypeScript toolchain"
|
||||||
@@ -39,8 +81,6 @@ touched=$({
|
|||||||
printf '%s\n' "${MESH_CHECK_CHANGED:-}" | tr ',' '\n' | sed -n 's#^modules/\([^/]*\)/.*#\1#p'
|
printf '%s\n' "${MESH_CHECK_CHANGED:-}" | tr ',' '\n' | sed -n 's#^modules/\([^/]*\)/.*#\1#p'
|
||||||
} | sed '/^$/d' | sort -u)
|
} | sed '/^$/d' | sort -u)
|
||||||
|
|
||||||
work=$(mktemp -d)
|
|
||||||
trap 'rm -rf "$work"' EXIT
|
|
||||||
failed=""
|
failed=""
|
||||||
for m in $touched; do
|
for m in $touched; do
|
||||||
d="modules/$m"
|
d="modules/$m"
|
||||||
|
|||||||
@@ -33,11 +33,14 @@
|
|||||||
# 6. the secrets (novox/hq issue 342): no mergeable file fills a key with a vault secret that a setting could
|
# 6. the secrets (novox/hq issue 342): no mergeable file fills a key with a vault secret that a setting could
|
||||||
# replace — the key is `protected`, or left out of the file's `takes-settings`. checks/secrets/known names
|
# replace — the key is `protected`, or left out of the file's `takes-settings`. checks/secrets/known names
|
||||||
# the nested ones the controller cannot protect alone yet, and only shrinks.
|
# the nested ones the controller cannot protect alone yet, and only shrinks.
|
||||||
|
# 7. the mail-store seat (novox/hq ADR 0378 §3, issue 800): no manifest but modules/mesh-mail claims it.
|
||||||
set -eu
|
set -eu
|
||||||
|
|
||||||
(cd checks/words && go test -count=1 . && go run . ../..)
|
(cd checks/words && go test -count=1 . && go run . ../..)
|
||||||
# The agent's home (novox/hq ADR 0266): no archive, and no file written into, under it.
|
# The agent's home (novox/hq ADR 0266): no archive, and no file written into, under it.
|
||||||
(cd checks/agenthome && go test -count=1 . && go run . ../..)
|
(cd checks/agenthome && go test -count=1 . && go run . ../..)
|
||||||
|
# The mail-store seat (novox/hq ADR 0378 §3): only mesh-mail may claim it.
|
||||||
|
(cd checks/mailstore && go test -count=1 . && go run . ../..)
|
||||||
# The secrets (novox/hq issue 342): a password is not a setting, so no setting can replace the vault's. The
|
# The secrets (novox/hq issue 342): a password is not a setting, so no setting can replace the vault's. The
|
||||||
# known list only shrinks: judged against the trunk's copy at the merge base, a line it does not hold is refused.
|
# known list only shrinks: judged against the trunk's copy at the merge base, a line it does not hold is refused.
|
||||||
secrets_base=""
|
secrets_base=""
|
||||||
|
|||||||
@@ -7,9 +7,15 @@ The clipboard manager as a module (novox/hq ADR 0208, research 026/04).
|
|||||||
- Claims the mesh's `node-clipboard` seat and serves its verbs `history` and `copy`. Requires
|
- Claims the mesh's `node-clipboard` seat and serves its verbs `history` and `copy`. Requires
|
||||||
`x11-display` on its own machine.
|
`x11-display` on its own machine.
|
||||||
- **Declares `rofi-greenclip` absent** (ADR 0180). This module replaces it.
|
- **Declares `rofi-greenclip` absent** (ADR 0180). This module replaces it.
|
||||||
- Starts `clipmenud` **once per session**, from the session's start (the `xinitrc` slot `normal`).
|
- Declares `clipmenud.service` as a **user unit of the operator's account**, enabled and running
|
||||||
It is not a user unit as well. Its packaged unit needs user-scoped units (mesh-host #72, not
|
(`scope: user`, ADR 0177; issue 743), so `$mod+.` shows the history again. It is the only starter: the
|
||||||
merged), and the session start alone is one starter.
|
session's `xinitrc` line is gone. A drop-in (`~/.config/systemd/user/clipmenud.service.d/mesh.conf`)
|
||||||
|
replaces `ExecStart` with the module's launcher, which finds the X display from its socket (the account's
|
||||||
|
service manager has no `DISPLAY`) and puts the text-only `xsel` first on the `PATH`; until the display is up
|
||||||
|
it waits (a quick exit would run into the start limit, which the drop-in also switches off, with `RestartSec=2s`). `DISPLAY` and `XAUTHORITY` come from the user manager when the session's xinitrc has handed them over (module xorg); with several X servers and no `DISPLAY` yet, it waits for them. The store is in the account's runtime directory (mode 0700),
|
||||||
|
so no other account reads it.
|
||||||
|
**After delivery the operator logs in again**: the running session keeps the `clipmenud` its xinitrc
|
||||||
|
started, and the new unit waits on clipmenud's session lock until that one ends.
|
||||||
- Binds `$mod+period` to `clipmenu`, as its own i3 drop-in (`50-clipmenu.conf`). clipmenu shows the
|
- Binds `$mod+period` to `clipmenu`, as its own i3 drop-in (`50-clipmenu.conf`). clipmenu shows the
|
||||||
history through `dmenu`, the seat command of `node-launcher`, so it looks like every other menu.
|
history through `dmenu`, the seat command of `node-launcher`, so it looks like every other menu.
|
||||||
- Its settings are environment contributions (ADR 0203), read by the daemon, the menu and the tools
|
- Its settings are environment contributions (ADR 0203), read by the daemon, the menu and the tools
|
||||||
@@ -23,11 +29,11 @@ The clipboard manager as a module (novox/hq ADR 0208, research 026/04).
|
|||||||
|
|
||||||
| tool | does |
|
| tool | does |
|
||||||
|---|---|
|
|---|---|
|
||||||
| `node-clipboard.history` | the history, newest first, each entry once with its id, first line, time, size and text (cut) |
|
| `node-clipboard.history` | the history, newest first: count, time and length of each entry, and whether the daemon collects. **Never a text, a first line or an id** (issue 750); the text is read through clipmenu on the operator's own screen |
|
||||||
| `node-clipboard.copy` | put text on the clipboard; it enters the history like any copy |
|
| `node-clipboard.copy` | put text on the clipboard, then read it back; an error with the reason (no `DISPLAY`, `xsel` exited, other contents) when it does not hold the text. It enters the history like any copy |
|
||||||
| `clipmenu_paste` | what the clipboard holds now |
|
| `clipmenu_paste` | how many bytes of text the clipboard holds now, never the text |
|
||||||
| `clipmenu_clear` | forget the history; the daemon's locks stay |
|
| `clipmenu_clear` | forget the history; the daemon's locks stay |
|
||||||
| `clipmenu_delete` | forget one entry, by id or first line |
|
| `clipmenu_delete` | forget one entry, by its place in the history (1 is the newest) and the time the history gave it, refused if the history changed since; not by text or id |
|
||||||
|
|
||||||
The history is read from clipmenu's own store, in the account's runtime directory, under clipmenu's
|
The history is read from clipmenu's own store, in the account's runtime directory, under clipmenu's
|
||||||
own lock, so a copy arriving meanwhile is neither lost nor half-written. `copy` hands the text to an
|
own lock, so a copy arriving meanwhile is neither lost nor half-written. `copy` hands the text to an
|
||||||
|
|||||||
@@ -15,8 +15,7 @@ import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
mostCopy = 1 << 20
|
mostCopy = 1 << 20
|
||||||
mostPaste = 64 << 10
|
|
||||||
// majorVersion is clipmenu's store layout: <dir>/clipmenu.<major>.<user>/.
|
// majorVersion is clipmenu's store layout: <dir>/clipmenu.<major>.<user>/.
|
||||||
majorVersion = 6
|
majorVersion = 6
|
||||||
)
|
)
|
||||||
@@ -71,15 +70,15 @@ func cksum(data []byte) string {
|
|||||||
|
|
||||||
func entryID(line string) string { return cksum([]byte(line + "\n")) }
|
func entryID(line string) string { return cksum([]byte(line + "\n")) }
|
||||||
|
|
||||||
// Entry is one clip in the history.
|
// Entry is one clip in the history as a caller sees it: when and how long, never what. The text of a
|
||||||
|
// clip, its first line and its id (a checksum of that line) stay in clipmenu's store, readable on the
|
||||||
|
// operator's own screen through clipmenu (issue 750).
|
||||||
type Entry struct {
|
type Entry struct {
|
||||||
ID string `json:"id"`
|
At string `json:"at"`
|
||||||
Line string `json:"line"`
|
Bytes int `json:"bytes"`
|
||||||
At string `json:"at"`
|
id string
|
||||||
Bytes int `json:"bytes"`
|
line string
|
||||||
Text string `json:"text,omitempty"`
|
at int64
|
||||||
Truncated bool `json:"truncated,omitempty"`
|
|
||||||
at int64
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// HistoryResult is what node-clipboard.history answers.
|
// HistoryResult is what node-clipboard.history answers.
|
||||||
@@ -119,14 +118,15 @@ func readStore(dir string) ([]Entry, error) {
|
|||||||
}
|
}
|
||||||
out := make([]Entry, 0, len(latest))
|
out := make([]Entry, 0, len(latest))
|
||||||
for line, ns := range latest {
|
for line, ns := range latest {
|
||||||
out = append(out, Entry{ID: entryID(line), Line: line, at: ns, At: time.Unix(0, ns).Format(time.RFC3339)})
|
out = append(out, Entry{id: entryID(line), line: line, at: ns, At: time.Unix(0, ns).Format(time.RFC3339)})
|
||||||
}
|
}
|
||||||
sort.Slice(out, func(i, j int) bool { return out[i].at > out[j].at })
|
sort.Slice(out, func(i, j int) bool { return out[i].at > out[j].at })
|
||||||
return out, scan.Err()
|
return out, scan.Err()
|
||||||
}
|
}
|
||||||
|
|
||||||
// History is the clipboard's history, newest first.
|
// History is the clipboard's history, newest first: how many entries, when each was copied and how
|
||||||
func History(limit, maxBytes int) (HistoryResult, error) {
|
// long each is, and whether the daemon is collecting. It never returns a copied text.
|
||||||
|
func History(limit int) (HistoryResult, error) {
|
||||||
dir, err := storeDir(findEnvironment())
|
dir, err := storeDir(findEnvironment())
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return HistoryResult{}, err
|
return HistoryResult{}, err
|
||||||
@@ -140,15 +140,8 @@ func History(limit, maxBytes int) (HistoryResult, error) {
|
|||||||
if i == limit {
|
if i == limit {
|
||||||
break
|
break
|
||||||
}
|
}
|
||||||
if info, err := os.Stat(filepath.Join(dir, e.ID)); err == nil {
|
if info, err := os.Stat(filepath.Join(dir, e.id)); err == nil {
|
||||||
e.Bytes = int(info.Size())
|
e.Bytes = int(info.Size())
|
||||||
if maxBytes > 0 {
|
|
||||||
raw, _ := os.ReadFile(filepath.Join(dir, e.ID))
|
|
||||||
if len(raw) > maxBytes {
|
|
||||||
raw, e.Truncated = raw[:maxBytes], true
|
|
||||||
}
|
|
||||||
e.Text = string(raw)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
out.Entries = append(out.Entries, e)
|
out.Entries = append(out.Entries, e)
|
||||||
}
|
}
|
||||||
@@ -159,6 +152,40 @@ func History(limit, maxBytes int) (HistoryResult, error) {
|
|||||||
type CopyResult struct {
|
type CopyResult struct {
|
||||||
Bytes int `json:"bytes"`
|
Bytes int `json:"bytes"`
|
||||||
Unit string `json:"unit"`
|
Unit string `json:"unit"`
|
||||||
|
// Verified says the clipboard was read back and held exactly the text.
|
||||||
|
Verified bool `json:"verified"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// readBackWait is the pause between reads of the clipboard while the owner starts; read-back tries 15 times.
|
||||||
|
var readBackWait = 200 * time.Millisecond
|
||||||
|
|
||||||
|
// readBack waits for the clipboard to hold text and answers why when it does not.
|
||||||
|
func readBack(s Session, unit, text string) error {
|
||||||
|
var last string
|
||||||
|
for i := 0; i < 15; i++ {
|
||||||
|
time.Sleep(readBackWait)
|
||||||
|
r, err := s.run(3*time.Second, "", "xsel", "--output", "--clipboard")
|
||||||
|
switch {
|
||||||
|
case err != nil:
|
||||||
|
last = err.Error()
|
||||||
|
case r.Code != 0:
|
||||||
|
last = "xsel: " + strings.TrimSpace(r.Stderr)
|
||||||
|
case r.Stdout == text:
|
||||||
|
return nil
|
||||||
|
default:
|
||||||
|
last = fmt.Sprintf("it holds %d bytes of something else", len(r.Stdout))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
why := ""
|
||||||
|
if s.Display == "" {
|
||||||
|
why = "; the session has no DISPLAY, so the owner could not reach the screen"
|
||||||
|
}
|
||||||
|
if j, err := s.run(5*time.Second, "", "journalctl", "--user", "-u", unit+".service", "-n", "5", "--no-pager", "-o", "cat"); err == nil {
|
||||||
|
if tail := strings.TrimSpace(j.Stdout); tail != "" {
|
||||||
|
why += "; the owner said: " + tail
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return fmt.Errorf("the clipboard does not hold the text (%d bytes): %s%s", len(text), last, why)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Copy puts text on the clipboard. The clipboard is owned by a process until another copies, so the
|
// Copy puts text on the clipboard. The clipboard is owned by a process until another copies, so the
|
||||||
@@ -194,17 +221,20 @@ func Copy(text string) (CopyResult, error) {
|
|||||||
os.Remove(f.Name())
|
os.Remove(f.Name())
|
||||||
return CopyResult{}, err
|
return CopyResult{}, err
|
||||||
}
|
}
|
||||||
return CopyResult{Bytes: len(text), Unit: unit + ".service"}, nil
|
if err := readBack(s, unit, text); err != nil {
|
||||||
|
return CopyResult{}, err
|
||||||
|
}
|
||||||
|
return CopyResult{Bytes: len(text), Unit: unit + ".service", Verified: true}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// PasteResult is what clipmenu_paste answers.
|
// PasteResult is what clipmenu_paste answers: how much text the clipboard holds, never the text. The
|
||||||
|
// clipboard is the newest history entry, and a copied secret must not reach a caller (issue 750).
|
||||||
type PasteResult struct {
|
type PasteResult struct {
|
||||||
Text string `json:"text"`
|
Bytes int `json:"bytes"`
|
||||||
Bytes int `json:"bytes"`
|
Text bool `json:"holds_text"`
|
||||||
Truncated bool `json:"truncated,omitempty"`
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Paste reads the clipboard now.
|
// Paste reads the clipboard now and says how long it is.
|
||||||
func Paste() (PasteResult, error) {
|
func Paste() (PasteResult, error) {
|
||||||
s, err := findSession()
|
s, err := findSession()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -217,11 +247,7 @@ func Paste() (PasteResult, error) {
|
|||||||
if r.Code != 0 {
|
if r.Code != 0 {
|
||||||
return PasteResult{}, fmt.Errorf("xsel: %s", strings.TrimSpace(r.Stderr))
|
return PasteResult{}, fmt.Errorf("xsel: %s", strings.TrimSpace(r.Stderr))
|
||||||
}
|
}
|
||||||
out := PasteResult{Text: r.Stdout, Bytes: len(r.Stdout), Truncated: r.Truncated}
|
return PasteResult{Bytes: len(r.Stdout), Text: len(r.Stdout) > 0}, nil
|
||||||
if len(out.Text) > mostPaste {
|
|
||||||
out.Text, out.Truncated = out.Text[:mostPaste], true
|
|
||||||
}
|
|
||||||
return out, nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// ChangeResult is what clear and delete answer.
|
// ChangeResult is what clear and delete answer.
|
||||||
@@ -298,10 +324,12 @@ func Clear() (ChangeResult, error) {
|
|||||||
return out, err
|
return out, err
|
||||||
}
|
}
|
||||||
|
|
||||||
// Delete forgets one entry, by id or by its first line.
|
// Delete forgets one entry, by its place in the history as node-clipboard.history lists it (1 is the
|
||||||
func Delete(id, line string) (ChangeResult, error) {
|
// newest) together with the time it reports, so a copy arriving since the listing cannot shift the
|
||||||
if (id == "") == (line == "") {
|
// place onto another entry. Not by its text or id: either would let a caller confirm a guess of a copied line.
|
||||||
return ChangeResult{}, errors.New("give the entry's id or its line, one of the two")
|
func Delete(place int, at string) (ChangeResult, error) {
|
||||||
|
if place < 1 {
|
||||||
|
return ChangeResult{}, errors.New("give the entry's place in the history, 1 for the newest")
|
||||||
}
|
}
|
||||||
dir, err := storeOf()
|
dir, err := storeOf()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -309,6 +337,17 @@ func Delete(id, line string) (ChangeResult, error) {
|
|||||||
}
|
}
|
||||||
var out ChangeResult
|
var out ChangeResult
|
||||||
err = withStoreLock(dir, func() error {
|
err = withStoreLock(dir, func() error {
|
||||||
|
entries, err := readStore(dir)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if place > len(entries) {
|
||||||
|
return fmt.Errorf("no entry %d: the clipboard history holds %d", place, len(entries))
|
||||||
|
}
|
||||||
|
if entries[place-1].At != at {
|
||||||
|
return fmt.Errorf("entry %d was copied at %s, not at %s: the history changed since it was listed; list it again", place, entries[place-1].At, at)
|
||||||
|
}
|
||||||
|
line := entries[place-1].line
|
||||||
raw, err := os.ReadFile(filepath.Join(dir, "line_cache"))
|
raw, err := os.ReadFile(filepath.Join(dir, "line_cache"))
|
||||||
if err != nil && !errors.Is(err, os.ErrNotExist) {
|
if err != nil && !errors.Is(err, os.ErrNotExist) {
|
||||||
return err
|
return err
|
||||||
@@ -319,7 +358,7 @@ func Delete(id, line string) (ChangeResult, error) {
|
|||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
_, text, _ := strings.Cut(l, " ")
|
_, text, _ := strings.Cut(l, " ")
|
||||||
if text == line || (id != "" && entryID(text) == id) {
|
if text == line {
|
||||||
out.Removed++
|
out.Removed++
|
||||||
_ = os.Remove(filepath.Join(dir, entryID(text)))
|
_ = os.Remove(filepath.Join(dir, entryID(text)))
|
||||||
continue
|
continue
|
||||||
@@ -327,7 +366,7 @@ func Delete(id, line string) (ChangeResult, error) {
|
|||||||
kept = append(kept, l)
|
kept = append(kept, l)
|
||||||
}
|
}
|
||||||
if out.Removed == 0 {
|
if out.Removed == 0 {
|
||||||
return fmt.Errorf("no entry %s%s in the clipboard history", id, line)
|
return fmt.Errorf("no entry %d in the clipboard history", place)
|
||||||
}
|
}
|
||||||
content := strings.Join(kept, "\n")
|
content := strings.Join(kept, "\n")
|
||||||
if content != "" {
|
if content != "" {
|
||||||
|
|||||||
@@ -1,6 +1,7 @@
|
|||||||
package main
|
package main
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"encoding/json"
|
||||||
"errors"
|
"errors"
|
||||||
"os"
|
"os"
|
||||||
"os/exec"
|
"os/exec"
|
||||||
@@ -51,46 +52,84 @@ func store(t *testing.T, clips map[string]string, order ...string) string {
|
|||||||
return dir
|
return dir
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestTheHistoryIsNewestFirstOnceEachWithItsText(t *testing.T) {
|
func TestTheHistoryAnswersCountTimesAndLengthsNeverContent(t *testing.T) {
|
||||||
store(t, map[string]string{"first": "first", "second (2 lines)": "second\nline two"}, "first", "second (2 lines)", "first")
|
store(t, map[string]string{"first": "first", "sk-ant-api03-SECRETKEY (2 lines)": "sk-ant-api03-SECRETKEY\nline two"}, "first", "sk-ant-api03-SECRETKEY (2 lines)", "first")
|
||||||
fakeProcess(t, nobody, "clipmenud")
|
fakeProcess(t, nobody, "clipmenud")
|
||||||
h, err := History(10, 4096)
|
h, err := History(10)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
if !h.Collecting || h.Total != 2 || h.Entries[0].Line != "first" || h.Entries[1].Text != "second\nline two" || h.Entries[1].Bytes != 15 {
|
if !h.Collecting || h.Total != 2 || len(h.Entries) != 2 || h.Entries[0].Bytes != 5 || h.Entries[1].Bytes != 31 || h.Entries[0].At == "" {
|
||||||
t.Fatalf("%+v", h)
|
t.Fatalf("%+v", h)
|
||||||
}
|
}
|
||||||
if h, _ := History(1, 3); len(h.Entries) != 1 || h.Entries[0].Text != "fir" || !h.Entries[0].Truncated {
|
raw, _ := json.Marshal(h)
|
||||||
t.Fatalf("limited and cut: %+v", h)
|
for _, leak := range []string{"first", "SECRETKEY", "line two"} {
|
||||||
|
if strings.Contains(string(raw), leak) {
|
||||||
|
t.Fatalf("the history answers %q: %s", leak, raw)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
if h, _ := History(10, 0); h.Entries[0].Text != "" || h.Entries[0].Bytes != 5 {
|
if h, _ := History(1); len(h.Entries) != 1 {
|
||||||
t.Fatalf("without text: %+v", h)
|
t.Fatalf("limited: %+v", h)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestAnEntryIsDeletedByIdOrLineWithItsText(t *testing.T) {
|
// at is the time History reports for the entry at place.
|
||||||
|
func at(t *testing.T, place int) string {
|
||||||
|
t.Helper()
|
||||||
|
h, err := History(100)
|
||||||
|
if err != nil || place > len(h.Entries) {
|
||||||
|
t.Fatalf("%+v, %v", h, err)
|
||||||
|
}
|
||||||
|
return h.Entries[place-1].At
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestADeleteWhoseListingIsStaleIsRefused(t *testing.T) {
|
||||||
|
dir := store(t, map[string]string{"a": "a", "b": "b"}, "a", "b")
|
||||||
|
listed := at(t, 1) // b
|
||||||
|
f, _ := os.OpenFile(filepath.Join(dir, "line_cache"), os.O_APPEND|os.O_WRONLY, 0o600)
|
||||||
|
f.WriteString("1800000000000000000 a\n") // a copy arrives: place 1 is now a
|
||||||
|
f.Close()
|
||||||
|
if _, err := Delete(1, listed); err == nil || !strings.Contains(err.Error(), "list it again") {
|
||||||
|
t.Fatalf("the wrong entry was deleted: %v", err)
|
||||||
|
}
|
||||||
|
if h, _ := History(10); h.Total != 2 {
|
||||||
|
t.Fatalf("%+v", h)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAnEntryIsDeletedByItsPlaceWithItsTextAndNeverByItsText(t *testing.T) {
|
||||||
dir := store(t, map[string]string{"a": "a", "b": "b", "c": "c"}, "a", "b", "c", "a")
|
dir := store(t, map[string]string{"a": "a", "b": "b", "c": "c"}, "a", "b", "c", "a")
|
||||||
r, err := Delete(entryID("a"), "")
|
// newest first: a, c, b
|
||||||
|
r, err := Delete(1, at(t, 1))
|
||||||
if err != nil || r.Removed != 2 || r.Remaining != 2 {
|
if err != nil || r.Removed != 2 || r.Remaining != 2 {
|
||||||
t.Fatalf("by id, both copies: %+v, %v", r, err)
|
t.Fatalf("by place, both copies: %+v, %v", r, err)
|
||||||
}
|
}
|
||||||
if _, err := os.Stat(filepath.Join(dir, entryID("a"))); !errors.Is(err, os.ErrNotExist) {
|
if _, err := os.Stat(filepath.Join(dir, entryID("a"))); !errors.Is(err, os.ErrNotExist) {
|
||||||
t.Fatal("the text stayed")
|
t.Fatal("the text stayed")
|
||||||
}
|
}
|
||||||
if r, err := Delete("", "b"); err != nil || r.Removed != 1 || r.Remaining != 1 {
|
if r, err := Delete(2, at(t, 2)); err != nil || r.Removed != 1 || r.Remaining != 1 {
|
||||||
t.Fatalf("by line: %+v, %v", r, err)
|
t.Fatalf("second place (b): %+v, %v", r, err)
|
||||||
}
|
}
|
||||||
if _, err := Delete("", "zzz"); err == nil {
|
if _, err := Delete(5, "x"); err == nil {
|
||||||
t.Fatal("a missing entry was reported deleted")
|
t.Fatal("a missing place was reported deleted")
|
||||||
}
|
}
|
||||||
if _, err := Delete("x", "y"); err == nil {
|
if _, err := Delete(0, "x"); err == nil {
|
||||||
t.Fatal("both an id and a line were accepted")
|
t.Fatal("place 0 was accepted")
|
||||||
}
|
}
|
||||||
cache, _ := os.ReadFile(filepath.Join(dir, "line_cache"))
|
cache, _ := os.ReadFile(filepath.Join(dir, "line_cache"))
|
||||||
if !strings.HasSuffix(string(cache), " c\n") || strings.Count(string(cache), "\n") != 1 {
|
if !strings.HasSuffix(string(cache), " c\n") || strings.Count(string(cache), "\n") != 1 {
|
||||||
t.Fatalf("line cache: %q", cache)
|
t.Fatalf("line cache: %q", cache)
|
||||||
}
|
}
|
||||||
|
for _, tool := range tools() {
|
||||||
|
if tool.Name == "clipmenu_delete" {
|
||||||
|
if _, ok := tool.Input["line"]; ok {
|
||||||
|
t.Fatal("delete still takes a line")
|
||||||
|
}
|
||||||
|
if _, ok := tool.Input["id"]; ok {
|
||||||
|
t.Fatal("delete still takes an id")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestClearForgetsEverythingButKeepsTheDaemonsLocks(t *testing.T) {
|
func TestClearForgetsEverythingButKeepsTheDaemonsLocks(t *testing.T) {
|
||||||
@@ -112,13 +151,21 @@ func TestClearForgetsEverythingButKeepsTheDaemonsLocks(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestCopyHandsTheTextToAnOwnerUnderTheAccountsServiceManager(t *testing.T) {
|
func copyBed(t *testing.T, xsel string) string {
|
||||||
|
t.Helper()
|
||||||
store(t, nil)
|
store(t, nil)
|
||||||
fakeProcess(t, nobody, "i3", "DISPLAY=:1")
|
fakeProcess(t, nobody, "i3", "DISPLAY=:1")
|
||||||
bin := fakeBinaries(t, map[string]string{"systemctl": "true", "systemd-run": `echo "$*" > "$LOG"; for last; do :; done; cat "$last" > "$LOG.text"`})
|
bin := fakeBinaries(t, map[string]string{"systemctl": "true", "xsel": xsel,
|
||||||
|
"systemd-run": `echo "$*" > "$LOG"; for last; do :; done; cat "$last" > "$LOG.text"`})
|
||||||
t.Setenv("LOG", filepath.Join(bin, "log"))
|
t.Setenv("LOG", filepath.Join(bin, "log"))
|
||||||
|
readBackWait = 0
|
||||||
|
return bin
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCopyHandsTheTextToAnOwnerUnderTheAccountsServiceManagerAndReadsItBack(t *testing.T) {
|
||||||
|
bin := copyBed(t, `cat "$LOG.text"`)
|
||||||
r, err := Copy("secret-free text")
|
r, err := Copy("secret-free text")
|
||||||
if err != nil || r.Bytes != 16 || !strings.HasPrefix(r.Unit, "clipmenu-copy-") {
|
if err != nil || r.Bytes != 16 || !r.Verified || !strings.HasPrefix(r.Unit, "clipmenu-copy-") {
|
||||||
t.Fatalf("%+v, %v", r, err)
|
t.Fatalf("%+v, %v", r, err)
|
||||||
}
|
}
|
||||||
asked, _ := os.ReadFile(filepath.Join(bin, "log"))
|
asked, _ := os.ReadFile(filepath.Join(bin, "log"))
|
||||||
@@ -134,7 +181,23 @@ func TestCopyHandsTheTextToAnOwnerUnderTheAccountsServiceManager(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestPasteReadsTheClipboardOrSaysThereIsNoSession(t *testing.T) {
|
func TestCopyFailsWhenTheClipboardHoldsSomethingElse(t *testing.T) {
|
||||||
|
copyBed(t, `printf 'old contents'`)
|
||||||
|
_, err := Copy("new text")
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "does not hold") || strings.Contains(err.Error(), "old contents") {
|
||||||
|
t.Fatalf("a copy that did not land was reported: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCopyFailsSayingWhyWhenTheOwnerCannotReachTheDisplay(t *testing.T) {
|
||||||
|
copyBed(t, `echo "xsel: Can't open display: (null)" >&2; exit 1`)
|
||||||
|
_, err := Copy("new text")
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "Can't open display") {
|
||||||
|
t.Fatalf("the reason was not given: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPasteSaysHowMuchTheClipboardHoldsNeverWhat(t *testing.T) {
|
||||||
fakeMachine(t)
|
fakeMachine(t)
|
||||||
if _, err := Paste(); !errors.Is(err, ErrNoSession) {
|
if _, err := Paste(); !errors.Is(err, ErrNoSession) {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
@@ -142,8 +205,9 @@ func TestPasteReadsTheClipboardOrSaysThereIsNoSession(t *testing.T) {
|
|||||||
fakeProcess(t, nobody, "i3", "DISPLAY=:1")
|
fakeProcess(t, nobody, "i3", "DISPLAY=:1")
|
||||||
fakeBinaries(t, map[string]string{"xsel": `[ "$*" = "--output --clipboard" ] && printf 'on the clipboard'`})
|
fakeBinaries(t, map[string]string{"xsel": `[ "$*" = "--output --clipboard" ] && printf 'on the clipboard'`})
|
||||||
p, err := Paste()
|
p, err := Paste()
|
||||||
if err != nil || p.Text != "on the clipboard" || p.Bytes != 16 {
|
raw, _ := json.Marshal(p)
|
||||||
t.Fatalf("%+v, %v", p, err)
|
if err != nil || p.Bytes != 16 || strings.Contains(string(raw), "on the clipboard") {
|
||||||
|
t.Fatalf("%s, %v", raw, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -157,7 +221,26 @@ func TestWithoutAStoreTheChangesSayWhy(t *testing.T) {
|
|||||||
if _, err := Clear(); err == nil || !strings.Contains(err.Error(), "clipmenud has not run") {
|
if _, err := Clear(); err == nil || !strings.Contains(err.Error(), "clipmenud has not run") {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
if h, err := History(5, 0); err != nil || h.Total != 0 || h.Collecting {
|
if h, err := History(5); err != nil || h.Total != 0 || h.Collecting {
|
||||||
t.Fatalf("an empty history: %+v, %v", h, err)
|
t.Fatalf("an empty history: %+v, %v", h, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// clipmenud is a bash script: its process is named "bash" and only its command line names clipmenud
|
||||||
|
// (issue 860: collecting was false while the daemon ran).
|
||||||
|
func TestCollectingSeesClipmenudRunAsABashScript(t *testing.T) {
|
||||||
|
store(t, map[string]string{"first": "first"}, "first")
|
||||||
|
fakeProcess(t, nobody, "bash")
|
||||||
|
cmdline := filepath.Join(procRoot, strconv.Itoa(nobody), "cmdline")
|
||||||
|
if err := os.WriteFile(cmdline, []byte("bash\x00/usr/bin/clipmenud\x00"), 0o644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if h, err := History(10); err != nil || !h.Collecting {
|
||||||
|
t.Fatalf("clipmenud runs as a bash script: %+v %v", h, err)
|
||||||
|
}
|
||||||
|
// another bash script is not the daemon
|
||||||
|
os.WriteFile(cmdline, []byte("bash\x00/usr/bin/other\x00"), 0o644)
|
||||||
|
if h, _ := History(10); h.Collecting {
|
||||||
|
t.Fatalf("a bash script that is not clipmenud counted: %+v", h)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -22,28 +22,23 @@ func tools() []stdio.Tool {
|
|||||||
return []stdio.Tool{
|
return []stdio.Tool{
|
||||||
{
|
{
|
||||||
Name: "node-clipboard.history",
|
Name: "node-clipboard.history",
|
||||||
Description: "What the operator copied, newest first: each entry's id, its first line, when, its size " +
|
Description: "What the operator copied, newest first: how many entries, when each was copied and how long " +
|
||||||
"and its text (each cut at max_bytes). Whether the clipboard daemon is collecting.",
|
"it is, never what it says. Whether the clipboard daemon is collecting. The text is read on the operator's own screen, through clipmenu.",
|
||||||
Input: map[string]any{
|
Input: map[string]any{
|
||||||
"limit": map[string]any{"type": "integer", "description": "at most this many entries (default 20, at most 500)"},
|
"limit": map[string]any{"type": "integer", "description": "at most this many entries (default 20, at most 500)"},
|
||||||
"max_bytes": map[string]any{"type": "integer", "description": "cut each entry's text at this many bytes; 0 leaves the text out (default 4096, at most 65536)"},
|
|
||||||
},
|
},
|
||||||
Run: func(args map[string]any) (any, error) {
|
Run: func(args map[string]any) (any, error) {
|
||||||
limit, err := whole(args, "limit", 20, 1, 500)
|
limit, err := whole(args, "limit", 20, 1, 500)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
most, err := whole(args, "max_bytes", 4096, 0, 65536)
|
return History(limit)
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
return History(limit, most)
|
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
Name: "node-clipboard.copy",
|
Name: "node-clipboard.copy",
|
||||||
Description: "Put text on the operator's clipboard, as if they had copied it; it enters the history " +
|
Description: "Put text on the operator's clipboard, as if they had copied it; it enters the history " +
|
||||||
"like any copy. Answers how many bytes.",
|
"like any copy. Reads the clipboard back and answers an error with the reason when it does not hold the text.",
|
||||||
Input: map[string]any{
|
Input: map[string]any{
|
||||||
"type": "object",
|
"type": "object",
|
||||||
"properties": map[string]any{
|
"properties": map[string]any{
|
||||||
@@ -61,7 +56,7 @@ func tools() []stdio.Tool {
|
|||||||
},
|
},
|
||||||
{
|
{
|
||||||
Name: "clipmenu_paste",
|
Name: "clipmenu_paste",
|
||||||
Description: "What the operator's clipboard holds right now, as text (cut at 64 KiB, said in truncated).",
|
Description: "How many bytes of text the operator's clipboard holds right now, never the text itself.",
|
||||||
Run: func(map[string]any) (any, error) { return Paste() },
|
Run: func(map[string]any) (any, error) { return Paste() },
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
@@ -71,19 +66,22 @@ func tools() []stdio.Tool {
|
|||||||
},
|
},
|
||||||
{
|
{
|
||||||
Name: "clipmenu_delete",
|
Name: "clipmenu_delete",
|
||||||
Description: "Forget one entry of the clipboard history, by its id as the history answers it, or by " +
|
Description: "Forget one entry of the clipboard history, by its place in node-clipboard.history's list " +
|
||||||
"its first line exactly.",
|
"(1 is the newest) and its time as the history gives it; refused when the two no longer match.",
|
||||||
Input: map[string]any{
|
Input: map[string]any{
|
||||||
"id": map[string]any{"type": "string", "description": "the entry's id"},
|
"place": map[string]any{"type": "integer", "description": "the entry's place in the history, 1 for the newest"},
|
||||||
"line": map[string]any{"type": "string", "description": "the entry's first line, exactly"},
|
"at": map[string]any{"type": "string", "description": "the entry's time, as the history reports it"},
|
||||||
},
|
},
|
||||||
Run: func(args map[string]any) (any, error) {
|
Run: func(args map[string]any) (any, error) {
|
||||||
id, err := text(args, "id", false)
|
place, err := whole(args, "place", 0, 0, 100000)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
line, _ := args["line"].(string)
|
at, err := text(args, "at", true)
|
||||||
return Delete(id, line)
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return Delete(place, at)
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,9 +1,16 @@
|
|||||||
package main
|
package main
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"context"
|
||||||
|
"net"
|
||||||
|
"os"
|
||||||
|
"os/exec"
|
||||||
"reflect"
|
"reflect"
|
||||||
"strings"
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"syscall"
|
||||||
"testing"
|
"testing"
|
||||||
|
"time"
|
||||||
)
|
)
|
||||||
|
|
||||||
// clipmenu's shape (novox/hq ADR 0208, research 026/04): it claims node-clipboard serving history
|
// clipmenu's shape (novox/hq ADR 0208, research 026/04): it claims node-clipboard serving history
|
||||||
@@ -27,20 +34,48 @@ func TestItClaimsTheClipboardSeatServingHistoryAndCopy(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestTheDaemonStartsOnceFromTheSessionsStart(t *testing.T) {
|
func TestTheDaemonIsTheOperatorsUserUnitEnabledAndRunning(t *testing.T) {
|
||||||
m := readManifest(t)
|
m := readManifest(t)
|
||||||
if len(m.Shell) != 1 || m.Shell[0].For != "xinitrc" || m.Shell[0].Slot != "normal" {
|
if len(m.Shell) != 0 {
|
||||||
t.Fatalf("%+v", m.Shell)
|
t.Fatalf("a second start from the session: %+v", m.Shell)
|
||||||
}
|
}
|
||||||
code := m.Shell[0].Code
|
svc := m.resource(t, "daemon")
|
||||||
// Started once, through the module's text-only xsel (a selection holding an image is never read).
|
want := map[string]any{"id": "daemon", "type": "service", "unit": "clipmenud.service", "scope": "user",
|
||||||
if strings.Count(code, "clipmenud &\n") != 1 || !strings.Contains(code, `PATH="/usr/local/lib/mesh-clipmenu:$PATH" clipmenud &`) ||
|
"user": "${machine:account}", "state": "running", "boot": "enabled", "reload-on": nil}
|
||||||
strings.Contains(code, "greenclip") {
|
for k, v := range want {
|
||||||
t.Fatalf("%q", code)
|
if k != "reload-on" && svc[k] != v {
|
||||||
|
t.Fatalf("daemon %s = %v, want %v: %v", k, svc[k], v, svc)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
for _, r := range m.Resources {
|
// ADR 0240 rule 8: a long-running resource says how it is ready. The launcher keeps the unit active while
|
||||||
if r["type"] == "service" || r["type"] == "process" {
|
// it waits for the display and execs clipmenud, so the unit's state is the daemon's own liveness.
|
||||||
t.Fatalf("a second start: %v", r)
|
if h, _ := svc["health"].(map[string]any); h["kind"] != "unit" {
|
||||||
|
t.Fatalf("daemon health = %v, want kind unit", svc["health"])
|
||||||
|
}
|
||||||
|
dropin := m.resource(t, "daemon-dropin")
|
||||||
|
content, _ := dropin["content"].(string)
|
||||||
|
if dropin["path"] != "${machine:account-home}/.config/systemd/user/clipmenud.service.d/mesh.conf" ||
|
||||||
|
dropin["owner"] != "${machine:account}" ||
|
||||||
|
!strings.Contains(content, "ExecStart=/usr/local/lib/mesh-clipmenu/clipmenud-session") ||
|
||||||
|
strings.Contains(content, "greenclip") {
|
||||||
|
t.Fatalf("drop-in: %v", dropin)
|
||||||
|
}
|
||||||
|
launcher, _ := m.resource(t, "launcher")["content"].(string)
|
||||||
|
if !strings.Contains(launcher, "DISPLAY") || !strings.Contains(launcher, "/tmp/.X11-unix") ||
|
||||||
|
!strings.Contains(launcher, `PATH="/usr/local/lib/mesh-clipmenu:$PATH"`) || !strings.Contains(launcher, "exec clipmenud") {
|
||||||
|
t.Fatalf("launcher: %q", launcher)
|
||||||
|
}
|
||||||
|
// Waits for the display instead of exiting into the start limit; DISPLAY from the manager wins.
|
||||||
|
if !strings.Contains(launcher, "until [ -n \"$DISPLAY\" ]") || !strings.Contains(launcher, "sleep 1") ||
|
||||||
|
strings.Contains(launcher, "exit 1") || strings.Contains(launcher, "exit 0") {
|
||||||
|
t.Fatalf("the launcher must wait, not exit: %q", launcher)
|
||||||
|
}
|
||||||
|
if !strings.Contains(content, "StartLimitIntervalSec=0") || !strings.Contains(content, "RestartSec=2s") {
|
||||||
|
t.Fatalf("drop-in lacks the safety net: %q", content)
|
||||||
|
}
|
||||||
|
for k, v := range m.Environment.Variables {
|
||||||
|
if !strings.Contains(content, "Environment="+k+"="+v) {
|
||||||
|
t.Fatalf("the unit lacks %s=%s", k, v)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -64,3 +99,126 @@ func TestTheToolsAgreeWithTheManifest(t *testing.T) {
|
|||||||
checkTheToolsAgree(t, m)
|
checkTheToolsAgree(t, m)
|
||||||
checkNoSecretsOrInstallationNames(t)
|
checkNoSecretsOrInstallationNames(t)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Run for real: DISPLAY from the manager is used at once; several sockets and no DISPLAY wait.
|
||||||
|
func TestTheLauncherUsesTheManagersDisplayAndWaitsOtherwise(t *testing.T) {
|
||||||
|
m := readManifest(t)
|
||||||
|
script, _ := m.resource(t, "launcher")["content"].(string)
|
||||||
|
dir := t.TempDir()
|
||||||
|
file := dir + "/launcher"
|
||||||
|
// clipmenud and the socket directory are faked: the script's own paths are rewritten.
|
||||||
|
body := strings.ReplaceAll(script, "/tmp/.X11-unix", dir+"/x11")
|
||||||
|
body = strings.Replace(body, "exec clipmenud", `echo "DISPLAY=$DISPLAY"`, 1)
|
||||||
|
if err := os.WriteFile(file, []byte(body), 0o755); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
os.MkdirAll(dir+"/x11", 0o755)
|
||||||
|
fakeBin := dir + "/bin"
|
||||||
|
os.MkdirAll(fakeBin, 0o755)
|
||||||
|
// the manager's environment: empty until the file appears
|
||||||
|
os.WriteFile(fakeBin+"/systemctl", []byte("#!/bin/sh\n[ -f \""+dir+"/env\" ] && cat \""+dir+"/env\"\nexit 0\n"), 0o755)
|
||||||
|
run := func(env ...string) (string, bool) {
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), 2500*time.Millisecond)
|
||||||
|
defer cancel()
|
||||||
|
cmd := exec.CommandContext(ctx, "sh", file)
|
||||||
|
cmd.Env = append([]string{"PATH=" + fakeBin + ":" + os.Getenv("PATH"), "HOME=" + dir}, env...)
|
||||||
|
out, _ := cmd.Output()
|
||||||
|
return strings.TrimSpace(string(out)), ctx.Err() == nil
|
||||||
|
}
|
||||||
|
if out, ok := run("DISPLAY=:7"); !ok || out != "DISPLAY=:7" {
|
||||||
|
t.Fatalf("with DISPLAY: %q %v", out, ok)
|
||||||
|
}
|
||||||
|
if _, ok := run(); ok {
|
||||||
|
t.Fatal("it exited with no X server")
|
||||||
|
}
|
||||||
|
for _, n := range []string{"X0", "X1"} {
|
||||||
|
l, _ := net.Listen("unix", dir+"/x11/"+n)
|
||||||
|
defer l.Close()
|
||||||
|
}
|
||||||
|
if _, ok := run(); ok {
|
||||||
|
t.Fatal("it chose one of two servers")
|
||||||
|
}
|
||||||
|
// the session hands the manager its display while the launcher waits
|
||||||
|
go func() {
|
||||||
|
time.Sleep(1200 * time.Millisecond)
|
||||||
|
os.WriteFile(dir+"/env", []byte("DISPLAY=:1\nOTHER=x\n"), 0o644)
|
||||||
|
}()
|
||||||
|
if out, ok := run(); !ok || out != "DISPLAY=:1" {
|
||||||
|
t.Fatalf("the display that arrived while waiting: %q %v", out, ok)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Run for real: while another clipmenud of the account holds the session lock the launcher waits without
|
||||||
|
// a word, and starts clipmenud once the lock is free (issue 862).
|
||||||
|
func TestTheLauncherWaitsQuietlyWhileAnotherClipmenudHoldsTheLock(t *testing.T) {
|
||||||
|
if _, err := exec.LookPath("flock"); err != nil {
|
||||||
|
t.Skip("flock is missing here")
|
||||||
|
}
|
||||||
|
m := readManifest(t)
|
||||||
|
script, _ := m.resource(t, "launcher")["content"].(string)
|
||||||
|
dir := t.TempDir()
|
||||||
|
body := strings.Replace(script, "exec clipmenud", `echo "started"`, 1)
|
||||||
|
file := dir + "/launcher"
|
||||||
|
if err := os.WriteFile(file, []byte(body), 0o755); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
store := dir + "/cm/clipmenu.6.op"
|
||||||
|
os.MkdirAll(store, 0o700)
|
||||||
|
lock := store + "/session_lock"
|
||||||
|
// the older clipmenud: it holds the lock the way clipmenud does, until it is killed
|
||||||
|
holder := exec.Command("flock", "-x", "-n", lock, "sleep", "60")
|
||||||
|
holder.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
|
||||||
|
if err := holder.Start(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
// flock's child shares the lock, so the whole group goes
|
||||||
|
release := func() { syscall.Kill(-holder.Process.Pid, syscall.SIGKILL); holder.Wait() }
|
||||||
|
defer release()
|
||||||
|
for i := 0; i < 50; i++ {
|
||||||
|
if probe := exec.Command("flock", "-n", lock, "true"); probe.Run() != nil {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
time.Sleep(50 * time.Millisecond)
|
||||||
|
}
|
||||||
|
var out syncBuffer
|
||||||
|
cmd := exec.Command("sh", file)
|
||||||
|
cmd.Env = []string{"PATH=" + os.Getenv("PATH"), "HOME=" + dir, "DISPLAY=:7", "USER=op", "CM_DIR=" + dir + "/cm"}
|
||||||
|
cmd.Stdout, cmd.Stderr = &out, &out
|
||||||
|
if err := cmd.Start(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
done := make(chan error, 1)
|
||||||
|
go func() { done <- cmd.Wait() }()
|
||||||
|
select {
|
||||||
|
case <-done:
|
||||||
|
t.Fatalf("the launcher went on while the lock was held: %q", out.String())
|
||||||
|
case <-time.After(2500 * time.Millisecond):
|
||||||
|
}
|
||||||
|
if out.Len() != 0 {
|
||||||
|
t.Fatalf("the launcher spoke while it waited: %q", out.String())
|
||||||
|
}
|
||||||
|
release()
|
||||||
|
select {
|
||||||
|
case <-done:
|
||||||
|
case <-time.After(5 * time.Second):
|
||||||
|
cmd.Process.Kill()
|
||||||
|
t.Fatal("the launcher did not start clipmenud after the lock was freed")
|
||||||
|
}
|
||||||
|
if got := strings.TrimSpace(out.String()); got != "started" {
|
||||||
|
t.Fatalf("after the lock was freed: %q", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
type syncBuffer struct {
|
||||||
|
mu sync.Mutex
|
||||||
|
b strings.Builder
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *syncBuffer) Write(p []byte) (int, error) {
|
||||||
|
s.mu.Lock()
|
||||||
|
defer s.mu.Unlock()
|
||||||
|
return s.b.Write(p)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *syncBuffer) Len() int { s.mu.Lock(); defer s.mu.Unlock(); return s.b.Len() }
|
||||||
|
func (s *syncBuffer) String() string { s.mu.Lock(); defer s.mu.Unlock(); return s.b.String() }
|
||||||
|
|||||||
@@ -403,7 +403,7 @@ func processesOf(comm string) []int {
|
|||||||
if owner, ok := ownerOf(dir); !ok || owner != uid {
|
if owner, ok := ownerOf(dir); !ok || owner != uid {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
if readTrimmed(filepath.Join(dir, "comm")) == comm {
|
if readTrimmed(filepath.Join(dir, "comm")) == comm || runsScript(dir, comm) {
|
||||||
pids = append(pids, pid)
|
pids = append(pids, pid)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -411,6 +411,24 @@ func processesOf(comm string) []int {
|
|||||||
return pids
|
return pids
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// runsScript is whether the process is an interpreter running the script named name: clipmenud is a
|
||||||
|
// bash script, so its comm is "bash" and only its command line, "bash /usr/bin/clipmenud", names it.
|
||||||
|
func runsScript(procDir, name string) bool {
|
||||||
|
raw, err := os.ReadFile(filepath.Join(procDir, "cmdline"))
|
||||||
|
if err != nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
args := strings.Split(strings.TrimRight(string(raw), "\x00"), "\x00")
|
||||||
|
if len(args) < 2 {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
switch filepath.Base(args[0]) {
|
||||||
|
case "bash", "sh", "dash", "zsh", "env":
|
||||||
|
return filepath.Base(args[1]) == name
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
// signalAll sends sig to every process of this uid named comm, and answers the pids it reached.
|
// signalAll sends sig to every process of this uid named comm, and answers the pids it reached.
|
||||||
func signalAll(comm string, sig syscall.Signal) []int {
|
func signalAll(comm string, sig syscall.Signal) []int {
|
||||||
var reached []int
|
var reached []int
|
||||||
|
|||||||
@@ -29,13 +29,6 @@
|
|||||||
"CM_HISTLENGTH": "15"
|
"CM_HISTLENGTH": "15"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"shell": [
|
|
||||||
{
|
|
||||||
"for": "xinitrc",
|
|
||||||
"slot": "normal",
|
|
||||||
"code": "# The clipboard's history (module clipmenu, novox/hq ADR 0208): clipmenud collects every copy from\n# here on, once per session. It keeps the history in the account's runtime directory, so a reboot\n# forgets it, and with it every password that was ever copied.\n# Through the module's xsel, which reads only a selection offering text (see the README).\nPATH=\"/usr/local/lib/mesh-clipmenu:$PATH\" clipmenud &\n"
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"resources": [
|
"resources": [
|
||||||
{
|
{
|
||||||
"id": "package",
|
"id": "package",
|
||||||
@@ -54,6 +47,47 @@
|
|||||||
"path": "/usr/local/lib/mesh-clipmenu/xsel",
|
"path": "/usr/local/lib/mesh-clipmenu/xsel",
|
||||||
"mode": "0755",
|
"mode": "0755",
|
||||||
"content": "#!/bin/sh\n# xsel as clipmenud sees it, written by the mesh (module clipmenu). Replaced at every push.\n#\n# clipmenud records text, and reads a selection with `timeout 1 xsel -o`. A selection holding an\n# image (a screenshot copied as image/png) is sent in pieces; one second is too short for megabytes,\n# timeout kills xsel half way, and the program owning the image waits for ever for a reader that is\n# gone. From then on nothing can ask the clipboard anything: pastes hang, and an Electron app that\n# asks on its main thread freezes. So a read goes ahead only when the selection offers text.\ncase \" $* \" in\n*\" -o \"* | *\" --output \"*)\n\tselection=clipboard\n\tcase \" $* \" in\n\t*\" --primary \"* | *\" -p \"*) selection=primary ;;\n\t*\" --secondary \"* | *\" -s \"*) selection=secondary ;;\n\tesac\n\ttargets=$(timeout 1 xclip -selection \"$selection\" -t TARGETS -o 2>/dev/null) || exit 1\n\tprintf '%s\\n' \"$targets\" | grep -qxE 'UTF8_STRING|STRING|TEXT|text/plain(;charset=utf-8)?' || exit 1\n\t;;\nesac\nexec /usr/bin/xsel \"$@\"\n"
|
"content": "#!/bin/sh\n# xsel as clipmenud sees it, written by the mesh (module clipmenu). Replaced at every push.\n#\n# clipmenud records text, and reads a selection with `timeout 1 xsel -o`. A selection holding an\n# image (a screenshot copied as image/png) is sent in pieces; one second is too short for megabytes,\n# timeout kills xsel half way, and the program owning the image waits for ever for a reader that is\n# gone. From then on nothing can ask the clipboard anything: pastes hang, and an Electron app that\n# asks on its main thread freezes. So a read goes ahead only when the selection offers text.\ncase \" $* \" in\n*\" -o \"* | *\" --output \"*)\n\tselection=clipboard\n\tcase \" $* \" in\n\t*\" --primary \"* | *\" -p \"*) selection=primary ;;\n\t*\" --secondary \"* | *\" -s \"*) selection=secondary ;;\n\tesac\n\ttargets=$(timeout 1 xclip -selection \"$selection\" -t TARGETS -o 2>/dev/null) || exit 1\n\tprintf '%s\\n' \"$targets\" | grep -qxE 'UTF8_STRING|STRING|TEXT|text/plain(;charset=utf-8)?' || exit 1\n\t;;\nesac\nexec /usr/bin/xsel \"$@\"\n"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "user-units-dir",
|
||||||
|
"type": "directory",
|
||||||
|
"path": "${machine:account-home}/.config/systemd/user",
|
||||||
|
"owner": "${machine:account}",
|
||||||
|
"mode": "0755"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "daemon-dropins",
|
||||||
|
"type": "directory",
|
||||||
|
"path": "${machine:account-home}/.config/systemd/user/clipmenud.service.d",
|
||||||
|
"owner": "${machine:account}",
|
||||||
|
"mode": "0755"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "daemon-dropin",
|
||||||
|
"type": "file",
|
||||||
|
"path": "${machine:account-home}/.config/systemd/user/clipmenud.service.d/mesh.conf",
|
||||||
|
"owner": "${machine:account}",
|
||||||
|
"mode": "0644",
|
||||||
|
"content": "# clipmenud in the operator's account (module clipmenu, novox/hq ADR 0208). Owned by the mesh: replaced at every push.\n# The package's unit starts clipmenud with no DISPLAY; the module's launcher finds the display and puts the\n# text-only xsel first on the PATH. The settings repeat the module's environment, which the account's\n# service manager does not read.\n[Unit]\nStartLimitIntervalSec=0\n\n[Service]\nRestartSec=2s\nEnvironment=CM_SELECTIONS=clipboard\nEnvironment=CM_MAX_CLIPS=500\nEnvironment=CM_HISTLENGTH=15\nExecStart=\nExecStart=/usr/local/lib/mesh-clipmenu/clipmenud-session\n"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "launcher",
|
||||||
|
"type": "file",
|
||||||
|
"path": "/usr/local/lib/mesh-clipmenu/clipmenud-session",
|
||||||
|
"mode": "0755",
|
||||||
|
"content": "#!/bin/sh\n# clipmenud as the operator's user unit starts it, written by the mesh (module clipmenu). Replaced at every push.\n#\n# The account's service manager starts before the X server does, and has a DISPLAY only once the session's\n# xinitrc has handed it over (module xorg). So this waits, quietly, until the display is known, and exits\n# only when the session ends with it: a quick exit would run the unit into its start limit. DISPLAY, when\n# the manager has it, wins. Without it, exactly one X socket names the display; none or several (a\n# second server beside the session) wait for the session's own to arrive.\n# A running process does not see what the session hands the manager later, so each pass asks the manager.\nuntil [ -n \"$DISPLAY\" ]; do\n\tfor pair in $(systemctl --user show-environment 2>/dev/null | grep -E '^(DISPLAY|XAUTHORITY)='); do\n\t\texport \"$pair\"\n\tdone\n\t[ -n \"$DISPLAY\" ] && break\n\tset -- /tmp/.X11-unix/X*\n\tif [ $# -eq 1 ] && [ -S \"$1\" ]; then\n\t\tDISPLAY=\":${1##*/X}\"\n\t\texport DISPLAY\n\telse\n\t\tsleep 1\n\tfi\ndone\nif [ -z \"$XAUTHORITY\" ] && [ -r \"$HOME/.Xauthority\" ]; then\n\tXAUTHORITY=\"$HOME/.Xauthority\"\n\texport XAUTHORITY\nfi\n# clipmenud allows one daemon per account: it takes an exclusive flock on its session_lock file and exits\n# with status 2 when that is held. After a delivery without a new login, the session's older clipmenud (from\n# its xinitrc) still holds it, and a start would run the unit round its restart loop. So this waits, quietly,\n# until the lock is free: it looks at the same file clipmenud locks (the directory is clipmenud's own rule)\n# and lets go at once. Exit status 1 is \"held\" for util-linux and BusyBox flock alike (no -E: BusyBox lacks it); any other outcome starts clipmenud, which says its own\n# error. Another daemon winning the gap between look and start costs one restart, not a loop.\nlock=\"${CM_DIR:-${XDG_RUNTIME_DIR-${TMPDIR-/tmp}}}/clipmenu.6.$USER/session_lock\"\nwhile [ -e \"$lock\" ]; do\n\tflock -n \"$lock\" true\n\t[ $? -eq 1 ] || break\n\tsleep 2\ndone\nPATH=\"/usr/local/lib/mesh-clipmenu:$PATH\"\nexport PATH\nexec clipmenud\n"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "daemon",
|
||||||
|
"type": "service",
|
||||||
|
"unit": "clipmenud.service",
|
||||||
|
"scope": "user",
|
||||||
|
"user": "${machine:account}",
|
||||||
|
"state": "running",
|
||||||
|
"boot": "enabled",
|
||||||
|
"health": {
|
||||||
|
"kind": "unit"
|
||||||
|
}
|
||||||
}
|
}
|
||||||
],
|
],
|
||||||
"build": {
|
"build": {
|
||||||
|
|||||||
@@ -3,7 +3,9 @@
|
|||||||
How loaded a machine's storage is, read from `/proc` and `/sys` and nothing else: the share of CPU time
|
How loaded a machine's storage is, read from `/proc` and `/sys` and nothing else: the share of CPU time
|
||||||
spent waiting on I/O, I/O pressure stall, each disk's busy share, throughput, IOPS, wait and queue
|
spent waiting on I/O, I/O pressure stall, each disk's busy share, throughput, IOPS, wait and queue
|
||||||
depth, the filesystems' space, and the units, containers and processes doing the I/O (novox/hq issue
|
depth, the filesystems' space, and the units, containers and processes doing the I/O (novox/hq issue
|
||||||
312). It changes nothing and declares no resources, so it can be assigned to every machine.
|
312). It also reads a machine's CPU use by systemd unit and by container, from each cgroup's cpu.stat,
|
||||||
|
so an overload outside every container can be named (novox/hq issue 741). It changes nothing and
|
||||||
|
declares no resources, so it can be assigned to every machine.
|
||||||
|
|
||||||
## Why it exists, and why here
|
## Why it exists, and why here
|
||||||
|
|
||||||
@@ -22,7 +24,7 @@ machine's resources a seat, these tools become its verbs.
|
|||||||
|
|
||||||
## Tools
|
## Tools
|
||||||
|
|
||||||
All four only read.
|
All five only read.
|
||||||
|
|
||||||
| tool | what | replaces |
|
| tool | what | replaces |
|
||||||
|---|---|---|
|
|---|---|---|
|
||||||
@@ -30,6 +32,7 @@ All four only read.
|
|||||||
| `disk_top` | Over a window. `by=unit` (the default) ranks systemd units and containers by bytes moved, from their cgroups. `by=container` ranks containers by name, and `by=process` ranks processes | `iotop` |
|
| `disk_top` | Over a window. `by=unit` (the default) ranks systemd units and containers by bytes moved, from their cgroups. `by=container` ranks containers by name, and `by=process` ranks processes | `iotop` |
|
||||||
| `disk_filesystems` | Each filesystem that stores data, said once however many places it is mounted: size, used and free GiB, used % as `df` counts it, inodes used % | `df -h`, `df -i` |
|
| `disk_filesystems` | Each filesystem that stores data, said once however many places it is mounted: size, used and free GiB, used % as `df` counts it, inodes used % | `df -h`, `df -i` |
|
||||||
| `disk_devices` | Each disk without sampling: model, size, whether it spins, scheduler, queue limit, what it is built on, and since boot the GiB read and written, the busy share and the wait per read and per write | `lsblk`, `cat /proc/diskstats` |
|
| `disk_devices` | Each disk without sampling: model, size, whether it spins, scheduler, queue limit, what it is built on, and since boot the GiB read and written, the busy share and the wait per read and per write | `lsblk`, `cat /proc/diskstats` |
|
||||||
|
| `cpu_top` | Samples a window (`seconds`, default 3, at most 10). It answers the machine's CPU in use (% of all CPUs and as CPUs busy, from `/proc/stat`), then `by=unit` (the default) ranks systemd units and containers by CPU, from their cgroups' cpu.stat, busiest first (`limit`, default 15), with the busy time no unit accounts for; `by=container` ranks containers by name | `top`, `systemd-cgtop` |
|
||||||
|
|
||||||
### How each number is read
|
### How each number is read
|
||||||
|
|
||||||
|
|||||||
@@ -68,9 +68,10 @@ func (m *Machine) bottomDevices() map[string]bool {
|
|||||||
return out
|
return out
|
||||||
}
|
}
|
||||||
|
|
||||||
// unitIO is every unit's storage I/O, by its cgroup's path: the innermost .service or .scope, because
|
// unitPaths is every systemd unit's cgroup path under /sys/fs/cgroup: the innermost .service or
|
||||||
// io.stat counts a cgroup's descendants too and an outer unit would count them twice.
|
// .scope, because a cgroup's own stat file counts its descendants too and an outer unit would count
|
||||||
func (m *Machine) unitIO(counted map[string]bool) map[string]cgroupIO {
|
// them twice. Shared by every reader that walks the cgroup tree (unitIO here, unitCPU in cpu.go).
|
||||||
|
func (m *Machine) unitPaths() []string {
|
||||||
base := m.path("/sys/fs/cgroup")
|
base := m.path("/sys/fs/cgroup")
|
||||||
var units []string
|
var units []string
|
||||||
filepath.WalkDir(base, func(p string, d fs.DirEntry, err error) error {
|
filepath.WalkDir(base, func(p string, d fs.DirEntry, err error) error {
|
||||||
@@ -89,11 +90,19 @@ func (m *Machine) unitIO(counted map[string]bool) map[string]cgroupIO {
|
|||||||
outer[dir] = true
|
outer[dir] = true
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
out := map[string]cgroupIO{}
|
out := make([]string, 0, len(units))
|
||||||
for _, u := range units {
|
for _, u := range units {
|
||||||
if outer[u] {
|
if !outer[u] {
|
||||||
continue
|
out = append(out, u)
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// unitIO is every unit's storage I/O, by its cgroup's path.
|
||||||
|
func (m *Machine) unitIO(counted map[string]bool) map[string]cgroupIO {
|
||||||
|
out := map[string]cgroupIO{}
|
||||||
|
for _, u := range m.unitPaths() {
|
||||||
text := m.read("/sys/fs/cgroup/" + u + "/io.stat")
|
text := m.read("/sys/fs/cgroup/" + u + "/io.stat")
|
||||||
if text == "" {
|
if text == "" {
|
||||||
continue
|
continue
|
||||||
|
|||||||
@@ -0,0 +1,176 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"path/filepath"
|
||||||
|
"sort"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// cgroupCPU is a cgroup's processor time, from its cpu.stat.
|
||||||
|
type cgroupCPU struct {
|
||||||
|
usageUsec uint64
|
||||||
|
}
|
||||||
|
|
||||||
|
// parseCPUStat reads a cgroup's cpu.stat ("usage_usec 123\nuser_usec …\nsystem_usec …"), readable by
|
||||||
|
// every account, unlike /proc/<pid>/stat for another account's process.
|
||||||
|
func parseCPUStat(s string) cgroupCPU {
|
||||||
|
var c cgroupCPU
|
||||||
|
for _, line := range strings.Split(s, "\n") {
|
||||||
|
k, v, ok := strings.Cut(line, " ")
|
||||||
|
if !ok {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if k == "usage_usec" {
|
||||||
|
c.usageUsec, _ = strconv.ParseUint(strings.TrimSpace(v), 10, 64)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return c
|
||||||
|
}
|
||||||
|
|
||||||
|
// unitCPU is every unit's processor time, by its cgroup's path. A cgroup whose cpu.stat cannot be
|
||||||
|
// read is left out, never an error for the whole answer.
|
||||||
|
func (m *Machine) unitCPU() map[string]cgroupCPU {
|
||||||
|
out := map[string]cgroupCPU{}
|
||||||
|
for _, u := range m.unitPaths() {
|
||||||
|
text := m.read("/sys/fs/cgroup/" + u + "/cpu.stat")
|
||||||
|
if text == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
out[u] = parseCPUStat(text)
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// MachineCPU is the machine's CPU in use over the window, from /proc/stat's cpu line. CPUsBusy is
|
||||||
|
// rounded only for display; raw, unexported, is the same count unrounded, for topCPU's share and
|
||||||
|
// unaccounted math (review round 1 of novox/hq issue 741: a small busy time that rounds CPUsBusy to
|
||||||
|
// 0.0 must not zero out every unit's share along with it).
|
||||||
|
type MachineCPU struct {
|
||||||
|
CPUs int `json:"cpus"`
|
||||||
|
BusyPct float64 `json:"busy_pct"` // % of all CPUs together
|
||||||
|
CPUsBusy float64 `json:"cpus_busy"` // as a count of CPUs, e.g. 7.5 of 12, rounded for display
|
||||||
|
raw float64
|
||||||
|
}
|
||||||
|
|
||||||
|
// machineCPUOver is the machine's CPU in use between two readings of /proc/stat: everything but idle
|
||||||
|
// and iowait, counted against the window's total ticks over every CPU.
|
||||||
|
func machineCPUOver(a, b cpuTimes) MachineCPU {
|
||||||
|
total := float64(b.total() - a.total())
|
||||||
|
if total <= 0 {
|
||||||
|
return MachineCPU{CPUs: b.cpus}
|
||||||
|
}
|
||||||
|
idle := sub(b.idle, a.idle) + sub(b.iowait, a.iowait)
|
||||||
|
busy := total - idle
|
||||||
|
if busy < 0 {
|
||||||
|
busy = 0
|
||||||
|
}
|
||||||
|
raw := busy / total * float64(b.cpus)
|
||||||
|
return MachineCPU{CPUs: b.cpus, BusyPct: round1(busy / total * 100), CPUsBusy: round1(raw), raw: raw}
|
||||||
|
}
|
||||||
|
|
||||||
|
// CPUUsage is a unit's or a container's share of CPU between two readings of its cgroup's cpu.stat.
|
||||||
|
// Cgroup is its cgroup's path, relative to /sys/fs/cgroup, said beside Unit always (review round 1 of
|
||||||
|
// novox/hq issue 741): two cgroups can share a leaf name (an outer "init.scope" and a user session's
|
||||||
|
// "user.slice/user@1000.service/init.scope"), and the path is how they are told apart.
|
||||||
|
type CPUUsage struct {
|
||||||
|
Unit string `json:"unit,omitempty"`
|
||||||
|
Cgroup string `json:"cgroup,omitempty"`
|
||||||
|
Container string `json:"container,omitempty"`
|
||||||
|
CPUPct float64 `json:"cpu_pct"` // % of one CPU: 100 = one CPU busy the whole window
|
||||||
|
SharePct float64 `json:"share_pct"` // this unit's share of the machine's busy time
|
||||||
|
raw float64
|
||||||
|
}
|
||||||
|
|
||||||
|
// topCPU ranks the units, or the containers, with the most CPU time between two readings of
|
||||||
|
// cpu.stat, busiest first. unaccountedPct is the machine's busy time no unit accounts for, as a
|
||||||
|
// share of it (never below 0): the sum is always taken over every unit, whichever by ranks by, so a
|
||||||
|
// limit on the list never changes it.
|
||||||
|
func (m *Machine) topCPU(a, b map[string]cgroupCPU, window time.Duration, by string, limit int, cpusBusy float64) ([]CPUUsage, float64) {
|
||||||
|
windowUsec := window.Seconds() * 1e6
|
||||||
|
type entry struct {
|
||||||
|
path, unit, container string
|
||||||
|
pct float64
|
||||||
|
}
|
||||||
|
var entries []entry
|
||||||
|
var names map[string]string
|
||||||
|
var sumPct float64
|
||||||
|
for path, end := range b {
|
||||||
|
start, ok := a[path]
|
||||||
|
if !ok {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
d := sub(end.usageUsec, start.usageUsec)
|
||||||
|
if d == 0 {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
pct := d / windowUsec * 100
|
||||||
|
sumPct += pct
|
||||||
|
unit := filepath.Base(path)
|
||||||
|
var container string
|
||||||
|
if id := containerID.FindStringSubmatch("/" + path); id != nil {
|
||||||
|
if names == nil {
|
||||||
|
names = m.containerNames()
|
||||||
|
}
|
||||||
|
container = id[1][:12]
|
||||||
|
if n, ok := names[id[1]]; ok {
|
||||||
|
container = n
|
||||||
|
}
|
||||||
|
}
|
||||||
|
entries = append(entries, entry{path: path, unit: unit, container: container, pct: pct})
|
||||||
|
}
|
||||||
|
|
||||||
|
// Units are keyed by their cgroup's own path, never by its leaf name (filepath.Base): two cgroups can
|
||||||
|
// share a leaf name (an outer "init.scope" and a user session's nested one), and keying by the name
|
||||||
|
// alone would merge them into one row (review round 1 of novox/hq issue 741). Containers stay keyed
|
||||||
|
// by their name, as before.
|
||||||
|
groups := map[string]*CPUUsage{}
|
||||||
|
for _, e := range entries {
|
||||||
|
key := e.path
|
||||||
|
if by == "container" {
|
||||||
|
if e.container == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
key = e.container
|
||||||
|
}
|
||||||
|
g, ok := groups[key]
|
||||||
|
if !ok {
|
||||||
|
g = &CPUUsage{Unit: e.unit, Cgroup: e.path, Container: e.container}
|
||||||
|
if by == "container" {
|
||||||
|
g.Unit, g.Cgroup = "", ""
|
||||||
|
}
|
||||||
|
groups[key] = g
|
||||||
|
}
|
||||||
|
g.raw += e.pct
|
||||||
|
}
|
||||||
|
out := make([]CPUUsage, 0, len(groups))
|
||||||
|
for _, g := range groups {
|
||||||
|
out = append(out, *g)
|
||||||
|
}
|
||||||
|
sort.Slice(out, func(i, j int) bool {
|
||||||
|
if out[i].raw != out[j].raw {
|
||||||
|
return out[i].raw > out[j].raw
|
||||||
|
}
|
||||||
|
return out[i].Unit+out[i].Cgroup+out[i].Container < out[j].Unit+out[j].Cgroup+out[j].Container
|
||||||
|
})
|
||||||
|
if len(out) > limit {
|
||||||
|
out = out[:limit]
|
||||||
|
}
|
||||||
|
for i := range out {
|
||||||
|
out[i].CPUPct = round1(out[i].raw)
|
||||||
|
if cpusBusy > 0 {
|
||||||
|
out[i].SharePct = round1(out[i].raw / (cpusBusy * 100) * 100)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
unaccounted := cpusBusy*100 - sumPct
|
||||||
|
if unaccounted < 0 {
|
||||||
|
unaccounted = 0
|
||||||
|
}
|
||||||
|
unaccountedPct := 0.0
|
||||||
|
if cpusBusy > 0 {
|
||||||
|
unaccountedPct = round1(unaccounted / (cpusBusy * 100) * 100)
|
||||||
|
}
|
||||||
|
return out, unaccountedPct
|
||||||
|
}
|
||||||
@@ -0,0 +1,182 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// aCPUHog is a machine whose CPU is 75 % busy (1.5 of 2 CPUs) over a 2 s window: a unit, another unit,
|
||||||
|
// and a container, each with a known usage_usec delta, leaving part of the machine's busy time
|
||||||
|
// unaccounted — the shape of issue 739, where 750 % of 12 CPUs went to processes outside containers.
|
||||||
|
func aCPUHog(t *testing.T) *fake {
|
||||||
|
f := newFake(t)
|
||||||
|
f.file("/proc/stat", "cpu 0 0 0 0 0 0 0 0 0 0\ncpu0 0 0 0 0 0 0 0 0 0 0\ncpu1 0 0 0 0 0 0 0 0 0 0\nintr 1\n")
|
||||||
|
f.then("/proc/stat", "cpu 100 0 50 50 0 0 0 0 0 0\ncpu0 50 0 25 25 0 0 0 0 0 0\ncpu1 50 0 25 25 0 0 0 0 0 0\nintr 2\n")
|
||||||
|
|
||||||
|
zero := "usage_usec 0\nuser_usec 0\nsystem_usec 0\n"
|
||||||
|
cg := func(path string, usec int) {
|
||||||
|
f.file("/sys/fs/cgroup/"+path+"/cpu.stat", zero)
|
||||||
|
f.then("/sys/fs/cgroup/"+path+"/cpu.stat", "usage_usec "+itoa(usec)+"\nuser_usec "+itoa(usec)+"\nsystem_usec 0\n")
|
||||||
|
}
|
||||||
|
cg("system.slice/alpha.service", 1000000) // 50 % of one CPU over 2 s
|
||||||
|
cg("system.slice/beta.service", 400000) // 20 %
|
||||||
|
cg("system.slice/docker-"+store+".scope", 600000) // 30 %, named "cpu-hog" by the runtime
|
||||||
|
f.answers["docker ps --no-trunc --format {{.ID}}\t{{.Names}}"] = store + "\tcpu-hog\n"
|
||||||
|
return f
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCPUTopRanksUnitsAndContainersByCPUOverTheWindow(t *testing.T) {
|
||||||
|
f := aCPUHog(t)
|
||||||
|
out, err := tool(t, f.machine(), "cpu_top").Run(map[string]any{"seconds": 2.0})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
m := out.(map[string]any)
|
||||||
|
if m["by"] != "unit" || f.waited != 2*time.Second {
|
||||||
|
t.Errorf("the default: %+v, waited %s", m, f.waited)
|
||||||
|
}
|
||||||
|
cpu := m["cpu"].(MachineCPU)
|
||||||
|
if cpu.CPUs != 2 || cpu.BusyPct != 75 || cpu.CPUsBusy != 1.5 {
|
||||||
|
t.Errorf("machine cpu %+v", cpu)
|
||||||
|
}
|
||||||
|
top := m["top"].([]CPUUsage)
|
||||||
|
if len(top) != 3 {
|
||||||
|
t.Fatalf("three units moved: %+v", top)
|
||||||
|
}
|
||||||
|
if top[0].Unit != "alpha.service" || top[0].CPUPct != 50 || top[0].SharePct != 33.3 {
|
||||||
|
t.Errorf("busiest first: %+v", top[0])
|
||||||
|
}
|
||||||
|
if top[1].Container != "cpu-hog" || top[1].CPUPct != 30 || top[1].SharePct != 20 {
|
||||||
|
t.Errorf("the container, named by the runtime: %+v", top[1])
|
||||||
|
}
|
||||||
|
if top[2].Unit != "beta.service" || top[2].CPUPct != 20 || top[2].SharePct != 13.3 {
|
||||||
|
t.Errorf("the least busy: %+v", top[2])
|
||||||
|
}
|
||||||
|
if m["unaccounted_pct"] != 33.3 {
|
||||||
|
t.Errorf("unaccounted: %+v (machine busy 150%% of one CPU, units sum to 100%%, 50%% of 150%% left over)", m["unaccounted_pct"])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCPUTopByContainerNamesOnlyContainers(t *testing.T) {
|
||||||
|
out, err := tool(t, aCPUHog(t).machine(), "cpu_top").Run(map[string]any{"by": "container", "seconds": 2.0})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
top := out.(map[string]any)["top"].([]CPUUsage)
|
||||||
|
if len(top) != 1 || top[0].Container != "cpu-hog" || top[0].Unit != "" || top[0].CPUPct != 30 {
|
||||||
|
t.Errorf("by container: %+v", top)
|
||||||
|
}
|
||||||
|
if _, err := tool(t, aCPUHog(t).machine(), "cpu_top").Run(map[string]any{"by": "process"}); err == nil {
|
||||||
|
t.Error("by=process was accepted; cpu_top only ranks units and containers")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCPUTopLimitsHowManyAreNamed(t *testing.T) {
|
||||||
|
out, err := tool(t, aCPUHog(t).machine(), "cpu_top").Run(map[string]any{"limit": 1.0, "seconds": 2.0})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
m := out.(map[string]any)
|
||||||
|
top := m["top"].([]CPUUsage)
|
||||||
|
if len(top) != 1 || top[0].Unit != "alpha.service" {
|
||||||
|
t.Errorf("limit 1: %+v", top)
|
||||||
|
}
|
||||||
|
// The limit narrows the list, never the unaccounted share, which is taken over every unit.
|
||||||
|
if m["unaccounted_pct"] != 33.3 {
|
||||||
|
t.Errorf("unaccounted with limit 1: %+v", m["unaccounted_pct"])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCPUTopLeavesOutACgroupThatAppearsOrVanishesWithinTheWindow(t *testing.T) {
|
||||||
|
f := aCPUHog(t)
|
||||||
|
f.then("/sys/fs/cgroup/system.slice/short-lived.service/cpu.stat", "usage_usec 500000\n")
|
||||||
|
out, err := tool(t, f.machine(), "cpu_top").Run(map[string]any{"seconds": 2.0})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
top := out.(map[string]any)["top"].([]CPUUsage)
|
||||||
|
for _, u := range top {
|
||||||
|
if u.Unit == "short-lived.service" {
|
||||||
|
t.Errorf("a unit only in the second reading was counted: %+v", top)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !strings.Contains(strings.Join(f.calls, ","), "docker ps") {
|
||||||
|
t.Errorf("the container runtime was never asked: %v", f.calls)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// aTinyLoad is a machine with 12 CPUs, only 0.04 of them busy over a 2 s window (busy 1 of 300 total
|
||||||
|
// ticks): rounded to one decimal for display, CPUsBusy reads 0.0, but a unit using half of that busy
|
||||||
|
// time still has a non-zero share — review round 1's decision that share_pct and unaccounted_pct are
|
||||||
|
// computed from the unrounded busy time, never from the rounded CPUsBusy.
|
||||||
|
func aTinyLoad(t *testing.T) *fake {
|
||||||
|
f := newFake(t)
|
||||||
|
cpus := "cpu0 0 0 0 0 0 0 0 0 0 0\ncpu1 0 0 0 0 0 0 0 0 0 0\ncpu2 0 0 0 0 0 0 0 0 0 0\ncpu3 0 0 0 0 0 0 0 0 0 0\n" +
|
||||||
|
"cpu4 0 0 0 0 0 0 0 0 0 0\ncpu5 0 0 0 0 0 0 0 0 0 0\ncpu6 0 0 0 0 0 0 0 0 0 0\ncpu7 0 0 0 0 0 0 0 0 0 0\n" +
|
||||||
|
"cpu8 0 0 0 0 0 0 0 0 0 0\ncpu9 0 0 0 0 0 0 0 0 0 0\ncpu10 0 0 0 0 0 0 0 0 0 0\ncpu11 0 0 0 0 0 0 0 0 0 0\n"
|
||||||
|
f.file("/proc/stat", "cpu 0 0 0 0 0 0 0 0 0 0\n"+cpus)
|
||||||
|
f.then("/proc/stat", "cpu 1 0 0 299 0 0 0 0 0 0\n"+cpus)
|
||||||
|
|
||||||
|
f.file("/sys/fs/cgroup/system.slice/tiny.service/cpu.stat", "usage_usec 0\n")
|
||||||
|
f.then("/sys/fs/cgroup/system.slice/tiny.service/cpu.stat", "usage_usec 60000\n") // 3 % of one CPU over 2 s
|
||||||
|
return f
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCPUTopSharesAreNonZeroEvenWhenRoundedCPUsBusyIsZero(t *testing.T) {
|
||||||
|
out, err := tool(t, aTinyLoad(t).machine(), "cpu_top").Run(map[string]any{"seconds": 2.0})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
m := out.(map[string]any)
|
||||||
|
cpu := m["cpu"].(MachineCPU)
|
||||||
|
if cpu.CPUs != 12 || cpu.CPUsBusy != 0 {
|
||||||
|
t.Fatalf("the machine 0.04 CPUs busy, rounded for display: %+v", cpu)
|
||||||
|
}
|
||||||
|
top := m["top"].([]CPUUsage)
|
||||||
|
if len(top) != 1 || top[0].Unit != "tiny.service" || top[0].CPUPct != 3 || top[0].SharePct != 75 {
|
||||||
|
t.Fatalf("a non-zero share from the unrounded busy time: %+v", top)
|
||||||
|
}
|
||||||
|
if m["unaccounted_pct"] != 25.0 {
|
||||||
|
t.Fatalf("unaccounted from the unrounded busy time: %+v (want 25, so shares plus unaccounted add to 100)",
|
||||||
|
m["unaccounted_pct"])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestCPUTopKeepsCgroupsWithTheSameUnitNameAsSeparateRows is review round 1's other decision: units are keyed
|
||||||
|
// by their cgroup path, never by filepath.Base, so two cgroups that both end in "init.scope" — one at the
|
||||||
|
// root, one under a user's session — are two rows, not one merged by their shared leaf name.
|
||||||
|
func TestCPUTopKeepsCgroupsWithTheSameUnitNameAsSeparateRows(t *testing.T) {
|
||||||
|
f := aCPUHog(t)
|
||||||
|
cg := func(path string, usec int) {
|
||||||
|
f.file("/sys/fs/cgroup/"+path+"/cpu.stat", "usage_usec 0\n")
|
||||||
|
f.then("/sys/fs/cgroup/"+path+"/cpu.stat", "usage_usec "+itoa(usec)+"\n")
|
||||||
|
}
|
||||||
|
cg("init.scope", 1000000) // the root init, 50 % of one CPU over 2 s
|
||||||
|
cg("user.slice/user@1000.service/init.scope", 400000) // a user session's init, 20 %
|
||||||
|
|
||||||
|
out, err := tool(t, f.machine(), "cpu_top").Run(map[string]any{"seconds": 2.0, "limit": 10.0})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
top := out.(map[string]any)["top"].([]CPUUsage)
|
||||||
|
var inits []CPUUsage
|
||||||
|
for _, u := range top {
|
||||||
|
if u.Unit == "init.scope" {
|
||||||
|
inits = append(inits, u)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(inits) != 2 {
|
||||||
|
t.Fatalf("two cgroups named init.scope, merged into %d row(s): %+v", len(inits), top)
|
||||||
|
}
|
||||||
|
byCgroup := map[string]CPUUsage{}
|
||||||
|
for _, u := range inits {
|
||||||
|
byCgroup[u.Cgroup] = u
|
||||||
|
}
|
||||||
|
if u, ok := byCgroup["init.scope"]; !ok || u.CPUPct != 50 {
|
||||||
|
t.Errorf("the root init.scope: %+v", byCgroup)
|
||||||
|
}
|
||||||
|
if u, ok := byCgroup["user.slice/user@1000.service/init.scope"]; !ok || u.CPUPct != 20 {
|
||||||
|
t.Errorf("the user session's init.scope: %+v", byCgroup)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -85,6 +85,57 @@ func Tools(m *Machine) []stdio.Tool {
|
|||||||
return out, nil
|
return out, nil
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
Name: "cpu_top",
|
||||||
|
Description: "What used the most CPU over a few seconds, from each unit's cgroup cpu.stat, which needs no root: the machine's " +
|
||||||
|
"CPU in use (as % of all CPUs and as CPUs busy, from /proc/stat), then by=unit (the default) each systemd unit and " +
|
||||||
|
"container ranked by CPU, busiest first, with the busy time no unit accounts for; by=container only the containers, " +
|
||||||
|
"by name. Replaces top, systemd-cgtop. (r)",
|
||||||
|
Input: map[string]any{
|
||||||
|
"seconds": map[string]any{"type": "integer", "description": "how long to sample, in seconds (default 3, at most 10)"},
|
||||||
|
"limit": map[string]any{"type": "integer", "description": "how many units or containers to name (default 15, at most 50)"},
|
||||||
|
"by": map[string]any{"type": "string", "enum": []string{"unit", "container"}, "description": "what to rank (default unit)"},
|
||||||
|
},
|
||||||
|
Run: func(args map[string]any) (any, error) {
|
||||||
|
n, err := count(args, "seconds", 3, 1, 10)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
w := time.Duration(n) * time.Second
|
||||||
|
limit, err := count(args, "limit", 15, 1, 50)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
by := text(args, "by")
|
||||||
|
switch by {
|
||||||
|
case "":
|
||||||
|
by = "unit"
|
||||||
|
case "unit", "container":
|
||||||
|
default:
|
||||||
|
return nil, fmt.Errorf("by is unit or container, not %q", by)
|
||||||
|
}
|
||||||
|
ac, err := parseCPU(m.read("/proc/stat"))
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
a := m.unitCPU()
|
||||||
|
m.Wait(w)
|
||||||
|
bc, err := parseCPU(m.read("/proc/stat"))
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
b := m.unitCPU()
|
||||||
|
machine := machineCPUOver(ac, bc)
|
||||||
|
top, unaccounted := m.topCPU(a, b, w, by, limit, machine.raw)
|
||||||
|
return map[string]any{
|
||||||
|
"window_s": w.Seconds(),
|
||||||
|
"by": by,
|
||||||
|
"cpu": machine,
|
||||||
|
"top": top,
|
||||||
|
"unaccounted_pct": unaccounted,
|
||||||
|
}, nil
|
||||||
|
},
|
||||||
|
},
|
||||||
{
|
{
|
||||||
Name: "disk_filesystems",
|
Name: "disk_filesystems",
|
||||||
Description: "The space of every filesystem that stores data (not proc, tmpfs or overlay layers): size, used and free in GiB, " +
|
Description: "The space of every filesystem that stores data (not proc, tmpfs or overlay layers): size, used and free in GiB, " +
|
||||||
|
|||||||
@@ -5,7 +5,8 @@
|
|||||||
"disk_load",
|
"disk_load",
|
||||||
"disk_top",
|
"disk_top",
|
||||||
"disk_filesystems",
|
"disk_filesystems",
|
||||||
"disk_devices"
|
"disk_devices",
|
||||||
|
"cpu_top"
|
||||||
],
|
],
|
||||||
"build": {
|
"build": {
|
||||||
"artifacts": [
|
"artifacts": [
|
||||||
|
|||||||
+22
-5
@@ -44,7 +44,7 @@ base64-encoded. The Go bundle `cmd/forge` serves the commit, and the reads an ag
|
|||||||
|
|
||||||
| tool | | what |
|
| tool | | what |
|
||||||
|---|---|---|
|
|---|---|---|
|
||||||
| `gitea_commit` | a | one commit on a branch of its own, created from `base` (default the default branch) when absent, carrying either `files` (each `{path, content \| content_base64 \| delete: true}`) or `patch` (a unified diff, as `git diff` writes it); answers the commit and its parent |
|
| `gitea_commit` | a | one commit on a branch of its own, created from `base` (default the default branch) when absent, carrying either `files` (each `{path, content \| content_base64 \| delete: true}`) `patch` (a unified diff, as `git diff` writes it), or `patch_file` with `patch_sha256` (a patch too large to type, see below); answers the commit and its parent |
|
||||||
| `gitea_commits` | r | a ref's history, newest first, optionally only commits touching a path: sha, author, date, subject |
|
| `gitea_commits` | r | a ref's history, newest first, optionally only commits touching a path: sha, author, date, subject |
|
||||||
| `gitea_compare` | r | what head holds that base does not: its commits and the files they touch |
|
| `gitea_compare` | r | what head holds that base does not: its commits and the files they touch |
|
||||||
| `gitea_pull_reviews` | r | a pull request's reviews — state, the commit reviewed and whether it is the head now, stale, dismissed — with their line comments |
|
| `gitea_pull_reviews` | r | a pull request's reviews — state, the commit reviewed and whether it is the head now, stale, dismissed — with their line comments |
|
||||||
@@ -52,13 +52,28 @@ base64-encoded. The Go bundle `cmd/forge` serves the commit, and the reads an ag
|
|||||||
| `gitea_search_code` | r | the lines of a ref's text files matching a literal string or a regular expression, optionally under a path |
|
| `gitea_search_code` | r | the lines of a ref's text files matching a literal string or a regular expression, optionally under a path |
|
||||||
|
|
||||||
**Where a commit may go** is the rule `gitea_api` already holds for a file written (novox/hq ADR 0266):
|
**Where a commit may go** is the rule `gitea_api` already holds for a file written (novox/hq ADR 0266):
|
||||||
never the repository's default branch, never a branch a protection rule names. A rule covers a branch as the
|
never the repository's default branch, never a branch a protection rule names. The tools' account may not read the
|
||||||
|
rules themselves (the forge keeps them for a repository's administrator, novox/hq ADR 0365): for it an existing branch
|
||||||
|
is judged by the protection the forge says it applies there, and a branch not made yet by the forge itself, which
|
||||||
|
refuses the commit that would make a branch a rule guards. A rule covers a branch as the
|
||||||
forge itself decides it (Gitea 1.27, novox/hq issue 539): a plain name covers that branch in any case; a glob
|
forge itself decides it (Gitea 1.27, novox/hq issue 539): a plain name covers that branch in any case; a glob
|
||||||
(`*`, `**`, `?`, `[abc]`, `[!abc]`, `{a,b}`, `\` escaping) covers case-sensitively with `/` as separator; a glob
|
(`*`, `**`, `?`, `[abc]`, `[!abc]`, `{a,b}`, `\` escaping) covers case-sensitively with `/` as separator; a glob
|
||||||
the forge cannot compile covers its own text. `cmd/forge/rules.go` and `guard.ts` hold this one rule. The rules are read before anything is sent; rules that cannot be read refuse the
|
the forge cannot compile covers its own text. `cmd/forge/rules.go` and `guard.ts` hold this one rule. The rules are read before anything is sent; rules that cannot be read refuse the
|
||||||
commit. Nothing about merging changes: a merge still needs an independent review at the head and both
|
commit. Nothing about merging changes: a merge still needs an independent review at the head and both
|
||||||
required checks green, through `gitea_merge_pull_request`.
|
required checks green, through `gitea_merge_pull_request`.
|
||||||
|
|
||||||
|
**A patch too large to type** (novox/hq issue 852: a feature's diff is megabytes) goes as `patch_file`, with
|
||||||
|
`patch_sha256`. The caller writes the patch into the module's spool directory `forge-patches`
|
||||||
|
(`/var/lib/mesh/forge-patches`, mode 0733 and owned by the machine's account, which the tools run as: anyone writes,
|
||||||
|
nobody lists; the bundle is given its path as `MESH_GITEA_PATCH_DIR`) and names it: `patch_file` is the file's NAME,
|
||||||
|
32 to 64 characters of `0-9a-f` that the caller chooses at random, never a path. The tool then reads that one file,
|
||||||
|
applies it exactly as it applies `patch` (same size limit, same answer) and removes it once it has found it, whether
|
||||||
|
the commit was made or not. A file refused before that (a bad name, a missing hash or spool directory, a name with no
|
||||||
|
file) stays in the spool. It refuses, sending nothing, a name of another shape, a file that is missing, empty, over
|
||||||
|
4 MiB, with more than one hard link or not a regular file (a symlink is never followed), and a file whose SHA-256 is not `patch_sha256`: the directory has no sticky bit
|
||||||
|
(the manifest allows none), so another writer could replace a file, and the hash is how the caller knows the tool read
|
||||||
|
the bytes it wrote. It reads nothing outside the directory.
|
||||||
|
|
||||||
**How the commit is made.** `files` become one call to the forge's multi-file contents endpoint: a path the
|
**How the commit is made.** `files` become one call to the forge's multi-file contents endpoint: a path the
|
||||||
ref holds is updated, one it does not is created, and an update or delete carries the blob read just
|
ref holds is updated, one it does not is created, and an update or delete carries the blob read just
|
||||||
before, so a file someone changed meanwhile is refused by the forge rather than overwritten. The blobs are read at
|
before, so a file someone changed meanwhile is refused by the forge rather than overwritten. The blobs are read at
|
||||||
@@ -66,7 +81,7 @@ the commit the branch (or, for a new branch, its base) was read at, never at its
|
|||||||
forge's diff endpoint, which applies it; for a small change to a large file this is far shorter than the
|
forge's diff endpoint, which applies it; for a small change to a large file this is far shorter than the
|
||||||
file. Neither endpoint takes a commit to build on, only a branch (Gitea 1.27), so the parent answered is the
|
file. Neither endpoint takes a commit to build on, only a branch (Gitea 1.27), so the parent answered is the
|
||||||
one the forge says it made the commit on; when the branch moved between the read and the commit, the answer
|
one the forge says it made the commit on; when the branch moved between the read and the commit, the answer
|
||||||
says so. One commit carries at most 200 files, 4 MiB per file and 16 MiB together, or a patch of 4 MiB. Either way the commit is the forge's own, made as the forge's admin account, so its sha is not the
|
says so. One commit carries at most 200 files, 4 MiB per file and 16 MiB together, or a patch of 4 MiB. Either way the commit is the forge's own, made as the tools' account, so its sha is not the
|
||||||
local commit's: the merge check judges the head on the forge. After the write the branch is read back, and a
|
local commit's: the merge check judges the head on the forge. After the write the branch is read back, and a
|
||||||
branch not at the commit the forge named is a failure, never a success.
|
branch not at the commit the forge named is a failure, never a success.
|
||||||
|
|
||||||
@@ -75,8 +90,10 @@ is read once and searched in memory; a binary file is passed over, a file over 4
|
|||||||
`skipped`, and at most `max` matches (default 100) are answered, with `truncated` when more were found. A
|
`skipped`, and at most `max` matches (default 100) are answered, with `truncated` when more were found. A
|
||||||
search that reaches the call's deadline stops there and answers what it found, with `stopped_at_deadline`.
|
search that reaches the call's deadline stops there and answers what it found, with `stopped_at_deadline`.
|
||||||
|
|
||||||
It reaches the forge through the same HTTP client as the npm-registry bundle (`internal/forgehttp`), and one
|
It reaches the forge through the same HTTP client as the npm-registry bundle (`internal/forgehttp`), as the
|
||||||
tool call takes at most 50 seconds.
|
tools' own account (novox/hq ADR 0365 §2, issue 469): the token the TypeScript bundle mints for `mesh-tools` and
|
||||||
|
keeps (`MESH_GITEA_TOKEN_FILE`), sent only when its scopes line names that account, never the admin login. One tool
|
||||||
|
call takes at most 50 seconds.
|
||||||
`go test ./cmd/forge` runs it against a fake forge: the files of one commit created, updated and deleted on
|
`go test ./cmd/forge` runs it against a fake forge: the files of one commit created, updated and deleted on
|
||||||
a new branch from its base; an existing branch committed on; the default branch, a protected branch and
|
a new branch from its base; an existing branch committed on; the default branch, a protected branch and
|
||||||
unreadable rules refused before anything is sent; a patch sent to the forge; a branch read back at another
|
unreadable rules refused before anything is sent; a patch sent to the forge; a branch read back at another
|
||||||
|
|||||||
+60
-12
@@ -8,6 +8,7 @@ import { ConfiguredToken, MintedToken, type TokenSource } from "./token.js";
|
|||||||
import { protectionBody, type BranchProtection, type ProtectionWanted } from "./protection.js";
|
import { protectionBody, type BranchProtection, type ProtectionWanted } from "./protection.js";
|
||||||
import { Refused, checkSentPath, pathSegment, pathSegments } from "./guard.js";
|
import { Refused, checkSentPath, pathSegment, pathSegments } from "./guard.js";
|
||||||
import { isCommit } from "./checks.js";
|
import { isCommit } from "./checks.js";
|
||||||
|
import { TOOLS_ACCOUNT } from "./tools-account.js";
|
||||||
|
|
||||||
/** A repository's path on the forge, its owner and name each one segment (guard.ts pathSegment): uniform for every
|
/** A repository's path on the forge, its owner and name each one segment (guard.ts pathSegment): uniform for every
|
||||||
* call, whether the names came from a caller or from the forge's own answer. */
|
* call, whether the names came from a caller or from the forge's own answer. */
|
||||||
@@ -124,10 +125,12 @@ export interface ReadyMark {
|
|||||||
|
|
||||||
/** **The accounts whose "ready at" comment counts** (novox/hq issue 722): the trusted setting ready-authors, which
|
/** **The accounts whose "ready at" comment counts** (novox/hq issue 722): the trusted setting ready-authors, which
|
||||||
* the mesh writes to the file MESH_GITEA_READY_AUTHORS_FILE names as a line `authors=<account> <account> …`. A file
|
* the mesh writes to the file MESH_GITEA_READY_AUTHORS_FILE names as a line `authors=<account> <account> …`. A file
|
||||||
* that cannot be read leaves the mesh's own forge account alone, said; never an empty list read as everyone. */
|
* that cannot be read leaves the mesh's own forge accounts alone, said; never an empty list read as everyone: the
|
||||||
|
* account the tools act as (novox/hq ADR 0365) and the admin account, whose marks were made before it. */
|
||||||
export function readyAuthors(file = process.env.MESH_GITEA_READY_AUTHORS_FILE, own = process.env.MESH_GITEA_ADMIN_USER,
|
export function readyAuthors(file = process.env.MESH_GITEA_READY_AUTHORS_FILE, own = process.env.MESH_GITEA_ADMIN_USER,
|
||||||
read: (f: string) => string = (f) => readFileSync(f, "utf8")): string[] {
|
read: (f: string) => string = (f) => readFileSync(f, "utf8"),
|
||||||
const fallback = own ? [own] : [];
|
tools = process.env.MESH_GITEA_TOOLS_USER || TOOLS_ACCOUNT): string[] {
|
||||||
|
const fallback = [...new Set([tools, own].filter((a): a is string => !!a))];
|
||||||
if (!file) return fallback;
|
if (!file) return fallback;
|
||||||
try {
|
try {
|
||||||
const line = read(file).split("\n").find((l) => l.startsWith("authors="));
|
const line = read(file).split("\n").find((l) => l.startsWith("authors="));
|
||||||
@@ -444,13 +447,15 @@ export class GiteaClient {
|
|||||||
* fault in the announcer cannot hide itself. Reads only. */
|
* fault in the announcer cannot hide itself. Reads only. */
|
||||||
async openPulls(readyAuthors: string[] = []): Promise<OpenPull[]> {
|
async openPulls(readyAuthors: string[] = []): Promise<OpenPull[]> {
|
||||||
const out: OpenPull[] = [];
|
const out: OpenPull[] = [];
|
||||||
const rules = new Map<string, BranchProtection[] | undefined>();
|
const rules = new Map<string, BranchProtection | null | undefined>();
|
||||||
for (const found of await this.searchOpenPulls()) {
|
for (const found of await this.searchOpenPulls()) {
|
||||||
const full = `${found.owner}/${found.name}`;
|
const full = `${found.owner}/${found.name}`;
|
||||||
const pull = await this.getPullRequest(found.owner, found.name, found.number);
|
const pull = await this.getPullRequest(found.owner, found.name, found.number);
|
||||||
if (pull.state !== "open" || !pull.head_sha) continue;
|
if (pull.state !== "open" || !pull.head_sha) continue;
|
||||||
if (!rules.has(full)) rules.set(full, await this.branchProtections(found.owner, found.name).catch(() => undefined));
|
// The base's protection as the forge applies it to that branch (appliedProtection): readable without administration.
|
||||||
const rule = rules.get(full)?.find((p) => (p.rule_name ?? p.branch_name) === (pull.base || "main"));
|
const base = `${full}:${pull.base || "main"}`;
|
||||||
|
if (!rules.has(base)) rules.set(base, await this.appliedProtection(found.owner, found.name, pull.base || "main").catch(() => undefined));
|
||||||
|
const rule = rules.get(base);
|
||||||
const checks = await this.headChecks(found.owner, found.name, pull.head_sha);
|
const checks = await this.headChecks(found.owner, found.name, pull.head_sha);
|
||||||
const labels = found.labels ?? [];
|
const labels = found.labels ?? [];
|
||||||
// The ready mark (novox/hq issue 722): its timeline is read, whole, only for a pull request that carries the label.
|
// The ready mark (novox/hq issue 722): its timeline is read, whole, only for a pull request that carries the label.
|
||||||
@@ -673,16 +678,44 @@ export class GiteaClient {
|
|||||||
|
|
||||||
// ---- Branch protection ----
|
// ---- Branch protection ----
|
||||||
|
|
||||||
/** A repository's branch protection rules. */
|
/** A repository's branch protection rules. The forge answers them to a repository's administrator alone, which the
|
||||||
|
* tools' account is not (novox/hq ADR 0365 §2): for it this is refused (403), and what a branch's protection asks
|
||||||
|
* is read with appliedProtection. */
|
||||||
async branchProtections(owner: string, repo: string): Promise<BranchProtection[]> {
|
async branchProtections(owner: string, repo: string): Promise<BranchProtection[]> {
|
||||||
return (await this.request<BranchProtection[]>(`${repoPath(owner, repo)}/branch_protections`)) ?? [];
|
return (await this.request<BranchProtection[]>(`${repoPath(owner, repo)}/branch_protections`)) ?? [];
|
||||||
}
|
}
|
||||||
|
|
||||||
/** The rule named for a branch, null when it has none. */
|
/** The rule named for a branch, null when it has none — from the rules' list, so a repository administrator's. */
|
||||||
async branchProtection(owner: string, repo: string, rule: string): Promise<BranchProtection | null> {
|
async branchProtection(owner: string, repo: string, rule: string): Promise<BranchProtection | null> {
|
||||||
return (await this.branchProtections(owner, repo)).find((p) => (p.rule_name ?? p.branch_name) === rule) ?? null;
|
return (await this.branchProtections(owner, repo)).find((p) => (p.rule_name ?? p.branch_name) === rule) ?? null;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The protection that covers an existing branch, as the forge applies it, or null when none does. Read from the
|
||||||
|
* branch itself, which anyone who reads the code may read — the rules' own list is a repository administrator's
|
||||||
|
* (novox/hq ADR 0365 §2) — and as the forge matches it, a glob rule included (its name is the rule's). The
|
||||||
|
* branch's `user_can_push` stands for `enable_push`: whether this account may push there. A branch that does not
|
||||||
|
* exist is null: nothing covers it yet, and the forge judges a push that makes it.
|
||||||
|
*/
|
||||||
|
async appliedProtection(owner: string, repo: string, branch: string): Promise<BranchProtection | null> {
|
||||||
|
let b: any;
|
||||||
|
try {
|
||||||
|
b = await this.request<any>(`${repoPath(owner, repo)}/branches/${pathSegments("branch", branch)}`);
|
||||||
|
} catch (err) {
|
||||||
|
if (/: 404 /.test(String((err as Error)?.message))) return null;
|
||||||
|
throw err;
|
||||||
|
}
|
||||||
|
if (!b?.protected) return null;
|
||||||
|
return {
|
||||||
|
rule_name: String(b.effective_branch_protection_name || branch),
|
||||||
|
branch_name: branch,
|
||||||
|
enable_push: Boolean(b.user_can_push),
|
||||||
|
enable_status_check: Boolean(b.enable_status_check),
|
||||||
|
status_check_contexts: Array.isArray(b.status_check_contexts) ? b.status_check_contexts.map(String) : [],
|
||||||
|
required_approvals: Number(b.required_approvals ?? 0),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
/** Make a branch require these commit statuses to merge (novox/hq ADR 0237): the rule named for the
|
/** Make a branch require these commit statuses to merge (novox/hq ADR 0237): the rule named for the
|
||||||
* branch is edited — its required statuses replaced by these, everything else it says kept unless
|
* branch is edited — its required statuses replaced by these, everything else it says kept unless
|
||||||
* asked — or created, refusing direct pushes unless asked otherwise. Answers the rule as it is now. */
|
* asked — or created, refusing direct pushes unless asked otherwise. Answers the rule as it is now. */
|
||||||
@@ -925,6 +958,15 @@ export class GiteaAdmin {
|
|||||||
throw new Error(`Gitea admin ${path}: ${res.status} ${detail}`);
|
throw new Error(`Gitea admin ${path}: ${res.status} ${detail}`);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** A repository's branch protection rules, read with the administrator account — the forge answers them to a
|
||||||
|
* repository's administrator alone, which the tools' account is not (novox/hq ADR 0365 §2, ADR 0375). Only a read. */
|
||||||
|
async branchProtections(owner: string, repo: string): Promise<BranchProtection[]> {
|
||||||
|
const path = `${repoPath(owner, repo)}/branch_protections`;
|
||||||
|
const res = await this.request(path);
|
||||||
|
if (res.status !== 200) GiteaAdmin.fail(path, res);
|
||||||
|
return Array.isArray(res.body) ? res.body : [];
|
||||||
|
}
|
||||||
|
|
||||||
/** Ensure the npm-owner org exists. 201 created, 2xx/404-then-created, and 422/409 (a concurrent
|
/** Ensure the npm-owner org exists. 201 created, 2xx/404-then-created, and 422/409 (a concurrent
|
||||||
* create won the race) are all success. */
|
* create won the race) are all success. */
|
||||||
async ensureOrg(name: string): Promise<void> {
|
async ensureOrg(name: string): Promise<void> {
|
||||||
@@ -948,12 +990,16 @@ export class GiteaAdmin {
|
|||||||
// unit the team does not have — so code read must be said here: without it gitea answers a
|
// unit the team does not have — so code read must be said here: without it gitea answers a
|
||||||
// member's clone of a private repository with "not found", which is how the builder's first
|
// member's clone of a private repository with "not found", which is how the builder's first
|
||||||
// credentialed clone failed against a team that named only packages.
|
// credentialed clone failed against a team that named only packages.
|
||||||
const units = {
|
return this.ensureTeamWith(org, team, {
|
||||||
permission: "read",
|
permission: "read",
|
||||||
units_map: { "repo.code": "read", "repo.packages": packageWrite ? "write" : "read" },
|
units_map: { "repo.code": "read", "repo.packages": packageWrite ? "write" : "read" },
|
||||||
includes_all_repositories: true,
|
includes_all_repositories: true,
|
||||||
can_create_org_repo: false,
|
can_create_org_repo: false,
|
||||||
};
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Ensure a team exists with exactly these units, and return its id (ensureTeam; the tools' team, tools-account.ts). */
|
||||||
|
async ensureTeamWith(org: string, team: string, units: Record<string, unknown>): Promise<number> {
|
||||||
const found = await this.findTeam(org, team);
|
const found = await this.findTeam(org, team);
|
||||||
if (found !== null) {
|
if (found !== null) {
|
||||||
const patch = await this.request(`/teams/${found}`, {
|
const patch = await this.request(`/teams/${found}`, {
|
||||||
@@ -988,7 +1034,7 @@ export class GiteaAdmin {
|
|||||||
* The edit path is taken only when the user actually exists. A 422 from the create is also what
|
* The edit path is taken only when the user actually exists. A 422 from the create is also what
|
||||||
* a plain validation failure returns, and reading it as "already there" made the follow-up edit
|
* a plain validation failure returns, and reading it as "already there" made the follow-up edit
|
||||||
* 404 — burying the create's own message, which is the one that says what is actually wrong. */
|
* 404 — burying the create's own message, which is the one that says what is actually wrong. */
|
||||||
async ensureUser(username: string, password: string, email: string): Promise<void> {
|
async ensureUser(username: string, password: string, email: string, plain = false): Promise<void> {
|
||||||
const res = await this.request("/admin/users", {
|
const res = await this.request("/admin/users", {
|
||||||
method: "POST",
|
method: "POST",
|
||||||
body: JSON.stringify({ username, email, password, must_change_password: false }),
|
body: JSON.stringify({ username, email, password, must_change_password: false }),
|
||||||
@@ -1002,7 +1048,9 @@ export class GiteaAdmin {
|
|||||||
// login_name is required by the admin edit endpoint; for a local user it is the username.
|
// login_name is required by the admin edit endpoint; for a local user it is the username.
|
||||||
// active and prohibit_login: a deactivated or login-prohibited user is refused like a wrong
|
// active and prohibit_login: a deactivated or login-prohibited user is refused like a wrong
|
||||||
// password, so the provisioner's check reports it lost; applying again must undo both.
|
// password, so the provisioner's check reports it lost; applying again must undo both.
|
||||||
body: JSON.stringify({ login_name: username, password, must_change_password: false, active: true, prohibit_login: false }),
|
// plain: never a site administrator — one made so by hand is made a plain account again (novox/hq ADR 0365).
|
||||||
|
body: JSON.stringify({ login_name: username, password, must_change_password: false, active: true, prohibit_login: false,
|
||||||
|
...(plain ? { admin: false } : {}) }),
|
||||||
});
|
});
|
||||||
if (patch.status === 200) return;
|
if (patch.status === 200) return;
|
||||||
GiteaAdmin.fail(`/admin/users/${username}`, patch);
|
GiteaAdmin.fail(`/admin/users/${username}`, patch);
|
||||||
|
|||||||
@@ -0,0 +1,120 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
// The commit and the reads act as the tools' own account (novox/hq ADR 0365 §2, issue 469): the token the module
|
||||||
|
// keeps for `mesh-tools`, never the admin login. That account is no repository's administrator, so the forge
|
||||||
|
// refuses it the protection rules (403, as gitea's reqAdmin does); the commit then judges an existing branch by the
|
||||||
|
// protection the forge says it applies, and leaves a branch not made yet to the forge's own judgement.
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"gitea/internal/forgehttp"
|
||||||
|
)
|
||||||
|
|
||||||
|
const toolsToken = "tools-token-never-shown"
|
||||||
|
|
||||||
|
// asTools serves the fake forge to a bundle holding the tools' token: only that token is taken, the rules are
|
||||||
|
// refused as to a non-administrator, and the branches named protected say so as the forge does.
|
||||||
|
func asTools(t *testing.T, f *fakeForge, protected map[string]string, scopes string) (*Forge, *[]string) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
file := filepath.Join(dir, "token")
|
||||||
|
if err := os.WriteFile(file, []byte(toolsToken+"\n"), 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(file+".scopes", []byte(scopes+"\n"), 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var seen []string
|
||||||
|
const repo = "/api/v1/repos/acme/app"
|
||||||
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
seen = append(seen, r.Method+" "+r.URL.Path)
|
||||||
|
if _, _, basic := r.BasicAuth(); basic || r.Header.Get("Authorization") != "token "+toolsToken {
|
||||||
|
http.Error(w, "unauthorized", 401)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if r.URL.Path == repo+"/branch_protections" {
|
||||||
|
http.Error(w, `{"message":"token does not have the administration of this repository"}`, 403)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if name := strings.TrimPrefix(r.URL.Path, repo+"/branches/"); r.Method == "GET" && name != r.URL.Path {
|
||||||
|
if rule, ok := protected[name]; ok {
|
||||||
|
_, _ = w.Write([]byte(`{"name":"` + name + `","commit":{"id":"c0"},"protected":true,"effective_branch_protection_name":"` + rule + `"}`))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
r.SetBasicAuth("mesh-admin", f.password) // the fake's own door, past the token checked above
|
||||||
|
f.handle(w, r)
|
||||||
|
}))
|
||||||
|
t.Cleanup(srv.Close)
|
||||||
|
return &Forge{Client: forgehttp.Client{URL: srv.URL, TokenFile: file, Account: "mesh-tools"}}, &seen
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTheToolsAccountCommitsOnANewBranchWithItsTokenThoughTheRulesAreNotItsToRead(t *testing.T) {
|
||||||
|
f := newFake(t)
|
||||||
|
forge, _ := asTools(t, f, nil, "read:user,write:issue,write:repository for mesh-tools")
|
||||||
|
got, err := commit(context.Background(), forge, ask(t, map[string]any{"branch": "fix/365",
|
||||||
|
"files": filesArg(map[string]any{"path": "x", "content": "x"})}))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(f.writes) != 1 || !got.Created {
|
||||||
|
t.Fatalf("%d writes, %+v", len(f.writes), got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTheToolsAccountIsRefusedAProtectedBranchTheForgeSaysItGuardsAndTheDefaultBranch(t *testing.T) {
|
||||||
|
for branch, rule := range map[string]string{"release/1.0": "release/*", "fix/1-old": "fix/1-old", "main": ""} {
|
||||||
|
f := newFake(t)
|
||||||
|
protected := map[string]string{}
|
||||||
|
if rule != "" {
|
||||||
|
protected[branch] = rule
|
||||||
|
}
|
||||||
|
forge, _ := asTools(t, f, protected, "read:user,write:issue,write:repository for mesh-tools")
|
||||||
|
_, err := commit(context.Background(), forge, ask(t, map[string]any{"branch": branch,
|
||||||
|
"files": filesArg(map[string]any{"path": "x", "content": "x"})}))
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "refused") || !strings.Contains(err.Error(), "ADR 0266") {
|
||||||
|
t.Errorf("%s: %v", branch, err)
|
||||||
|
}
|
||||||
|
if rule != "" && !strings.Contains(err.Error(), rule) {
|
||||||
|
t.Errorf("%s: the rule %s is not named: %v", branch, rule, err)
|
||||||
|
}
|
||||||
|
if len(f.writes)+len(f.patches) != 0 {
|
||||||
|
t.Errorf("%s: something was written", branch)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestATokenKeptForTheAdministratorIsNeverSent(t *testing.T) {
|
||||||
|
f := newFake(t)
|
||||||
|
// What the module kept before ADR 0365: the admin's token, its scopes line naming no account.
|
||||||
|
forge, seen := asTools(t, f, nil, "read:user,write:issue,write:repository")
|
||||||
|
_, err := commit(context.Background(), forge, ask(t, map[string]any{"branch": "fix/365",
|
||||||
|
"files": filesArg(map[string]any{"path": "x", "content": "x"})}))
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "not mesh-tools's") {
|
||||||
|
t.Fatalf("%v", err)
|
||||||
|
}
|
||||||
|
if len(*seen) != 0 {
|
||||||
|
t.Fatalf("the forge was asked %v", *seen)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTheBundleHandedATokenFileNeedsNoAdminLogin(t *testing.T) {
|
||||||
|
t.Setenv("MESH_GITEA_URL", "http://127.0.0.1:3000")
|
||||||
|
t.Setenv("MESH_GITEA_TOKEN_FILE", "/state/token")
|
||||||
|
t.Setenv("MESH_GITEA_TOOLS_USER", "")
|
||||||
|
t.Setenv("MESH_GITEA_ADMIN_USER", "")
|
||||||
|
t.Setenv("MESH_GITEA_ADMIN_PASSWORD_FILE", "")
|
||||||
|
f, err := ForgeFromEnv()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if f.TokenFile != "/state/token" || f.Account != "mesh-tools" || f.User != "" || f.PasswordFile != "" {
|
||||||
|
t.Fatalf("%+v", f.Client)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -9,6 +9,8 @@ import (
|
|||||||
"net/url"
|
"net/url"
|
||||||
"strings"
|
"strings"
|
||||||
"unicode/utf8"
|
"unicode/utf8"
|
||||||
|
|
||||||
|
"gitea/internal/forgehttp"
|
||||||
)
|
)
|
||||||
|
|
||||||
// gitea_commit (novox/hq issue 375): one commit on a branch of its own, so an agent follows the branch / pull
|
// gitea_commit (novox/hq issue 375): one commit on a branch of its own, so an agent follows the branch / pull
|
||||||
@@ -33,6 +35,11 @@ type CommitAsk struct {
|
|||||||
Message string
|
Message string
|
||||||
Files []FileChange
|
Files []FileChange
|
||||||
Patch string // a unified diff, as git diff writes it, instead of Files
|
Patch string // a unified diff, as git diff writes it, instead of Files
|
||||||
|
// PatchFile is the name of a file in the spool directory holding the patch (novox/hq issue 852), with the
|
||||||
|
// SHA-256 its caller says it has; commitAsk reads it into Patch.
|
||||||
|
PatchFile string
|
||||||
|
PatchSHA256 string
|
||||||
|
patchFileGiven bool
|
||||||
}
|
}
|
||||||
|
|
||||||
// Committed is what gitea_commit made.
|
// Committed is what gitea_commit made.
|
||||||
@@ -67,6 +74,11 @@ func readCommitAsk(args map[string]any) (CommitAsk, error) {
|
|||||||
a := CommitAsk{Owner: text(args, "owner"), Repo: text(args, "repo"), Branch: text(args, "branch"),
|
a := CommitAsk{Owner: text(args, "owner"), Repo: text(args, "repo"), Branch: text(args, "branch"),
|
||||||
Base: text(args, "base"), Message: strings.TrimSpace(raw(args, "message"))}
|
Base: text(args, "base"), Message: strings.TrimSpace(raw(args, "message"))}
|
||||||
a.Patch = raw(args, "patch")
|
a.Patch = raw(args, "patch")
|
||||||
|
if v, ok := args["patch_file"]; ok && v != nil {
|
||||||
|
a.patchFileGiven = true
|
||||||
|
a.PatchFile = raw(args, "patch_file")
|
||||||
|
}
|
||||||
|
a.PatchSHA256 = strings.ToLower(strings.TrimSpace(raw(args, "patch_sha256")))
|
||||||
if a.Branch == "" {
|
if a.Branch == "" {
|
||||||
return a, refuse("branch is required: the branch the commit goes on, a branch of its own (created from base when absent). Nothing was sent")
|
return a, refuse("branch is required: the branch the commit goes on, a branch of its own (created from base when absent). Nothing was sent")
|
||||||
}
|
}
|
||||||
@@ -109,11 +121,23 @@ func readCommitAsk(args map[string]any) (CommitAsk, error) {
|
|||||||
a.Files = append(a.Files, fc)
|
a.Files = append(a.Files, fc)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
sources := 0
|
||||||
|
for _, g := range []bool{len(a.Files) > 0, strings.TrimSpace(a.Patch) != "", a.patchFileGiven} {
|
||||||
|
if g {
|
||||||
|
sources++
|
||||||
|
}
|
||||||
|
}
|
||||||
switch {
|
switch {
|
||||||
case len(a.Files) == 0 && strings.TrimSpace(a.Patch) == "":
|
case sources == 0:
|
||||||
return a, refuse("give files (each {path, content | content_base64 | delete: true}) or patch (a unified diff, as git diff writes it). Nothing was sent")
|
return a, refuse("give files (each {path, content | content_base64 | delete: true}), patch (a unified diff, as git diff writes it) or patch_file (the name of a patch in the spool directory). Nothing was sent")
|
||||||
case len(a.Files) > 0 && strings.TrimSpace(a.Patch) != "":
|
case sources > 1:
|
||||||
return a, refuse("give files or patch, not both: one commit carries one of them. Nothing was sent")
|
return a, refuse("give files, patch or patch_file, not more than one: one commit carries one of them. Nothing was sent")
|
||||||
|
case a.PatchSHA256 != "" && !a.patchFileGiven:
|
||||||
|
return a, refuse("patch_sha256 goes with patch_file only. Nothing was sent")
|
||||||
|
case a.patchFileGiven:
|
||||||
|
if err := checkPatchFileAsk(a); err != nil {
|
||||||
|
return a, err
|
||||||
|
}
|
||||||
case len(a.Files) > maxFiles:
|
case len(a.Files) > maxFiles:
|
||||||
return a, refuse("%d files is more than the %d one commit carries. Nothing was sent", len(a.Files), maxFiles)
|
return a, refuse("%d files is more than the %d one commit carries. Nothing was sent", len(a.Files), maxFiles)
|
||||||
case len(a.Patch) > maxPatchBytes:
|
case len(a.Patch) > maxPatchBytes:
|
||||||
@@ -132,13 +156,13 @@ func readCommitAsk(args map[string]any) (CommitAsk, error) {
|
|||||||
return a, refuse("%s is named twice in files. Nothing was sent", fc.Path)
|
return a, refuse("%s is named twice in files. Nothing was sent", fc.Path)
|
||||||
}
|
}
|
||||||
seen[fc.Path] = true
|
seen[fc.Path] = true
|
||||||
given := 0
|
have := 0
|
||||||
for _, g := range []bool{fc.Content != nil, fc.ContentBase64 != nil, fc.Delete} {
|
for _, g := range []bool{fc.Content != nil, fc.ContentBase64 != nil, fc.Delete} {
|
||||||
if g {
|
if g {
|
||||||
given++
|
have++
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if given != 1 {
|
if have != 1 {
|
||||||
return a, refuse("files[%d] (%s) needs exactly one of content, content_base64 or delete: true. Nothing was sent", i, fc.Path)
|
return a, refuse("files[%d] (%s) needs exactly one of content, content_base64 or delete: true. Nothing was sent", i, fc.Path)
|
||||||
}
|
}
|
||||||
if fc.Content != nil && !utf8.ValidString(*fc.Content) {
|
if fc.Content != nil && !utf8.ValidString(*fc.Content) {
|
||||||
@@ -211,7 +235,32 @@ func commit(ctx context.Context, f *Forge, a CommitAsk) (*Committed, error) {
|
|||||||
// Read the rules, or refuse: a branch the rules could not be read for is never written (fails closed).
|
// Read the rules, or refuse: a branch the rules could not be read for is never written (fails closed).
|
||||||
var rules []Protection
|
var rules []Protection
|
||||||
if err := f.GetJSON(ctx, repo+"/branch_protections", &rules); err != nil {
|
if err := f.GetJSON(ctx, repo+"/branch_protections", &rules); err != nil {
|
||||||
return nil, fmt.Errorf("the protection rules of %s/%s could not be read, so the commit was not judged and nothing was sent: %w", a.Owner, a.Repo, err)
|
var status *forgehttp.StatusError
|
||||||
|
if !errors.As(err, &status) || status.Status != http.StatusForbidden {
|
||||||
|
return nil, fmt.Errorf("the protection rules of %s/%s could not be read, so the commit was not judged and nothing was sent: %w", a.Owner, a.Repo, err)
|
||||||
|
}
|
||||||
|
// The rules are a repository administrator's to read, and the tools' account is none (novox/hq ADR 0365
|
||||||
|
// §2). What it may read is the protection the forge applies to an existing branch: one protected is
|
||||||
|
// refused here. A branch not made yet is covered by no rule this account can see, and the forge itself
|
||||||
|
// judges the commit that makes it against every rule, refusing one a rule guards.
|
||||||
|
if err := targetRefused(a.Branch, r.DefaultBranch, nil); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
b, err := f.branch(ctx, repo, a.Branch)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("whether a protection rule guards %s of %s/%s could not be read, so nothing was sent: %w", a.Branch, a.Owner, a.Repo, err)
|
||||||
|
}
|
||||||
|
if b != nil && b.Protected {
|
||||||
|
rule := b.Rule
|
||||||
|
if rule == "" {
|
||||||
|
rule = a.Branch
|
||||||
|
}
|
||||||
|
rules = []Protection{{RuleName: rule}}
|
||||||
|
if !ruleCovers(rule, a.Branch) {
|
||||||
|
return nil, refuse("a commit on %s is refused: the protection rule %s guards it (novox/hq ADR 0266). Put it on a "+
|
||||||
|
"branch of its own. Nothing was sent", a.Branch, rule)
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
if err := targetRefused(a.Branch, r.DefaultBranch, rules); err != nil {
|
if err := targetRefused(a.Branch, r.DefaultBranch, rules); err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
|
|||||||
@@ -11,8 +11,8 @@ import (
|
|||||||
"gitea/internal/forgehttp"
|
"gitea/internal/forgehttp"
|
||||||
)
|
)
|
||||||
|
|
||||||
// The forge as this bundle reaches it: its own HTTP interface on the machine, as the forge's admin account
|
// The forge as this bundle reaches it: its own HTTP interface on the machine, as the tools' own account (novox/hq
|
||||||
// (internal/forgehttp, shared with the npm-registry bundle beside it).
|
// ADR 0365 §2; internal/forgehttp, shared with the npm-registry bundle beside it).
|
||||||
|
|
||||||
// Forge is the forge's HTTP interface.
|
// Forge is the forge's HTTP interface.
|
||||||
type Forge struct {
|
type Forge struct {
|
||||||
@@ -91,12 +91,15 @@ func (p Protection) name() string {
|
|||||||
return p.BranchName
|
return p.BranchName
|
||||||
}
|
}
|
||||||
|
|
||||||
// Branch is one branch with the commit it points at.
|
// Branch is one branch with the commit it points at, and the protection the forge applies to it: what an account
|
||||||
|
// that may not read the rules themselves (the tools' account, novox/hq ADR 0365) knows of them.
|
||||||
type Branch struct {
|
type Branch struct {
|
||||||
Name string `json:"name"`
|
Name string `json:"name"`
|
||||||
Commit struct {
|
Commit struct {
|
||||||
ID string `json:"id"`
|
ID string `json:"id"`
|
||||||
} `json:"commit"`
|
} `json:"commit"`
|
||||||
|
Protected bool `json:"protected"`
|
||||||
|
Rule string `json:"effective_branch_protection_name"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// branch is the named branch, nil when the forge has none of that name.
|
// branch is the named branch, nil when the forge has none of that name.
|
||||||
|
|||||||
@@ -619,11 +619,17 @@ func TestToolsAreTheOnesTheReadmeNamesAndTheManifestBuilds(t *testing.T) {
|
|||||||
if len(a.Loads) != 1 || a.Loads[0] != a.Binary {
|
if len(a.Loads) != 1 || a.Loads[0] != a.Binary {
|
||||||
t.Errorf("the bundle loads %v, not its binary %s", a.Loads, a.Binary)
|
t.Errorf("the bundle loads %v, not its binary %s", a.Loads, a.Binary)
|
||||||
}
|
}
|
||||||
for _, k := range []string{"MESH_GITEA_URL", "MESH_GITEA_ADMIN_USER", "MESH_GITEA_ADMIN_PASSWORD_FILE"} {
|
// The tools' own account, never the admin login (novox/hq ADR 0365 §2).
|
||||||
|
for _, k := range []string{"MESH_GITEA_URL", "MESH_GITEA_TOOLS_USER", "MESH_GITEA_TOKEN_FILE"} {
|
||||||
if a.Env[k] == "" {
|
if a.Env[k] == "" {
|
||||||
t.Errorf("the bundle is not given %s", k)
|
t.Errorf("the bundle is not given %s", k)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
for _, k := range []string{"MESH_GITEA_ADMIN_USER", "MESH_GITEA_ADMIN_PASSWORD_FILE"} {
|
||||||
|
if a.Env[k] != "" {
|
||||||
|
t.Errorf("the bundle is given %s: it acts as the tools' account, not the administrator", k)
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if !found {
|
if !found {
|
||||||
|
|||||||
@@ -55,7 +55,9 @@ func Tools(forge func() (*Forge, error)) []stdio.Tool {
|
|||||||
"the branch holds them — or patch, a unified diff as git diff writes it, which the forge applies. " +
|
"the branch holds them — or patch, a unified diff as git diff writes it, which the forge applies. " +
|
||||||
"At most 200 files, 4 MiB each and 16 MiB together, or a 4 MiB patch. Answers the commit it made and " +
|
"At most 200 files, 4 MiB each and 16 MiB together, or a 4 MiB patch. Answers the commit it made and " +
|
||||||
"the parent the forge made it on, read back from the branch, and says when the branch moved under " +
|
"the parent the forge made it on, read back from the branch, and says when the branch moved under " +
|
||||||
"it. Replaces git commit and git " +
|
"it. A patch too large to type goes as patch_file with patch_sha256: the name of a file in the " +
|
||||||
|
"spool directory (read once, applied as patch is, removed afterwards, never read from anywhere " +
|
||||||
|
"else). Replaces git commit and git " +
|
||||||
"push of a feature branch. (a)",
|
"push of a feature branch. (a)",
|
||||||
Input: map[string]any{
|
Input: map[string]any{
|
||||||
"type": "object",
|
"type": "object",
|
||||||
@@ -79,7 +81,9 @@ func Tools(forge func() (*Forge, error)) []stdio.Tool {
|
|||||||
"required": []string{"path"},
|
"required": []string{"path"},
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
"patch": str("a unified diff of the change, as git diff or git format-patch writes it; give this or files"),
|
"patch": str("a unified diff of the change, as git diff or git format-patch writes it; give this, patch_file or files"),
|
||||||
|
"patch_file": str("for a patch too large to type: the NAME (32 to 64 characters of 0-9a-f, chosen at random by the caller, no path) of a file the caller wrote into the forge patch spool directory (the module's forge-patches directory); needs patch_sha256; give this, patch or files"),
|
||||||
|
"patch_sha256": str("with patch_file: the file's SHA-256, in hex; the commit is refused, and nothing is sent, when the file's differs"),
|
||||||
},
|
},
|
||||||
"required": []string{"owner", "repo", "branch", "message"},
|
"required": []string{"owner", "repo", "branch", "message"},
|
||||||
},
|
},
|
||||||
@@ -88,7 +92,7 @@ func Tools(forge func() (*Forge, error)) []stdio.Tool {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
return commit(ctx, f, a)
|
return commitAsk(ctx, f, os.Getenv(patchDirEnv), a)
|
||||||
}),
|
}),
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
|
|||||||
@@ -0,0 +1,109 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"crypto/sha256"
|
||||||
|
"encoding/hex"
|
||||||
|
"errors"
|
||||||
|
"io"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"regexp"
|
||||||
|
"strings"
|
||||||
|
"syscall"
|
||||||
|
)
|
||||||
|
|
||||||
|
// A patch too large to type into a call is handed over as a file (novox/hq issue 852). The caller writes it into
|
||||||
|
// one spool directory the module declares (mode 0733: anyone writes, nobody lists) and names it in patch_file,
|
||||||
|
// with the file's SHA-256. The tool runs as the machine's account, so it reads nothing but that directory: the
|
||||||
|
// name is 32 to 64 characters of [0-9a-f], so it holds no separator and no dot; the file must be a regular file
|
||||||
|
// (never a symlink); and its hash must be the caller's, because without the sticky bit another writer could have
|
||||||
|
// replaced it. The file is removed once taken, whether the commit was made or not.
|
||||||
|
|
||||||
|
const patchDirEnv = "MESH_GITEA_PATCH_DIR"
|
||||||
|
|
||||||
|
var (
|
||||||
|
patchFileName = regexp.MustCompile(`^[0-9a-f]{32,64}$`)
|
||||||
|
patchSHA = regexp.MustCompile(`^[0-9a-f]{64}$`)
|
||||||
|
)
|
||||||
|
|
||||||
|
// checkPatchFileAsk judges the arguments of a patch_file ask, before any file is looked at.
|
||||||
|
func checkPatchFileAsk(a CommitAsk) error {
|
||||||
|
if !patchFileName.MatchString(a.PatchFile) {
|
||||||
|
return refuse("patch_file %q is not a name: 32 to 64 characters of 0-9a-f, no path (the file's name in the spool directory, chosen at random). Nothing was sent", a.PatchFile)
|
||||||
|
}
|
||||||
|
if !patchSHA.MatchString(a.PatchSHA256) {
|
||||||
|
return refuse("patch_file needs patch_sha256: the file's SHA-256, 64 hex characters. Nothing was sent")
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// commitAsk makes the commit the ask names; a patch_file is first read from the spool directory into the patch
|
||||||
|
// and removed afterwards.
|
||||||
|
func commitAsk(ctx context.Context, f *Forge, spool string, a CommitAsk) (*Committed, error) {
|
||||||
|
if !a.patchFileGiven {
|
||||||
|
return commit(ctx, f, a)
|
||||||
|
}
|
||||||
|
if err := checkPatchFileAsk(a); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if spool == "" {
|
||||||
|
return nil, refuse("patch_file is not available: %s is not set, so there is no spool directory. Nothing was sent", patchDirEnv)
|
||||||
|
}
|
||||||
|
path := filepath.Join(spool, a.PatchFile)
|
||||||
|
defer func() {
|
||||||
|
// Whatever happened, the file is not left: removing a symlink removes the link, never its target.
|
||||||
|
if st, err := os.Lstat(path); err == nil && !st.IsDir() {
|
||||||
|
_ = os.Remove(path)
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
body, err := readSpooled(path, a.PatchFile)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
got := sha256.Sum256(body)
|
||||||
|
if hex.EncodeToString(got[:]) != a.PatchSHA256 {
|
||||||
|
return nil, refuse("the file %s is not the patch the caller hashed: its SHA-256 is %s, patch_sha256 says %s. Nothing was sent",
|
||||||
|
a.PatchFile, hex.EncodeToString(got[:]), a.PatchSHA256)
|
||||||
|
}
|
||||||
|
a.Patch = string(body)
|
||||||
|
return commit(ctx, f, a)
|
||||||
|
}
|
||||||
|
|
||||||
|
// readSpooled reads a regular file of at most maxPatchBytes, never through a symlink.
|
||||||
|
func readSpooled(path, name string) ([]byte, error) {
|
||||||
|
st, err := os.Lstat(path)
|
||||||
|
switch {
|
||||||
|
case errors.Is(err, os.ErrNotExist):
|
||||||
|
return nil, refuse("patch_file %s: no such file in the spool directory. Nothing was sent", name)
|
||||||
|
case err != nil:
|
||||||
|
return nil, refuse("patch_file %s cannot be read: %v. Nothing was sent", name, err)
|
||||||
|
case !st.Mode().IsRegular():
|
||||||
|
return nil, refuse("patch_file %s is not a regular file (a symlink or a directory is never followed). Nothing was sent", name)
|
||||||
|
case st.Size() > maxPatchBytes:
|
||||||
|
return nil, refuse("patch_file %s is %s, more than the %s one commit carries: split it into several commits. Nothing was sent", name, mib(int(st.Size())), mib(maxPatchBytes))
|
||||||
|
}
|
||||||
|
fh, err := os.OpenFile(path, os.O_RDONLY|syscall.O_NOFOLLOW|syscall.O_NONBLOCK, 0)
|
||||||
|
if err != nil {
|
||||||
|
return nil, refuse("patch_file %s cannot be opened: %v. Nothing was sent", name, err)
|
||||||
|
}
|
||||||
|
defer fh.Close()
|
||||||
|
fst, err := fh.Stat()
|
||||||
|
if err != nil || !fst.Mode().IsRegular() {
|
||||||
|
return nil, refuse("patch_file %s is not a regular file. Nothing was sent", name)
|
||||||
|
}
|
||||||
|
// A hard link to a file elsewhere is no symlink, but it is not the caller's own file either.
|
||||||
|
if sys, ok := fst.Sys().(*syscall.Stat_t); !ok || sys.Nlink > 1 {
|
||||||
|
return nil, refuse("patch_file %s has more than one hard link: it must be a file of its own. Nothing was sent", name)
|
||||||
|
}
|
||||||
|
body, err := io.ReadAll(io.LimitReader(fh, maxPatchBytes+1))
|
||||||
|
switch {
|
||||||
|
case err != nil:
|
||||||
|
return nil, refuse("patch_file %s cannot be read: %v. Nothing was sent", name, err)
|
||||||
|
case len(body) > maxPatchBytes:
|
||||||
|
return nil, refuse("patch_file %s is more than the %s one commit carries: split it into several commits. Nothing was sent", name, mib(maxPatchBytes))
|
||||||
|
case strings.TrimSpace(string(body)) == "":
|
||||||
|
return nil, refuse("patch_file %s is empty. Nothing was sent", name)
|
||||||
|
}
|
||||||
|
return body, nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,183 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"crypto/sha256"
|
||||||
|
"encoding/hex"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
// novox/hq issue 852: a patch too large to type into a call is handed over as a file in one spool directory.
|
||||||
|
|
||||||
|
const spoolName = "0123456789abcdef0123456789abcdef"
|
||||||
|
const spoolPatch = "diff --git a/README.md b/README.md\n--- a/README.md\n+++ b/README.md\n@@ -1 +1 @@\n-hello\n+hi\n"
|
||||||
|
|
||||||
|
func sum(s string) string { h := sha256.Sum256([]byte(s)); return hex.EncodeToString(h[:]) }
|
||||||
|
|
||||||
|
func spoolWith(t *testing.T, name, content string) string {
|
||||||
|
t.Helper()
|
||||||
|
dir := t.TempDir()
|
||||||
|
if name != "" {
|
||||||
|
if err := os.WriteFile(filepath.Join(dir, name), []byte(content), 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return dir
|
||||||
|
}
|
||||||
|
|
||||||
|
func fileAsk(t *testing.T, name, sha string) CommitAsk {
|
||||||
|
t.Helper()
|
||||||
|
return ask(t, map[string]any{"branch": "fix/9", "patch_file": name, "patch_sha256": sha})
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAPatchFileIsAppliedAsThePatchWouldBeAndTheFileIsGone(t *testing.T) {
|
||||||
|
f := newFake(t)
|
||||||
|
dir := spoolWith(t, spoolName, spoolPatch)
|
||||||
|
got, err := commitAsk(context.Background(), f.serve(), dir, fileAsk(t, spoolName, sum(spoolPatch)))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(f.patches) != 1 || f.patches[0]["content"] != spoolPatch || f.patches[0]["new_branch"] != "fix/9" || got.Commit != "p1" {
|
||||||
|
t.Fatalf("%v %+v", f.patches, got)
|
||||||
|
}
|
||||||
|
if _, err := os.Lstat(filepath.Join(dir, spoolName)); !os.IsNotExist(err) {
|
||||||
|
t.Fatalf("the file is still there: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAPatchFileIsRemovedWhenTheForgeRefusesIt(t *testing.T) {
|
||||||
|
f := newFake(t)
|
||||||
|
dir := spoolWith(t, spoolName, spoolPatch)
|
||||||
|
// the default branch is refused after the file was read
|
||||||
|
a := ask(t, map[string]any{"branch": "main", "patch_file": spoolName, "patch_sha256": sum(spoolPatch)})
|
||||||
|
if _, err := commitAsk(context.Background(), f.serve(), dir, a); err == nil {
|
||||||
|
t.Fatal("taken")
|
||||||
|
}
|
||||||
|
if _, err := os.Lstat(filepath.Join(dir, spoolName)); !os.IsNotExist(err) {
|
||||||
|
t.Fatalf("the file is still there: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAPatchFileThatCannotBeTakenIsRefusedBeforeAnythingIsSent(t *testing.T) {
|
||||||
|
outside := filepath.Join(t.TempDir(), "secret")
|
||||||
|
if err := os.WriteFile(outside, []byte(spoolPatch), 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
type tc struct {
|
||||||
|
name string
|
||||||
|
setup func(dir string)
|
||||||
|
file string
|
||||||
|
sha string
|
||||||
|
want string
|
||||||
|
}
|
||||||
|
for _, c := range []tc{
|
||||||
|
{"wrong sha", func(d string) { _ = os.WriteFile(filepath.Join(d, spoolName), []byte(spoolPatch), 0o600) }, spoolName, sum("other"), "sha256"},
|
||||||
|
{"not found", func(string) {}, spoolName, sum(spoolPatch), "no such file"},
|
||||||
|
{"symlink", func(d string) { _ = os.Symlink(outside, filepath.Join(d, spoolName)) }, spoolName, sum(spoolPatch), "regular file"},
|
||||||
|
{"hard link", func(d string) { _ = os.Link(outside, filepath.Join(d, spoolName)) }, spoolName, sum(spoolPatch), "hard link"},
|
||||||
|
{"directory", func(d string) { _ = os.Mkdir(filepath.Join(d, spoolName), 0o700) }, spoolName, sum(spoolPatch), "regular file"},
|
||||||
|
{"too large", func(d string) {
|
||||||
|
_ = os.WriteFile(filepath.Join(d, spoolName), []byte(strings.Repeat("x", maxPatchBytes+1)), 0o600)
|
||||||
|
}, spoolName, sum(spoolPatch), "MiB"},
|
||||||
|
{"empty", func(d string) { _ = os.WriteFile(filepath.Join(d, spoolName), nil, 0o600) }, spoolName, sum(""), "empty"},
|
||||||
|
} {
|
||||||
|
f := newFake(t)
|
||||||
|
dir := t.TempDir()
|
||||||
|
c.setup(dir)
|
||||||
|
_, err := commitAsk(context.Background(), f.serve(), dir, fileAsk(t, c.file, c.sha))
|
||||||
|
if err == nil || !strings.Contains(err.Error(), c.want) || !strings.Contains(err.Error(), "Nothing was sent") {
|
||||||
|
t.Errorf("%s: %v", c.name, err)
|
||||||
|
}
|
||||||
|
if len(f.patches)+len(f.writes) != 0 {
|
||||||
|
t.Errorf("%s: something was sent", c.name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if b, err := os.ReadFile(outside); err != nil || string(b) != spoolPatch {
|
||||||
|
t.Errorf("the file outside the spool was touched: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAPatchFileAskThatCannotBeReadIsRefused(t *testing.T) {
|
||||||
|
ok := sum(spoolPatch)
|
||||||
|
for name, args := range map[string]map[string]any{
|
||||||
|
"no sha": {"patch_file": spoolName},
|
||||||
|
"sha not hex": {"patch_file": spoolName, "patch_sha256": strings.Repeat("z", 64)},
|
||||||
|
"sha too short": {"patch_file": spoolName, "patch_sha256": ok[:63]},
|
||||||
|
"empty name": {"patch_file": "", "patch_sha256": ok, "files": filesArg()},
|
||||||
|
"dotdot": {"patch_file": "../" + spoolName, "patch_sha256": ok},
|
||||||
|
"slash": {"patch_file": "a/" + spoolName, "patch_sha256": ok},
|
||||||
|
"absolute": {"patch_file": "/etc/passwd", "patch_sha256": ok},
|
||||||
|
"too short": {"patch_file": strings.Repeat("a", 31), "patch_sha256": ok},
|
||||||
|
"too long": {"patch_file": strings.Repeat("a", 65), "patch_sha256": ok},
|
||||||
|
"upper case": {"patch_file": strings.ToUpper(spoolName), "patch_sha256": ok},
|
||||||
|
"not hex": {"patch_file": strings.Repeat("g", 32), "patch_sha256": ok},
|
||||||
|
"with patch": {"patch_file": spoolName, "patch_sha256": ok, "patch": "diff"},
|
||||||
|
"with files": {"patch_file": spoolName, "patch_sha256": ok, "files": filesArg(map[string]any{"path": "x", "content": "x"})},
|
||||||
|
"sha without it": {"patch": "diff", "patch_sha256": ok},
|
||||||
|
} {
|
||||||
|
full := map[string]any{"owner": "acme", "repo": "app", "message": "m", "branch": "b"}
|
||||||
|
for k, v := range args {
|
||||||
|
full[k] = v
|
||||||
|
}
|
||||||
|
_, err := readCommitAsk(full)
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "Nothing was sent") {
|
||||||
|
t.Errorf("%s: %v", name, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// 64 characters is taken; so is an upper-case sha.
|
||||||
|
for _, n := range []string{spoolName, strings.Repeat("ab", 32)} {
|
||||||
|
if _, err := readCommitAsk(map[string]any{"owner": "acme", "repo": "app", "message": "m", "branch": "b",
|
||||||
|
"patch_file": n, "patch_sha256": strings.ToUpper(sum(spoolPatch))}); err != nil {
|
||||||
|
t.Errorf("%s: %v", n, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAPatchFileWithoutASpoolDirectoryIsRefused(t *testing.T) {
|
||||||
|
f := newFake(t)
|
||||||
|
_, err := commitAsk(context.Background(), f.serve(), "", fileAsk(t, spoolName, sum(spoolPatch)))
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "MESH_GITEA_PATCH_DIR") || !strings.Contains(err.Error(), "Nothing was sent") {
|
||||||
|
t.Fatalf("%v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTheManifestDeclaresTheSpoolDirectoryAndGivesItToTheBundle(t *testing.T) {
|
||||||
|
raw, err := os.ReadFile("../../module.json")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var m struct {
|
||||||
|
Resources []struct {
|
||||||
|
ID, Type, Path, Mode, Owner string
|
||||||
|
} `json:"resources"`
|
||||||
|
Build struct {
|
||||||
|
Artifacts []struct {
|
||||||
|
From string `json:"from"`
|
||||||
|
Env map[string]string `json:"env"`
|
||||||
|
} `json:"artifacts"`
|
||||||
|
} `json:"build"`
|
||||||
|
}
|
||||||
|
if err := jsonInto(raw, &m); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
found := false
|
||||||
|
for _, r := range m.Resources {
|
||||||
|
if r.ID == "forge-patches" {
|
||||||
|
found = true
|
||||||
|
if r.Type != "directory" || r.Mode != "0733" || r.Owner != "${machine:account}" || r.Path == "" {
|
||||||
|
t.Errorf("%+v", r)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !found {
|
||||||
|
t.Error("no forge-patches directory")
|
||||||
|
}
|
||||||
|
for _, a := range m.Build.Artifacts {
|
||||||
|
if a.From == "cmd/forge" && a.Env["MESH_GITEA_PATCH_DIR"] != "${dir:forge-patches}" {
|
||||||
|
t.Errorf("env %v", a.Env)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
+24
-28
@@ -42,9 +42,9 @@ import { commitStatusTool } from "./delivery.js";
|
|||||||
import { mergeTool } from "./merge.js";
|
import { mergeTool } from "./merge.js";
|
||||||
import { GiteaAdmin, GiteaClient, movedSince } from "./client.js";
|
import { GiteaAdmin, GiteaClient, movedSince } from "./client.js";
|
||||||
import { CHECK_CONTEXT, REPO_CHECK_CONTEXT, commentFor, headsToAnnounce, statusesFor, type Announced, type Checked,
|
import { CHECK_CONTEXT, REPO_CHECK_CONTEXT, commentFor, headsToAnnounce, statusesFor, type Announced, type Checked,
|
||||||
type RepoCheckFacts, reposToLook } from "./pulls.js";
|
repoCheckFacts, reposToLook } from "./pulls.js";
|
||||||
import { GateWriter, clonedAs, type BuildStarted } from "./gate.js";
|
import { GateWriter, clonedAs, type BuildStarted } from "./gate.js";
|
||||||
import { HOOK_PATH, HOOK_PORT, Pace, claimHead, ensureSystemHook, keptSecret, receiver, type ForgeEvent } from "./webhook.js";
|
import { HOOK_PATH, HOOK_PORT, Pace, claimHead, ensureSystemHook, eventsTool, keptSecret, receiver, type ForgeEvent } from "./webhook.js";
|
||||||
import { announcementOf, keepTook, lateVerdictComment, outOfDateWords, sameCommit, tookFile, weighTrunks } from "./trunk.js";
|
import { announcementOf, keepTook, lateVerdictComment, outOfDateWords, sameCommit, tookFile, weighTrunks } from "./trunk.js";
|
||||||
|
|
||||||
// Without a way to a token — configured, or mintable with the admin account (token.ts) — there is
|
// Without a way to a token — configured, or mintable with the admin account (token.ts) — there is
|
||||||
@@ -400,22 +400,6 @@ async function setVerdict(client: GiteaClient, event: { body: unknown }): Promis
|
|||||||
console.log(`[gitea] ${c.owner}/${c.repo}#${c.number ?? "?"} at ${c.commit.slice(0, 8)}: merge check ${c.verdict}`);
|
console.log(`[gitea] ${c.owner}/${c.repo}#${c.number ?? "?"} at ${c.commit.slice(0, 8)}: merge check ${c.verdict}`);
|
||||||
}
|
}
|
||||||
|
|
||||||
// **What only the forge knows of a repository check said as a warning** (novox/hq issue 293): whether the
|
|
||||||
// head holds a merge-check.sh at all, and — when it does not — whether the base branch's protection
|
|
||||||
// requires mesh/repo-check. Asked only for a warning; unknown is left undefined, which statusesFor reads
|
|
||||||
// as possibly required: a failure on a status nothing requires blocks nothing, a success on one that is
|
|
||||||
// required would let an untested repository merge.
|
|
||||||
async function repoCheckFacts(client: GiteaClient, c: Checked, base: string | undefined): Promise<RepoCheckFacts> {
|
|
||||||
if (c["repo-check"]?.verdict !== "warning") return {};
|
|
||||||
const defined = await client.holdsFile(c.owner, c.repo, c.commit, "merge-check.sh").catch(() => undefined);
|
|
||||||
if (defined !== false) return { defined };
|
|
||||||
const rules = await client.branchProtections(c.owner, c.repo).catch(() => undefined);
|
|
||||||
if (!rules) return { defined };
|
|
||||||
const rule = rules.find((p) => (p.rule_name ?? p.branch_name) === (base || "main"));
|
|
||||||
const required = !!rule?.enable_status_check && (rule.status_check_contexts ?? []).includes(REPO_CHECK_CONTEXT);
|
|
||||||
return { defined, required };
|
|
||||||
}
|
|
||||||
|
|
||||||
// **This module is the one writer of a head's mesh/merge-gate** (novox/hq issue 446): the announcement's pending words,
|
// **This module is the one writer of a head's mesh/merge-gate** (novox/hq issue 446): the announcement's pending words,
|
||||||
// where the check is when a build machine takes it, and the verdict — each head's writes in one chain, the verdict last
|
// where the check is when a build machine takes it, and the verdict — each head's writes in one chain, the verdict last
|
||||||
// (gate.ts). A second writer elsewhere raced the verdict and could leave a pass reading pending.
|
// (gate.ts). A second writer elsewhere raced the verdict and could leave a pass reading pending.
|
||||||
@@ -429,10 +413,16 @@ const gates = new GateWriter({
|
|||||||
// **The status tool is served here, beside the writer** (novox/hq issue 456): the runtime starts each bundle as its own
|
// **The status tool is served here, beside the writer** (novox/hq issue 456): the runtime starts each bundle as its own
|
||||||
// process, so a recheck's pending written from the tools' bundle bypassed this writer's chain. Without a forge to
|
// process, so a recheck's pending written from the tools' bundle bypassed this writer's chain. Without a forge to
|
||||||
// reach, nothing is served, as the tools' bundle serves nothing then.
|
// reach, nothing is served, as the tools' bundle serves nothing then.
|
||||||
registerModuleTools("gitea", () => (gitea ? [commitStatusTool({
|
/** How fast the polls look: 30 s until the forge's events are heard, five minutes while they are; and what
|
||||||
|
* gitea_events says of them (webhook.ts). */
|
||||||
|
const pace = new Pace();
|
||||||
|
if (!gitea) pace.unheard("the forge is not watched: no way to a token", false);
|
||||||
|
|
||||||
|
// gitea_events is served whether or not the forge is watched: it says so (novox/hq issue 729).
|
||||||
|
registerModuleTools("gitea", () => [eventsTool(pace), ...(gitea ? [commitStatusTool({
|
||||||
setCommitStatus: (owner, repo, sha, status) => (gitea as GiteaClient).setCommitStatus(owner, repo, sha, status),
|
setCommitStatus: (owner, repo, sha, status) => (gitea as GiteaClient).setCommitStatus(owner, repo, sha, status),
|
||||||
gates,
|
gates,
|
||||||
}), mergeTool(gitea, { gates })] : []));
|
}), mergeTool(gitea, { gates })] : [])]);
|
||||||
|
|
||||||
// **A build machine took an ask** (the build seat's `started`, novox/hq ADR 0157): said on the head it checks, when
|
// **A build machine took an ask** (the build seat's `started`, novox/hq ADR 0157): said on the head it checks, when
|
||||||
// the head is an open pull request's this module announced. Any other build — a branch, a merge's — says nothing.
|
// the head is an open pull request's this module announced. Any other build — a branch, a merge's — says nothing.
|
||||||
@@ -451,9 +441,6 @@ async function buildStarted(event: { body: unknown }): Promise<void> {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/** How fast the polls look: 30 s until the forge's events are heard, five minutes while they are (webhook.ts). */
|
|
||||||
const pace = new Pace();
|
|
||||||
|
|
||||||
/** What the forge's event asks of the announcer at once (issue 729): a new head is announced, and a merge. */
|
/** What the forge's event asks of the announcer at once (issue 729): a new head is announced, and a merge. */
|
||||||
async function actOn(client: GiteaClient, event: ForgeEvent): Promise<void> {
|
async function actOn(client: GiteaClient, event: ForgeEvent): Promise<void> {
|
||||||
const b = event.body as { owner?: string; repo?: string; number?: number };
|
const b = event.body as { owner?: string; repo?: string; number?: number };
|
||||||
@@ -476,23 +463,32 @@ async function listenForForge(client: GiteaClient): Promise<boolean> {
|
|||||||
const url = process.env.MESH_GITEA_HOOK_URL;
|
const url = process.env.MESH_GITEA_HOOK_URL;
|
||||||
if (!stateDir || !url) {
|
if (!stateDir || !url) {
|
||||||
console.log("[gitea] no webhook receiver: MESH_GITEA_STATE_DIR or MESH_GITEA_HOOK_URL unset, so the polls alone watch the forge");
|
console.log("[gitea] no webhook receiver: MESH_GITEA_STATE_DIR or MESH_GITEA_HOOK_URL unset, so the polls alone watch the forge");
|
||||||
|
pace.unheard("MESH_GITEA_STATE_DIR or MESH_GITEA_HOOK_URL unset", false);
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
try {
|
try {
|
||||||
const secret = keptSecret(join(stateDir, "webhook.secret"));
|
const secret = keptSecret(join(stateDir, "webhook.secret"));
|
||||||
const port = Number(process.env.MESH_GITEA_HOOK_PORT ?? HOOK_PORT);
|
const port = Number(process.env.MESH_GITEA_HOOK_PORT ?? HOOK_PORT);
|
||||||
const server = receiver({ secret, port, heard: () => pace.heard(),
|
const host = process.env.MESH_GITEA_HOOK_HOST ?? "0.0.0.0";
|
||||||
|
const server = receiver({ secret, port, heard: (kind, delivery) => pace.heard(kind, delivery),
|
||||||
|
refused: (reason, from) => pace.refused(reason, from),
|
||||||
publish: (event) => emit(event.name, event.body),
|
publish: (event) => emit(event.name, event.body),
|
||||||
act: (event) => actOn(client, event) });
|
act: (event) => actOn(client, event) });
|
||||||
await new Promise<void>((resolve, reject) => {
|
await new Promise<void>((resolve, reject) => {
|
||||||
server.once("error", reject);
|
server.once("error", reject);
|
||||||
server.listen(port, process.env.MESH_GITEA_HOOK_HOST ?? "0.0.0.0", () => resolve());
|
server.listen(port, host, () => resolve());
|
||||||
});
|
});
|
||||||
const did = await ensureSystemHook(GiteaAdmin.fromEnv(), url.replace(/\/+$/, "") + HOOK_PATH, secret);
|
const hook = await ensureSystemHook(GiteaAdmin.fromEnv(), url.replace(/\/+$/, "") + HOOK_PATH, secret);
|
||||||
console.log(`[gitea] listening for the forge's events on port ${port}; its system webhook ${did}, every event, signed`);
|
pace.listening(host, port, hook.url, hook);
|
||||||
|
console.log(`[gitea] listening for the forge's events on ${host}:${port}; its system webhook ${hook.id} ${hook.did} ` +
|
||||||
|
`at ${hook.url} (in place of ${hook.removed.system} system, ${hook.removed.default} default and ` +
|
||||||
|
`${hook.removed.repository} repository webhook(s)${hook.removed.repositories.length ? ` in ${hook.removed.repositories.join(", ")}` : ""}), ` +
|
||||||
|
`${hook.events.length} event families, signed`);
|
||||||
return true;
|
return true;
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
console.error(`[gitea] the forge's events are not heard, so the polls keep their pace — ${err instanceof Error ? err.message : err}`);
|
const why = err instanceof Error ? err.message : String(err);
|
||||||
|
pace.unheard(why);
|
||||||
|
console.error(`[gitea] the forge's events are not heard, so the polls keep their pace — ${why}`);
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,7 +1,17 @@
|
|||||||
// Package forgehttp is the forge's HTTP interface as this module's Go bundles reach it (cmd/forge,
|
// Package forgehttp is the forge's HTTP interface as this module's Go bundles reach it (cmd/forge,
|
||||||
// cmd/npm-registry): the forge on its own machine, as its admin account, whose password the vault delivered
|
// cmd/npm-registry), on the forge's own machine. One copy for both bundles (novox/hq issue 539).
|
||||||
// into a file this module owns (novox/hq ADR 0086), by basic authentication. The password is read when a call
|
//
|
||||||
// is made and never logged, answered or put in an error. One copy for both bundles (novox/hq issue 539).
|
// Two ways in, by what the manifest hands the bundle:
|
||||||
|
//
|
||||||
|
// - **as the tools' account** (novox/hq ADR 0365 §2), when MESH_GITEA_TOKEN_FILE names the token the module
|
||||||
|
// keeps for it (token.ts mints it, for `mesh-tools`, and keeps it at 0600 with its scopes line beside it). A
|
||||||
|
// token whose scopes line does not say it is the tools' account's — one kept from before, the administrator's —
|
||||||
|
// is never sent: the call fails, saying so, until the module has minted the account's own. This bundle mints
|
||||||
|
// nothing; a token the forge rejects is renewed by the module's next call, and said here.
|
||||||
|
// - **as the admin account**, whose password the vault delivered into a file this module owns (novox/hq ADR
|
||||||
|
// 0086), by basic authentication, when no token file is named.
|
||||||
|
//
|
||||||
|
// A credential is read when a call is made and never logged, answered or put in an error.
|
||||||
package forgehttp
|
package forgehttp
|
||||||
|
|
||||||
import (
|
import (
|
||||||
@@ -25,12 +35,31 @@ type Client struct {
|
|||||||
URL string // e.g. http://127.0.0.1:3000
|
URL string // e.g. http://127.0.0.1:3000
|
||||||
User string
|
User string
|
||||||
PasswordFile string
|
PasswordFile string
|
||||||
HTTP *http.Client // when nil, a client with Timeout
|
// TokenFile, when set, is the tools' account's token (MESH_GITEA_TOKEN_FILE), sent instead of the admin login;
|
||||||
Timeout time.Duration // one request's limit when HTTP is nil; 20 s when zero
|
// Account the account its scopes line must name (MESH_GITEA_TOOLS_USER, default mesh-tools).
|
||||||
|
TokenFile string
|
||||||
|
Account string
|
||||||
|
HTTP *http.Client // when nil, a client with Timeout
|
||||||
|
Timeout time.Duration // one request's limit when HTTP is nil; 20 s when zero
|
||||||
}
|
}
|
||||||
|
|
||||||
// FromEnv is the forge as the manifest hands it to a bundle, and the names of the variables it left unset.
|
// ToolsAccount is the account the forge's tools act as when the manifest names none (token.ts, tools-account.ts).
|
||||||
|
const ToolsAccount = "mesh-tools"
|
||||||
|
|
||||||
|
// FromEnv is the forge as the manifest hands it to a bundle, and the names of the variables it left unset. A bundle
|
||||||
|
// handed MESH_GITEA_TOKEN_FILE acts as the tools' account and needs no admin login.
|
||||||
func FromEnv() (Client, []string) {
|
func FromEnv() (Client, []string) {
|
||||||
|
if file := os.Getenv("MESH_GITEA_TOKEN_FILE"); file != "" {
|
||||||
|
c := Client{URL: strings.TrimRight(os.Getenv("MESH_GITEA_URL"), "/"), TokenFile: file,
|
||||||
|
Account: os.Getenv("MESH_GITEA_TOOLS_USER")}
|
||||||
|
if c.Account == "" {
|
||||||
|
c.Account = ToolsAccount
|
||||||
|
}
|
||||||
|
if c.URL == "" {
|
||||||
|
return c, []string{"MESH_GITEA_URL"}
|
||||||
|
}
|
||||||
|
return c, nil
|
||||||
|
}
|
||||||
c := Client{
|
c := Client{
|
||||||
URL: strings.TrimRight(os.Getenv("MESH_GITEA_URL"), "/"),
|
URL: strings.TrimRight(os.Getenv("MESH_GITEA_URL"), "/"),
|
||||||
User: os.Getenv("MESH_GITEA_ADMIN_USER"),
|
User: os.Getenv("MESH_GITEA_ADMIN_USER"),
|
||||||
@@ -80,14 +109,45 @@ func (c *Client) password() (string, error) {
|
|||||||
return strings.TrimSpace(string(raw)), nil
|
return strings.TrimSpace(string(raw)), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// token is the tools' account's token, refused unless its scopes line names that account.
|
||||||
|
func (c *Client) token() (string, error) {
|
||||||
|
raw, err := os.ReadFile(c.TokenFile)
|
||||||
|
if err != nil {
|
||||||
|
return "", fmt.Errorf("the tools' token is not kept yet (the module mints it for %s at its first call): %w", c.Account, err)
|
||||||
|
}
|
||||||
|
line, err := os.ReadFile(c.TokenFile + ".scopes")
|
||||||
|
if err != nil || !strings.HasSuffix(strings.TrimSpace(string(line)), " for "+c.Account) {
|
||||||
|
return "", fmt.Errorf("the token kept at %s is not %s's (novox/hq ADR 0365): the module mints that account's own at its next call; nothing was sent", c.TokenFile, c.Account)
|
||||||
|
}
|
||||||
|
token := strings.TrimSpace(string(raw))
|
||||||
|
if token == "" {
|
||||||
|
return "", fmt.Errorf("the token kept at %s is empty; nothing was sent", c.TokenFile)
|
||||||
|
}
|
||||||
|
return token, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// authorize puts the bundle's credential on a request: the tools' token, or the admin login.
|
||||||
|
func (c *Client) authorize(req *http.Request) error {
|
||||||
|
if c.TokenFile != "" {
|
||||||
|
token, err := c.token()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
req.Header.Set("Authorization", "token "+token)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
password, err := c.password()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
req.SetBasicAuth(c.User, password)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
// Do sends one request, with payload as its JSON body when not nil, and answers the body, read up to limit;
|
// Do sends one request, with payload as its JSON body when not nil, and answers the body, read up to limit;
|
||||||
// 404 is ErrNotFound. One byte past the limit is read, so a body the limit cut is told from one that fits it
|
// 404 is ErrNotFound. One byte past the limit is read, so a body the limit cut is told from one that fits it
|
||||||
// exactly: a cut body is refused, never read as if it were whole.
|
// exactly: a cut body is refused, never read as if it were whole.
|
||||||
func (c *Client) Do(ctx context.Context, method, path string, payload any, limit int64) ([]byte, error) {
|
func (c *Client) Do(ctx context.Context, method, path string, payload any, limit int64) ([]byte, error) {
|
||||||
password, err := c.password()
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
var body io.Reader
|
var body io.Reader
|
||||||
if payload != nil {
|
if payload != nil {
|
||||||
raw, err := json.Marshal(payload)
|
raw, err := json.Marshal(payload)
|
||||||
@@ -100,7 +160,9 @@ func (c *Client) Do(ctx context.Context, method, path string, payload any, limit
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
req.SetBasicAuth(c.User, password)
|
if err := c.authorize(req); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
req.Header.Set("Accept", "application/json")
|
req.Header.Set("Accept", "application/json")
|
||||||
if payload != nil {
|
if payload != nil {
|
||||||
req.Header.Set("Content-Type", "application/json")
|
req.Header.Set("Content-Type", "application/json")
|
||||||
@@ -118,6 +180,9 @@ func (c *Client) Do(ctx context.Context, method, path string, payload any, limit
|
|||||||
return nil, fmt.Errorf("the forge's answer to %s %s is larger than %d MiB and was not read whole", method, path, limit>>20)
|
return nil, fmt.Errorf("the forge's answer to %s %s is larger than %d MiB and was not read whole", method, path, limit>>20)
|
||||||
}
|
}
|
||||||
switch {
|
switch {
|
||||||
|
case res.StatusCode == http.StatusUnauthorized && c.TokenFile != "":
|
||||||
|
return nil, fmt.Errorf("the forge rejected the tools' token for %s (401): the module mints it afresh at its next call; nothing was done: %w",
|
||||||
|
method+" "+path, &StatusError{method, path, res.StatusCode, "unauthorized"})
|
||||||
case res.StatusCode == http.StatusNotFound:
|
case res.StatusCode == http.StatusNotFound:
|
||||||
return nil, fmt.Errorf("%s %s: %w", method, path, ErrNotFound)
|
return nil, fmt.Errorf("%s %s: %w", method, path, ErrNotFound)
|
||||||
case res.StatusCode >= 300:
|
case res.StatusCode >= 300:
|
||||||
|
|||||||
@@ -54,9 +54,10 @@ export function mergeTool(gitea: GiteaClient, given: MergeDeps): ToolDefinition
|
|||||||
const method = args.method ? String(args.method) : "merge";
|
const method = args.method ? String(args.method) : "merge";
|
||||||
if (!MERGE_METHODS.has(method)) throw new Refused(`${method} is not a merge method: ${[...MERGE_METHODS].join(", ")}`);
|
if (!MERGE_METHODS.has(method)) throw new Refused(`${method} is not a merge method: ${[...MERGE_METHODS].join(", ")}`);
|
||||||
const deleteBranch = args.delete_branch === undefined ? true : Boolean(args.delete_branch);
|
const deleteBranch = args.delete_branch === undefined ? true : Boolean(args.delete_branch);
|
||||||
// **Only a head whose required statuses all succeeded** (novox/hq ADR 0266): read here rather than left
|
// **Only a head whose required statuses all succeeded** (novox/hq ADR 0266): read here as well as by the
|
||||||
// to the forge, whose administrator — the account this tool's token is minted for — may merge past a
|
// forge, which holds the tools' own account to it (novox/hq ADR 0365 §2). The protection is the one the forge
|
||||||
// rule that lets it. A rule that cannot be read is no rule passed.
|
// applies to the base, read from the branch: the rules' own list is a repository administrator's. A rule that
|
||||||
|
// cannot be read is no rule passed.
|
||||||
// **Only the head that was reviewed** (novox/hq issue 381): mesh-controller#197 was reviewed at 065cfa0d,
|
// **Only the head that was reviewed** (novox/hq issue 381): mesh-controller#197 was reviewed at 065cfa0d,
|
||||||
// force-pushed to 3ced96f, checked green there, and merged at 3ced96f, because this tool merged whatever the
|
// force-pushed to 3ced96f, checked green there, and merged at 3ced96f, because this tool merged whatever the
|
||||||
// head was and nothing told it which head the review had approved. The caller names it; a head that moved
|
// head was and nothing told it which head the review had approved. The caller names it; a head that moved
|
||||||
@@ -80,7 +81,7 @@ export function mergeTool(gitea: GiteaClient, given: MergeDeps): ToolDefinition
|
|||||||
throw new Refused(`${owner}/${repo}#${number} is not merged: its head is ${pull.head_sha.slice(0, 12)}, not the ` +
|
throw new Refused(`${owner}/${repo}#${number} is not merged: its head is ${pull.head_sha.slice(0, 12)}, not the ` +
|
||||||
`${named} you named. If you reviewed an earlier head, review the one it has now, then name it. Nothing was merged`);
|
`${named} you named. If you reviewed an earlier head, review the one it has now, then name it. Nothing was merged`);
|
||||||
}
|
}
|
||||||
const rule = await gitea.branchProtection(owner, repo, pull.base);
|
const rule = await gitea.appliedProtection(owner, repo, pull.base);
|
||||||
// Only into a branch whose protection requires the checks: a pull request whose base was moved to a
|
// Only into a branch whose protection requires the checks: a pull request whose base was moved to a
|
||||||
// branch nothing guards would merge past every check (novox/hq ADR 0266).
|
// branch nothing guards would merge past every check (novox/hq ADR 0266).
|
||||||
if (!rule || !rule.enable_status_check || !(rule.status_check_contexts ?? []).filter(Boolean).length) {
|
if (!rule || !rule.enable_status_check || !(rule.status_check_contexts ?? []).filter(Boolean).length) {
|
||||||
|
|||||||
@@ -159,8 +159,8 @@
|
|||||||
"settings": {
|
"settings": {
|
||||||
"ready-authors": {
|
"ready-authors": {
|
||||||
"kind": "preference",
|
"kind": "preference",
|
||||||
"default": "mesh-admin jschoubben",
|
"default": "mesh-tools mesh-admin jschoubben",
|
||||||
"why": "the forge accounts whose comment \"ready at <full head sha>\" marks a pull request ready for mesh-delivery to merge, separated by spaces: the mesh's own forge account and the operator's (novox/hq issue 722). A comment by any other account is ignored and said in the forge's log. Trusted: who may have a change merged is the operator's word"
|
"why": "the forge accounts whose comment \"ready at <full head sha>\" marks a pull request ready for mesh-delivery to merge, separated by spaces: the account the forge's tools act as (novox/hq ADR 0365), the admin account for marks made before the tools acted as their own, and the operator's (novox/hq issue 722). A comment by any other account is ignored and said in the forge's log. Trusted: who may have a change merged is the operator's word"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"resources": [
|
"resources": [
|
||||||
@@ -184,6 +184,13 @@
|
|||||||
"trusted": true,
|
"trusted": true,
|
||||||
"content": "# The accounts whose \"ready at <sha>\" comment marks a pull request ready (novox/hq issue 722), from the setting\n# ready-authors, written by the mesh. Replaced on every push; change the setting, never this file.\nauthors=${setting:ready-authors}\n"
|
"content": "# The accounts whose \"ready at <sha>\" comment marks a pull request ready (novox/hq issue 722), from the setting\n# ready-authors, written by the mesh. Replaced on every push; change the setting, never this file.\nauthors=${setting:ready-authors}\n"
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
"id": "forge-patches",
|
||||||
|
"type": "directory",
|
||||||
|
"path": "/var/lib/mesh/forge-patches",
|
||||||
|
"mode": "0733",
|
||||||
|
"owner": "${machine:account}"
|
||||||
|
},
|
||||||
{
|
{
|
||||||
"id": "state",
|
"id": "state",
|
||||||
"type": "directory",
|
"type": "directory",
|
||||||
@@ -308,6 +315,7 @@
|
|||||||
"MESH_GITEA_CONFIG_FILE": "${dir:mesh-state}/config.json",
|
"MESH_GITEA_CONFIG_FILE": "${dir:mesh-state}/config.json",
|
||||||
"MESH_GITEA_ADMIN_USER": "mesh-admin",
|
"MESH_GITEA_ADMIN_USER": "mesh-admin",
|
||||||
"MESH_GITEA_ADMIN_PASSWORD_FILE": "${dir:state}/admin.secret",
|
"MESH_GITEA_ADMIN_PASSWORD_FILE": "${dir:state}/admin.secret",
|
||||||
|
"MESH_GITEA_TOOLS_USER": "mesh-tools",
|
||||||
"MESH_GITEA_STATE_DIR": "${dir:runtime-state}",
|
"MESH_GITEA_STATE_DIR": "${dir:runtime-state}",
|
||||||
"MESH_RECEIVES": "${dir:grants}/npm.json",
|
"MESH_RECEIVES": "${dir:grants}/npm.json",
|
||||||
"MESH_GITEA_READY_AUTHORS_FILE": "${dir:mesh-state}/ready-authors",
|
"MESH_GITEA_READY_AUTHORS_FILE": "${dir:mesh-state}/ready-authors",
|
||||||
@@ -345,8 +353,9 @@
|
|||||||
],
|
],
|
||||||
"env": {
|
"env": {
|
||||||
"MESH_GITEA_URL": "http://127.0.0.1:${port:3000}",
|
"MESH_GITEA_URL": "http://127.0.0.1:${port:3000}",
|
||||||
"MESH_GITEA_ADMIN_USER": "mesh-admin",
|
"MESH_GITEA_TOOLS_USER": "mesh-tools",
|
||||||
"MESH_GITEA_ADMIN_PASSWORD_FILE": "${dir:state}/admin.secret"
|
"MESH_GITEA_TOKEN_FILE": "${dir:runtime-state}/token",
|
||||||
|
"MESH_GITEA_PATCH_DIR": "${dir:forge-patches}"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
|
|||||||
+19
-1
@@ -9,7 +9,7 @@
|
|||||||
//
|
//
|
||||||
// Pure functions here, so they are tested without a forge; index.ts does the asking and the setting.
|
// Pure functions here, so they are tested without a forge; index.ts does the asking and the setting.
|
||||||
|
|
||||||
import type { CommitStatus, GiteaPull, GiteaRepo } from "./client.js";
|
import type { CommitStatus, GiteaClient, GiteaPull, GiteaRepo } from "./client.js";
|
||||||
import { movedSince } from "./client.js";
|
import { movedSince } from "./client.js";
|
||||||
|
|
||||||
/** The status context a merge check's gate is kept under: one per commit, the newest replacing the last. */
|
/** The status context a merge check's gate is kept under: one per commit, the newest replacing the last. */
|
||||||
@@ -176,6 +176,24 @@ export interface RepoCheckFacts {
|
|||||||
required?: boolean;
|
required?: boolean;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// **What only the forge knows of a repository check said as a warning** (novox/hq issue 293): whether the
|
||||||
|
// head holds a merge-check.sh at all, and — when it does not — whether the base branch's protection
|
||||||
|
// requires mesh/repo-check. Asked only for a warning; unknown is left undefined, which statusesFor reads
|
||||||
|
// as possibly required: a failure on a status nothing requires blocks nothing, a success on one that is
|
||||||
|
// required would let an untested repository merge. The base's protection is the one the forge applies, read
|
||||||
|
// from the branch: the rules' own list is a repository administrator's, which the tools' account is not
|
||||||
|
// (novox/hq ADR 0365).
|
||||||
|
export async function repoCheckFacts(
|
||||||
|
forge: Pick<GiteaClient, "holdsFile" | "appliedProtection">, c: Checked, base: string | undefined): Promise<RepoCheckFacts> {
|
||||||
|
if (c["repo-check"]?.verdict !== "warning") return {};
|
||||||
|
const defined = await forge.holdsFile(c.owner, c.repo, c.commit, "merge-check.sh").catch(() => undefined);
|
||||||
|
if (defined !== false) return { defined };
|
||||||
|
const rule = await forge.appliedProtection(c.owner, c.repo, base || "main").catch(() => undefined);
|
||||||
|
if (rule === undefined) return { defined };
|
||||||
|
const required = !!rule?.enable_status_check && (rule.status_check_contexts ?? []).includes(REPO_CHECK_CONTEXT);
|
||||||
|
return { defined, required };
|
||||||
|
}
|
||||||
|
|
||||||
/** The description of a required repository check that is not defined. */
|
/** The description of a required repository check that is not defined. */
|
||||||
export const UNDEFINED_REQUIRED = "fail: mesh/repo-check is required here and this head defines no merge-check.sh — " +
|
export const UNDEFINED_REQUIRED = "fail: mesh/repo-check is required here and this head defines no merge-check.sh — " +
|
||||||
"a person adds one, or lifts the requirement from the branch's protection";
|
"a person adds one, or lifts the requirement from the branch's protection";
|
||||||
|
|||||||
@@ -47,11 +47,18 @@ test("every open pull request is read with what its base requires and what its h
|
|||||||
if (url.startsWith("/api/v1/repos/novox/mesh-sdk/pulls/14")) {
|
if (url.startsWith("/api/v1/repos/novox/mesh-sdk/pulls/14")) {
|
||||||
return send({ number: 14, state: "open", base: { ref: "main" }, head: { ref: "y", sha: "bbb2222" }, html_url: "s" });
|
return send({ number: 14, state: "open", base: { ref: "main" }, head: { ref: "y", sha: "bbb2222" }, html_url: "s" });
|
||||||
}
|
}
|
||||||
if (url.startsWith("/api/v1/repos/novox/hq/branch_protections")) {
|
// The base's protection, read from the branch as the tools' account may (novox/hq ADR 0365); the rules' list is refused it.
|
||||||
return send([{ rule_name: "main", enable_status_check: true, status_check_contexts: ["mesh/repo-check"] }]);
|
if (url.includes("/branch_protections")) {
|
||||||
|
res.statusCode = 403;
|
||||||
|
return res.end(JSON.stringify({ message: "the rules are a repository administrator's" }));
|
||||||
}
|
}
|
||||||
if (url.startsWith("/api/v1/repos/novox/mesh-sdk/branch_protections")) {
|
if (url.startsWith("/api/v1/repos/novox/hq/branches/main")) {
|
||||||
return send([{ rule_name: "main", enable_status_check: true, status_check_contexts: ["mesh/merge-gate", "mesh/repo-check"] }]);
|
return send({ name: "main", protected: true, effective_branch_protection_name: "main", enable_status_check: true,
|
||||||
|
status_check_contexts: ["mesh/repo-check"] });
|
||||||
|
}
|
||||||
|
if (url.startsWith("/api/v1/repos/novox/mesh-sdk/branches/main")) {
|
||||||
|
return send({ name: "main", protected: true, effective_branch_protection_name: "main", enable_status_check: true,
|
||||||
|
status_check_contexts: ["mesh/merge-gate", "mesh/repo-check"] });
|
||||||
}
|
}
|
||||||
if (url.includes("/commits/aaa1111/status")) return send({ statuses: [] });
|
if (url.includes("/commits/aaa1111/status")) return send({ statuses: [] });
|
||||||
if (url.includes("/commits/bbb2222/status")) return send({ statuses: [{ context: "mesh/merge-gate", status: "pending",
|
if (url.includes("/commits/bbb2222/status")) return send({ statuses: [{ context: "mesh/merge-gate", status: "pending",
|
||||||
@@ -251,6 +258,8 @@ test("a long thread hides nothing: every page of the timeline is read", async ()
|
|||||||
test("the accounts that may mark are the setting's, and only the mesh's own when it cannot be read", async () => {
|
test("the accounts that may mark are the setting's, and only the mesh's own when it cannot be read", async () => {
|
||||||
const { readyAuthors } = await import("../client.ts");
|
const { readyAuthors } = await import("../client.ts");
|
||||||
assert.deepEqual(readyAuthors("f", "mesh-admin", () => "# x\nauthors=mesh-admin jschoubben\n"), ["mesh-admin", "jschoubben"]);
|
assert.deepEqual(readyAuthors("f", "mesh-admin", () => "# x\nauthors=mesh-admin jschoubben\n"), ["mesh-admin", "jschoubben"]);
|
||||||
assert.deepEqual(readyAuthors("f", "mesh-admin", () => { throw new Error("gone"); }), ["mesh-admin"]);
|
// The tools' account and the admin's, whose marks were made before ADR 0365 (issue 469).
|
||||||
assert.deepEqual(readyAuthors("f", "mesh-admin", () => "authors=\n"), ["mesh-admin"]);
|
assert.deepEqual(readyAuthors("f", "mesh-admin", () => { throw new Error("gone"); }, "mesh-tools"), ["mesh-tools", "mesh-admin"]);
|
||||||
|
assert.deepEqual(readyAuthors("f", "mesh-admin", () => "authors=\n", "mesh-tools"), ["mesh-tools", "mesh-admin"]);
|
||||||
|
assert.deepEqual(readyAuthors(undefined, "mesh-admin", undefined, "mesh-tools"), ["mesh-tools", "mesh-admin"]);
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -30,6 +30,9 @@ const state = {
|
|||||||
// What the tools said on the bus.
|
// What the tools said on the bus.
|
||||||
emitted: [] as { type: string; body: any }[],
|
emitted: [] as { type: string; body: any }[],
|
||||||
rules: [] as any[],
|
rules: [] as any[],
|
||||||
|
// Whether the token is a repository administrator's. The tools' account is not (novox/hq ADR 0365): the forge
|
||||||
|
// refuses it the rules' list, as gitea's reqAdmin does, and it reads a branch's protection from the branch.
|
||||||
|
admin: false,
|
||||||
lastCommit: "2026-01-01T00:00:00Z",
|
lastCommit: "2026-01-01T00:00:00Z",
|
||||||
// The pull request's head; the head the merge commit joined (the same, unless a test says otherwise) — the forge's
|
// The pull request's head; the head the merge commit joined (the same, unless a test says otherwise) — the forge's
|
||||||
// record of the pull request keeps saying `head` either way, as it would in the race issue 381 is about; and the
|
// record of the pull request keeps saying `head` either way, as it would in the race issue 381 is about; and the
|
||||||
@@ -59,6 +62,7 @@ beforeEach(() => {
|
|||||||
state.trunk = TRUNK;
|
state.trunk = TRUNK;
|
||||||
state.emitted = [];
|
state.emitted = [];
|
||||||
state.rules = [protectedMain()];
|
state.rules = [protectedMain()];
|
||||||
|
state.admin = false;
|
||||||
state.head = REVIEWED;
|
state.head = REVIEWED;
|
||||||
state.landed = "";
|
state.landed = "";
|
||||||
state.merged = false;
|
state.merged = false;
|
||||||
@@ -87,8 +91,18 @@ const server = createServer(async (req, res) => {
|
|||||||
if (method !== "GET") state.sent.push({ method, path, body });
|
if (method !== "GET") state.sent.push({ method, path, body });
|
||||||
const repo = "/repos/novox/mesh-catalog";
|
const repo = "/repos/novox/mesh-catalog";
|
||||||
if (method === "GET" && path === repo) return json(res, 200, { full_name: "novox/mesh-catalog", default_branch: "main" });
|
if (method === "GET" && path === repo) return json(res, 200, { full_name: "novox/mesh-catalog", default_branch: "main" });
|
||||||
|
if (path.startsWith(`${repo}/branch_protections`) && !state.admin) {
|
||||||
|
return json(res, 403, { message: "user should be an owner or a collaborator with admin write of a repository" });
|
||||||
|
}
|
||||||
if (method === "GET" && path === `${repo}/branch_protections`) return json(res, 200, state.rules);
|
if (method === "GET" && path === `${repo}/branch_protections`) return json(res, 200, state.rules);
|
||||||
if (method === "GET" && path === `${repo}/branches/main`) return json(res, 200, { name: "main", commit: { id: state.trunk } });
|
const branch = method === "GET" && path.startsWith(`${repo}/branches/`) ? decodeURIComponent(path.slice(`${repo}/branches/`.length)) : null;
|
||||||
|
if (branch !== null) {
|
||||||
|
// As the forge says it to any reader of the code: the rule that covers the branch, as the forge matches it.
|
||||||
|
const rule = (state.rules as any[]).find((r) => ruleCovers(String(r.rule_name ?? r.branch_name), branch));
|
||||||
|
return json(res, 200, { name: branch, commit: { id: branch === "main" ? state.trunk : "c0ffee" }, protected: !!rule,
|
||||||
|
...(rule ? { effective_branch_protection_name: rule.rule_name ?? rule.branch_name, enable_status_check: !!rule.enable_status_check,
|
||||||
|
status_check_contexts: rule.status_check_contexts ?? [], required_approvals: rule.required_approvals ?? 0, user_can_push: false } : {}) });
|
||||||
|
}
|
||||||
if (method === "GET" && path === `${repo}/pulls/7/files`) {
|
if (method === "GET" && path === `${repo}/pulls/7/files`) {
|
||||||
return json(res, 200, url.searchParams.get("page") === "1" ? [{ filename: "modules/gitea/x.ts", status: "modified" }] : []);
|
return json(res, 200, url.searchParams.get("page") === "1" ? [{ filename: "modules/gitea/x.ts", status: "modified" }] : []);
|
||||||
}
|
}
|
||||||
@@ -199,6 +213,7 @@ test("a protection rule covers a branch as the forge decides it: classes, altern
|
|||||||
});
|
});
|
||||||
|
|
||||||
test("gitea_api still writes what an author writes", async () => {
|
test("gitea_api still writes what an author writes", async () => {
|
||||||
|
state.admin = true; // reads the rules' list, a repository administrator's (novox/hq ADR 0365)
|
||||||
await tool("gitea_api").run({ path: "/repos/novox/mesh-catalog/issues/3/comments", method: "POST", body: { body: "hi" } });
|
await tool("gitea_api").run({ path: "/repos/novox/mesh-catalog/issues/3/comments", method: "POST", body: { body: "hi" } });
|
||||||
await tool("gitea_api").run({ path: "/repos/novox/mesh-catalog/pulls/7/reviews", method: "POST", body: { event: "COMMENT" } });
|
await tool("gitea_api").run({ path: "/repos/novox/mesh-catalog/pulls/7/reviews", method: "POST", body: { event: "COMMENT" } });
|
||||||
await tool("gitea_api").run({ path: "/repos/novox/mesh-catalog/branch_protections" });
|
await tool("gitea_api").run({ path: "/repos/novox/mesh-catalog/branch_protections" });
|
||||||
@@ -206,6 +221,7 @@ test("gitea_api still writes what an author writes", async () => {
|
|||||||
});
|
});
|
||||||
|
|
||||||
test("gitea_branch_protection_set only strengthens", async () => {
|
test("gitea_branch_protection_set only strengthens", async () => {
|
||||||
|
state.admin = true; // a repository administrator's token: the tools' account is refused the rules by the forge
|
||||||
const set = (args: Record<string, unknown>) =>
|
const set = (args: Record<string, unknown>) =>
|
||||||
tool("gitea_branch_protection_set").run({ owner: "novox", repo: "mesh-catalog", branch: "main", ...args });
|
tool("gitea_branch_protection_set").run({ owner: "novox", repo: "mesh-catalog", branch: "main", ...args });
|
||||||
await assert.rejects(set({ status_checks: "" }), /drops the required status mesh\/merge-gate, mesh\/repo-check/);
|
await assert.rejects(set({ status_checks: "" }), /drops the required status mesh\/merge-gate, mesh\/repo-check/);
|
||||||
@@ -396,6 +412,7 @@ test("gitea_merge_pull_request reads back a fast-forward, and says it cannot rea
|
|||||||
});
|
});
|
||||||
|
|
||||||
test("gitea_delete_repo deletes only the tools' own user's repository, unguarded, untouched for 30 days, named in full", async () => {
|
test("gitea_delete_repo deletes only the tools' own user's repository, unguarded, untouched for 30 days, named in full", async () => {
|
||||||
|
state.admin = true; // reads the rules' list, a repository administrator's (novox/hq ADR 0365)
|
||||||
const del = (owner: string, name: string, confirm: unknown) => tool("gitea_delete_repo").run({ owner, name, confirm });
|
const del = (owner: string, name: string, confirm: unknown) => tool("gitea_delete_repo").run({ owner, name, confirm });
|
||||||
await assert.rejects(del("novox", "mesh-catalog", true), /confirm must be the repository's full name/);
|
await assert.rejects(del("novox", "mesh-catalog", true), /confirm must be the repository's full name/);
|
||||||
await assert.rejects(del("novox", "mesh-catalog", "novox/mesh-catalog"), /not a repository of the tools' own user/);
|
await assert.rejects(del("novox", "mesh-catalog", "novox/mesh-catalog"), /not a repository of the tools' own user/);
|
||||||
@@ -427,6 +444,7 @@ test("gitea_merge_pull_request merges only into a branch whose protection requir
|
|||||||
});
|
});
|
||||||
|
|
||||||
test("a rule named for a branch is not made weaker than the glob rule that guards it now", async () => {
|
test("a rule named for a branch is not made weaker than the glob rule that guards it now", async () => {
|
||||||
|
state.admin = true; // reads the rules' list, a repository administrator's (novox/hq ADR 0365)
|
||||||
state.rules.push({ rule_name: "release/*", enable_push: false, enable_status_check: true,
|
state.rules.push({ rule_name: "release/*", enable_push: false, enable_status_check: true,
|
||||||
status_check_contexts: ["mesh/merge-gate"], block_admin_merge_override: true });
|
status_check_contexts: ["mesh/merge-gate"], block_admin_merge_override: true });
|
||||||
const set = (args: Record<string, unknown>) =>
|
const set = (args: Record<string, unknown>) =>
|
||||||
|
|||||||
@@ -0,0 +1,59 @@
|
|||||||
|
// The controller's build-source check asks gitea_branch_protection_get for a repository's rules, which the forge
|
||||||
|
// answers to a repository's administrator alone. The tools' account is not one (novox/hq ADR 0365 §2), so the read
|
||||||
|
// goes with the forge's administrator account, and only the read (ADR 0375, issue 772): a protection edit through the
|
||||||
|
// tools still gets the forge's refusal.
|
||||||
|
|
||||||
|
import { test, after } from "node:test";
|
||||||
|
import assert from "node:assert/strict";
|
||||||
|
import { createServer } from "node:http";
|
||||||
|
|
||||||
|
import { GiteaAdmin, GiteaClient } from "../dist/client.js";
|
||||||
|
import { ConfiguredToken } from "../dist/token.js";
|
||||||
|
import { getGiteaTools } from "../dist/tools/index.js";
|
||||||
|
|
||||||
|
const adminBasic = "Basic " + Buffer.from("mesh-admin:secret").toString("base64");
|
||||||
|
const rules = [
|
||||||
|
{ rule_name: "main", enable_push: false, enable_status_check: true, status_check_contexts: ["mesh/merge-gate"], block_admin_merge_override: true },
|
||||||
|
{ branch_name: "release", enable_push: false },
|
||||||
|
];
|
||||||
|
const seen: string[] = [];
|
||||||
|
|
||||||
|
const server = createServer((req, res) => {
|
||||||
|
const path = (req.url ?? "").replace(/^\/api\/v1/, "");
|
||||||
|
const as = req.headers.authorization === adminBasic ? "admin" : "tools";
|
||||||
|
seen.push(`${req.method} ${path} as ${as}`);
|
||||||
|
const send = (status: number, body: unknown) => { res.writeHead(status, { "Content-Type": "application/json" }); res.end(JSON.stringify(body)); };
|
||||||
|
if (path.includes("/branch_protections")) {
|
||||||
|
if (as !== "admin") return send(403, { message: "user should be an owner or a collaborator with admin write of a repository" });
|
||||||
|
if (req.method === "GET") return send(200, rules);
|
||||||
|
}
|
||||||
|
send(404, { message: "not found" });
|
||||||
|
});
|
||||||
|
await new Promise<void>((r) => server.listen(0, "127.0.0.1", () => r()));
|
||||||
|
after(() => new Promise<void>((r) => server.close(() => r())));
|
||||||
|
const { port } = server.address() as { port: number };
|
||||||
|
const url = `http://127.0.0.1:${port}`;
|
||||||
|
const tools = getGiteaTools(new GiteaClient(url, new ConfiguredToken("tools-token")), () => new GiteaAdmin(url, "mesh-admin", "secret"));
|
||||||
|
const tool = (name: string) => tools.find((t) => t.name === name)!;
|
||||||
|
|
||||||
|
test("gitea_branch_protection_get answers every rule, read with the administrator account the tools' token cannot read with", async () => {
|
||||||
|
const out: any = await tool("gitea_branch_protection_get").run({ owner: "novox", repo: "mesh-controller" });
|
||||||
|
assert.deepEqual(out.rules.map((r: any) => r.rule), ["main", "release"]);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("gitea_branch_protection_get answers the rule for a branch, null when it has none", async () => {
|
||||||
|
const main: any = await tool("gitea_branch_protection_get").run({ owner: "novox", repo: "mesh-controller", branch: "main" });
|
||||||
|
assert.equal(main.branch, "main");
|
||||||
|
assert.equal(main.rule.rule, "main");
|
||||||
|
assert.equal(main.rule.admin_may_override, false);
|
||||||
|
const none: any = await tool("gitea_branch_protection_get").run({ owner: "novox", repo: "mesh-controller", branch: "dev" });
|
||||||
|
assert.equal(none.rule, null);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("gitea_branch_protection_set still gets the forge's refusal, as the tools' account", async () => {
|
||||||
|
await assert.rejects(
|
||||||
|
tool("gitea_branch_protection_set").run({ owner: "novox", repo: "mesh-controller", branch: "main", status_checks: "mesh/merge-gate" }),
|
||||||
|
/403/,
|
||||||
|
);
|
||||||
|
assert.ok(!seen.some((s) => /^(PATCH|POST|PUT|DELETE) .*branch_protections.* as admin$/.test(s)), seen.join("\n"));
|
||||||
|
});
|
||||||
@@ -10,12 +10,12 @@
|
|||||||
import { test, after } from "node:test";
|
import { test, after } from "node:test";
|
||||||
import assert from "node:assert/strict";
|
import assert from "node:assert/strict";
|
||||||
import { createServer, type IncomingMessage, type ServerResponse } from "node:http";
|
import { createServer, type IncomingMessage, type ServerResponse } from "node:http";
|
||||||
import { mkdtemp, readFile, rm, stat, writeFile } from "node:fs/promises";
|
import { mkdir, mkdtemp, readFile, rm, stat, writeFile } from "node:fs/promises";
|
||||||
import { tmpdir } from "node:os";
|
import { tmpdir } from "node:os";
|
||||||
import { join } from "node:path";
|
import { join } from "node:path";
|
||||||
|
|
||||||
import { collectTools } from "@novox/mesh-sdk/tools";
|
import { collectTools } from "@novox/mesh-sdk/tools";
|
||||||
import { AdminRefused, MINT_RETRY_AFTER_MS, MintedToken, TOKEN_SCOPES } from "../dist/token.js";
|
import { AdminRefused, MintedToken, TOKEN_SCOPES } from "../dist/token.js";
|
||||||
import { GiteaClient } from "../dist/client.js";
|
import { GiteaClient } from "../dist/client.js";
|
||||||
import { getGiteaTools } from "../dist/tools/index.js";
|
import { getGiteaTools } from "../dist/tools/index.js";
|
||||||
|
|
||||||
@@ -29,9 +29,12 @@ interface Forge {
|
|||||||
mints: number;
|
mints: number;
|
||||||
lastScopes: string[] | null;
|
lastScopes: string[] | null;
|
||||||
tokens: Map<string, string>;
|
tokens: Map<string, string>;
|
||||||
/** Tokens beyond the one per name `tokens` holds, by id: what two mints racing leave (novox/hq issue 739). */
|
/** Tokens beside `tokens` that carry a name already held there: what two mints at once leave behind. */
|
||||||
twins: Map<number, { name: string; sha1: string }>;
|
twins: { id: number; name: string; sha1: string }[];
|
||||||
tokenCalls: number;
|
/** How each DELETE named its token: "id" or "name". */
|
||||||
|
deletedBy: string[];
|
||||||
|
/** Plant a token by name, beside any other of the same name, as a raced mint would. */
|
||||||
|
plant(name: string): void;
|
||||||
scopesOf: Map<string, string[]>;
|
scopesOf: Map<string, string[]>;
|
||||||
admins: Map<string, string>;
|
admins: Map<string, string>;
|
||||||
pullState: string;
|
pullState: string;
|
||||||
@@ -42,12 +45,14 @@ interface Forge {
|
|||||||
|
|
||||||
function fakeForge(): Promise<Forge> {
|
function fakeForge(): Promise<Forge> {
|
||||||
const forge = {
|
const forge = {
|
||||||
|
mintedFor: "",
|
||||||
mints: 0,
|
mints: 0,
|
||||||
lastScopes: null as string[] | null,
|
lastScopes: null as string[] | null,
|
||||||
tokens: new Map<string, string>(), // name -> value
|
tokens: new Map<string, string>(), // name -> value
|
||||||
twins: new Map<number, { name: string; sha1: string }>(),
|
twins: [] as { id: number; name: string; sha1: string }[],
|
||||||
idOf: new Map<string, number>(), // value -> id
|
deletedBy: [] as string[],
|
||||||
tokenCalls: 0,
|
ids: new Map<string, number>(), // value -> id, as gitea numbers its tokens
|
||||||
|
nextId: 0,
|
||||||
scopesOf: new Map<string, string[]>(), // value -> scopes, so a route can enforce them like gitea does
|
scopesOf: new Map<string, string[]>(), // value -> scopes, so a route can enforce them like gitea does
|
||||||
admins: new Map([[ADMIN, PASSWORD]]),
|
admins: new Map([[ADMIN, PASSWORD]]),
|
||||||
pullState: "open",
|
pullState: "open",
|
||||||
@@ -79,41 +84,51 @@ function fakeForge(): Promise<Forge> {
|
|||||||
const tokens = url.pathname.match(/^\/api\/v1\/users\/([^/]+)\/tokens(?:\/([^/]+))?$/);
|
const tokens = url.pathname.match(/^\/api\/v1\/users\/([^/]+)\/tokens(?:\/([^/]+))?$/);
|
||||||
if (tokens) {
|
if (tokens) {
|
||||||
const user = basic(req);
|
const user = basic(req);
|
||||||
forge.tokenCalls++;
|
// gitea's reqSelfOrAdmin: the account itself, or the administrator for another (the tools' account, ADR 0365).
|
||||||
if (user === null || user !== decodeURIComponent(tokens[1])) return json(res, 401, { message: "auth required" });
|
if (user === null || (user !== decodeURIComponent(tokens[1]) && user !== ADMIN)) return json(res, 401, { message: "auth required" });
|
||||||
const named = (name: string): number =>
|
// This fake keeps one account's tokens: the tools' account's when the module mints for it (ADR 0365), and the
|
||||||
(forge.tokens.has(name) ? 1 : 0) + [...forge.twins.values()].filter((t) => t.name === name).length;
|
// admin's own list is then empty — nothing of it is retired.
|
||||||
|
if (decodeURIComponent(tokens[1]) === ADMIN && forge.mintedFor === "mesh-tools") {
|
||||||
|
return req.method === "GET" ? json(res, 200, []) : json(res, 404, { message: "token not found" });
|
||||||
|
}
|
||||||
|
if (req.method === "POST") forge.mintedFor = decodeURIComponent(tokens[1]);
|
||||||
if (req.method === "POST") {
|
if (req.method === "POST") {
|
||||||
const { name, scopes } = await body(req);
|
const { name, scopes } = await body(req);
|
||||||
if (named(name) > 0) return json(res, 400, { message: "token name has already been used" });
|
if (forge.tokens.has(name)) return json(res, 400, { message: "token name has already been used" });
|
||||||
forge.mints++;
|
forge.mints++;
|
||||||
forge.lastScopes = scopes;
|
forge.lastScopes = scopes;
|
||||||
const sha1 = `minted-${forge.mints}-${Math.random().toString(36).slice(2)}`;
|
const sha1 = `minted-${forge.mints}-${Math.random().toString(36).slice(2)}`;
|
||||||
forge.tokens.set(name, sha1);
|
forge.tokens.set(name, sha1);
|
||||||
forge.idOf.set(sha1, forge.mints);
|
|
||||||
forge.scopesOf.set(sha1, scopes);
|
forge.scopesOf.set(sha1, scopes);
|
||||||
return json(res, 201, { id: forge.mints, name, sha1, scopes, token_last_eight: sha1.slice(-8) });
|
forge.ids.set(sha1, ++forge.nextId);
|
||||||
|
return json(res, 201, { id: forge.ids.get(sha1), name, sha1, scopes, token_last_eight: sha1.slice(-8) });
|
||||||
}
|
}
|
||||||
|
// gitea's list: every token of the account, a page at a time (at most three to a page here, so a
|
||||||
|
// client that reads only the first page misses some), the values never shown.
|
||||||
|
const listed = (): { id: number; name: string; sha1: string }[] => [
|
||||||
|
...[...forge.tokens].map(([name, sha1]) => ({ id: forge.ids.get(sha1) ?? 0, name, sha1 })),
|
||||||
|
...forge.twins,
|
||||||
|
];
|
||||||
if (req.method === "GET" && !tokens[2]) {
|
if (req.method === "GET" && !tokens[2]) {
|
||||||
const page = Number(url.searchParams.get("page") ?? "1");
|
const page = Math.max(1, Number(url.searchParams.get("page") ?? "1"));
|
||||||
const all = [...[...forge.tokens].map(([name, v]) => ({ id: forge.idOf.get(v) ?? 0, name })),
|
const limit = Math.min(3, Math.max(1, Number(url.searchParams.get("limit") ?? "3")));
|
||||||
...[...forge.twins].map(([id, t]) => ({ id, name: t.name }))];
|
const all = listed();
|
||||||
return json(res, 200, page === 1 ? all : []);
|
res.setHeader("X-Total-Count", String(all.length));
|
||||||
|
return json(res, 200, all.slice((page - 1) * limit, page * limit)
|
||||||
|
.map((t) => ({ id: t.id, name: t.name, sha1: "", token_last_eight: t.sha1.slice(-8), scopes: [] })));
|
||||||
}
|
}
|
||||||
if (req.method === "DELETE" && tokens[2]) {
|
if (req.method === "DELETE" && tokens[2]) {
|
||||||
|
// gitea: "identified by ID and if not available by name" — and a name held twice is refused.
|
||||||
const key = decodeURIComponent(tokens[2]);
|
const key = decodeURIComponent(tokens[2]);
|
||||||
// gitea: "identified by ID and if not available by name"
|
const byId = /^\d+$/.test(key);
|
||||||
if (/^\d+$/.test(key)) {
|
const matches = listed().filter((t) => (byId ? String(t.id) === key : t.name === key));
|
||||||
const id = Number(key);
|
if (matches.length === 0) return json(res, 404, { message: "token not found" });
|
||||||
if (forge.twins.delete(id)) return json(res, 204, null);
|
if (matches.length > 1) return json(res, 422, { message: `multiple matches for token name '${key}'` });
|
||||||
const hit = [...forge.tokens].find(([, v]) => forge.idOf.get(v) === id);
|
const t = matches[0];
|
||||||
if (!hit) return json(res, 404, { message: "token not found" });
|
forge.deletedBy.push(byId ? "id" : "name");
|
||||||
forge.tokens.delete(hit[0]);
|
const twin = forge.twins.findIndex((w) => w.id === t.id);
|
||||||
return json(res, 204, null);
|
if (twin >= 0) forge.twins.splice(twin, 1);
|
||||||
}
|
else forge.tokens.delete(t.name);
|
||||||
if (named(key) > 1) return json(res, 422, { message: `multiple matches for token name '${key}'` });
|
|
||||||
if (!forge.tokens.has(key)) return json(res, 404, { message: "token not found" });
|
|
||||||
forge.tokens.delete(key);
|
|
||||||
return json(res, 204, null);
|
return json(res, 204, null);
|
||||||
}
|
}
|
||||||
return json(res, 405, { message: "method not allowed" });
|
return json(res, 405, { message: "method not allowed" });
|
||||||
@@ -179,6 +194,17 @@ function fakeForge(): Promise<Forge> {
|
|||||||
{ full_name: "novox/hq", name: "hq", owner: { login: "novox" }, private: true, html_url: "http://fake/novox/hq" },
|
{ full_name: "novox/hq", name: "hq", owner: { login: "novox" }, private: true, html_url: "http://fake/novox/hq" },
|
||||||
]);
|
]);
|
||||||
}
|
}
|
||||||
|
// What setting up the tools' account asks with the admin login (tools-account.ts, ADR 0365): answered plainly.
|
||||||
|
if (basic(req) === ADMIN) {
|
||||||
|
const p = url.pathname;
|
||||||
|
if (req.method === "GET" && p === "/api/v1/user") return json(res, 200, { login: ADMIN, is_admin: true });
|
||||||
|
if (req.method === "GET" && p === "/api/v1/users/mesh-tools") return json(res, 200, { login: "mesh-tools", is_admin: false });
|
||||||
|
if (req.method === "POST" && p === "/api/v1/admin/users") return json(res, 422, { message: "user already exists" });
|
||||||
|
if (req.method === "PATCH" && p === "/api/v1/admin/users/mesh-tools") return json(res, 200, { login: "mesh-tools" });
|
||||||
|
if (req.method === "GET" && p === "/api/v1/orgs/novox/teams") return json(res, 200, [{ id: 7, name: "mesh-tools" }]);
|
||||||
|
if (req.method === "PATCH" && p === "/api/v1/teams/7") return json(res, 200, { id: 7 });
|
||||||
|
if (req.method === "PUT" && p === "/api/v1/teams/7/members/mesh-tools") return json(res, 204, null);
|
||||||
|
}
|
||||||
const adminUser = url.pathname.match(/^\/api\/v1\/admin\/users\/([^/]+)$/);
|
const adminUser = url.pathname.match(/^\/api\/v1\/admin\/users\/([^/]+)$/);
|
||||||
if (adminUser && req.method === "PATCH") {
|
if (adminUser && req.method === "PATCH") {
|
||||||
const h = req.headers.authorization ?? "";
|
const h = req.headers.authorization ?? "";
|
||||||
@@ -206,9 +232,15 @@ function fakeForge(): Promise<Forge> {
|
|||||||
get pullState() { return forge.pullState; },
|
get pullState() { return forge.pullState; },
|
||||||
get pullTitle() { return forge.pullTitle; },
|
get pullTitle() { return forge.pullTitle; },
|
||||||
get branchDeleted() { return forge.branchDeleted; },
|
get branchDeleted() { return forge.branchDeleted; },
|
||||||
get tokenCalls() { return forge.tokenCalls; },
|
|
||||||
tokens: forge.tokens,
|
tokens: forge.tokens,
|
||||||
twins: forge.twins,
|
twins: forge.twins,
|
||||||
|
deletedBy: forge.deletedBy,
|
||||||
|
plant: (name: string) => {
|
||||||
|
const sha1 = `planted-${++forge.nextId}`;
|
||||||
|
forge.ids.set(sha1, forge.nextId);
|
||||||
|
if (forge.tokens.has(name)) forge.twins.push({ id: forge.nextId, name, sha1 });
|
||||||
|
else forge.tokens.set(name, sha1);
|
||||||
|
},
|
||||||
scopesOf: forge.scopesOf,
|
scopesOf: forge.scopesOf,
|
||||||
admins: forge.admins,
|
admins: forge.admins,
|
||||||
close: () => new Promise((r) => server.close(() => r())),
|
close: () => new Promise((r) => server.close(() => r())),
|
||||||
@@ -237,9 +269,8 @@ async function delivered(forge: Forge): Promise<{ env: NodeJS.ProcessEnv; file:
|
|||||||
}
|
}
|
||||||
|
|
||||||
/** A client as a fresh process would build it: a new source over the kept file, its log captured. */
|
/** A client as a fresh process would build it: a new source over the kept file, its log captured. */
|
||||||
function minted(env: NodeJS.ProcessEnv, logs: string[], now?: () => number): GiteaClient {
|
function minted(env: NodeJS.ProcessEnv, logs: string[]): GiteaClient {
|
||||||
const source = new MintedToken({
|
const source = new MintedToken({
|
||||||
now,
|
|
||||||
url: env.MESH_GITEA_URL!,
|
url: env.MESH_GITEA_URL!,
|
||||||
admin: env.MESH_GITEA_ADMIN_USER!,
|
admin: env.MESH_GITEA_ADMIN_USER!,
|
||||||
passwordFile: env.MESH_GITEA_ADMIN_PASSWORD_FILE!,
|
passwordFile: env.MESH_GITEA_ADMIN_PASSWORD_FILE!,
|
||||||
@@ -354,44 +385,6 @@ test("the kept file is gone but the forge still holds a token by that name: repl
|
|||||||
assert.ok(logs.some((l) => l.includes('already holds a token named "mesh-tools"')), logs.join("\n"));
|
assert.ok(logs.some((l) => l.includes('already holds a token named "mesh-tools"')), logs.join("\n"));
|
||||||
});
|
});
|
||||||
|
|
||||||
test("two tokens carry the name (two mints raced): each is dropped by its id and one is minted, not a 422 for good (issue 739)", async () => {
|
|
||||||
const { env, logs } = await delivered(forge);
|
|
||||||
await minted(env, logs).listRepos();
|
|
||||||
// What two processes minting at once leave: a second token by the same name, whose value nobody kept.
|
|
||||||
// Two tokens by the name, neither of them the one kept.
|
|
||||||
forge.tokens.clear();
|
|
||||||
forge.twins.set(9001, { name: "mesh-tools", sha1: "raced-first" });
|
|
||||||
forge.twins.set(9002, { name: "mesh-tools", sha1: "raced-second" });
|
|
||||||
|
|
||||||
const repos = await minted(env, logs).listRepos();
|
|
||||||
|
|
||||||
assert.equal(repos.length, 1);
|
|
||||||
assert.equal(forge.twins.size, 0);
|
|
||||||
assert.equal([...forge.tokens.keys()].filter((n) => n === "mesh-tools").length, 1);
|
|
||||||
assert.ok(logs.some((l) => l.includes('holds 2 tokens named "mesh-tools"')), logs.join("\n"));
|
|
||||||
});
|
|
||||||
|
|
||||||
test("a mint that failed is not asked of the forge again within the retry window: the same error, no basic-auth call (issue 739)", async () => {
|
|
||||||
const { env, logs } = await delivered(forge);
|
|
||||||
forge.admins.delete(ADMIN);
|
|
||||||
try {
|
|
||||||
let clock = 5_000_000;
|
|
||||||
const client = minted(env, logs, () => clock);
|
|
||||||
await assert.rejects(client.listRepos(), AdminRefused);
|
|
||||||
const calls = forge.tokenCalls;
|
|
||||||
|
|
||||||
await assert.rejects(client.listRepos(), AdminRefused);
|
|
||||||
await assert.rejects(client.listRepos(), AdminRefused);
|
|
||||||
assert.equal(forge.tokenCalls, calls);
|
|
||||||
|
|
||||||
clock += MINT_RETRY_AFTER_MS;
|
|
||||||
await assert.rejects(client.listRepos(), AdminRefused);
|
|
||||||
assert.equal(forge.tokenCalls, calls + 1);
|
|
||||||
} finally {
|
|
||||||
forge.admins.set(ADMIN, PASSWORD);
|
|
||||||
}
|
|
||||||
});
|
|
||||||
|
|
||||||
test("concurrent first calls share one mint", async () => {
|
test("concurrent first calls share one mint", async () => {
|
||||||
const { env, logs } = await delivered(forge);
|
const { env, logs } = await delivered(forge);
|
||||||
const client = minted(env, logs);
|
const client = minted(env, logs);
|
||||||
@@ -402,12 +395,61 @@ test("concurrent first calls share one mint", async () => {
|
|||||||
assert.equal(forge.mints, before + 1);
|
assert.equal(forge.mints, before + 1);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test("two tokens hold the name (two mints raced): both are deleted by their id, and one fresh token is minted", async () => {
|
||||||
|
const { env, file, logs } = await delivered(forge);
|
||||||
|
forge.tokens.clear();
|
||||||
|
forge.twins.length = 0;
|
||||||
|
for (const other of ["laptop", "backup", "ci"]) forge.plant(other); // the account's other tokens: kept
|
||||||
|
forge.plant("mesh-tools");
|
||||||
|
forge.plant("mesh-tools"); // gitea refuses to delete either by name from here on
|
||||||
|
forge.deletedBy.length = 0;
|
||||||
|
const before = forge.mints;
|
||||||
|
|
||||||
|
const repos = await minted(env, logs).listRepos();
|
||||||
|
|
||||||
|
assert.equal(repos.length, 1);
|
||||||
|
assert.equal(forge.mints, before + 1);
|
||||||
|
assert.deepEqual(forge.deletedBy, ["id", "id"]);
|
||||||
|
assert.equal(forge.twins.length, 0);
|
||||||
|
assert.deepEqual([...forge.tokens.keys()].sort(), ["backup", "ci", "laptop", "mesh-tools"]);
|
||||||
|
assert.equal(await readFile(file, "utf8"), forge.tokens.get("mesh-tools") + "\n");
|
||||||
|
assert.ok(logs.some((l) => l.includes('holds 2 tokens named "mesh-tools"')), logs.join("\n"));
|
||||||
|
});
|
||||||
|
|
||||||
|
test("two processes on one kept file, both without a token, mint once: the second waits for the first and reuses it", async () => {
|
||||||
|
const { env, logs } = await delivered(forge);
|
||||||
|
forge.tokens.clear();
|
||||||
|
forge.twins.length = 0;
|
||||||
|
const before = forge.mints;
|
||||||
|
// Two sources over one state directory, as the tools bundle and the watcher are two processes.
|
||||||
|
const first = minted(env, logs);
|
||||||
|
const second = minted(env, logs);
|
||||||
|
|
||||||
|
await Promise.all([first.listRepos(), second.listRepos()]);
|
||||||
|
|
||||||
|
assert.equal(forge.mints, before + 1);
|
||||||
|
assert.equal(forge.twins.length, 0);
|
||||||
|
assert.ok(logs.some((l) => l.includes("another process minted it")), logs.join("\n"));
|
||||||
|
});
|
||||||
|
|
||||||
|
test("a lock left by a process that died mid-mint is taken over, not waited on for ever", async () => {
|
||||||
|
const { env, file, logs } = await delivered(forge);
|
||||||
|
forge.tokens.clear();
|
||||||
|
forge.twins.length = 0;
|
||||||
|
await mkdir(join(env.MESH_GITEA_STATE_DIR!), { recursive: true });
|
||||||
|
await writeFile(`${file}.lock`, `2147483646 ${Date.now()} gone\n`); // a pid no process holds
|
||||||
|
const before = forge.mints;
|
||||||
|
|
||||||
|
assert.equal((await minted(env, logs).listRepos()).length, 1);
|
||||||
|
assert.equal(forge.mints, before + 1);
|
||||||
|
await assert.rejects(stat(`${file}.lock`), /ENOENT/);
|
||||||
|
});
|
||||||
|
|
||||||
test("the admin account is refused: said plainly, nothing kept, and the next call fails the same way rather than crashing", async () => {
|
test("the admin account is refused: said plainly, nothing kept, and the next call fails the same way rather than crashing", async () => {
|
||||||
const { env, file, logs } = await delivered(forge);
|
const { env, file, logs } = await delivered(forge);
|
||||||
forge.admins.delete(ADMIN); // the forge's data came from a predecessor; mesh-admin was never created there
|
forge.admins.delete(ADMIN); // the forge's data came from a predecessor; mesh-admin was never created there
|
||||||
try {
|
try {
|
||||||
let clock = 1_000_000;
|
const client = minted(env, logs);
|
||||||
const client = minted(env, logs, () => clock);
|
|
||||||
const before = forge.mints;
|
const before = forge.mints;
|
||||||
|
|
||||||
await assert.rejects(client.listRepos(), (err: unknown) => {
|
await assert.rejects(client.listRepos(), (err: unknown) => {
|
||||||
@@ -422,9 +464,8 @@ test("the admin account is refused: said plainly, nothing kept, and the next cal
|
|||||||
assert.equal(forge.mints, before);
|
assert.equal(forge.mints, before);
|
||||||
await assert.rejects(stat(file), /ENOENT/);
|
await assert.rejects(stat(file), /ENOENT/);
|
||||||
|
|
||||||
// The account appears (the operator created it): the next call after the retry window mints and works.
|
// The account appears (the operator created it): the very next call mints and works.
|
||||||
forge.admins.set(ADMIN, PASSWORD);
|
forge.admins.set(ADMIN, PASSWORD);
|
||||||
clock += MINT_RETRY_AFTER_MS;
|
|
||||||
assert.equal((await client.listRepos()).length, 1);
|
assert.equal((await client.listRepos()).length, 1);
|
||||||
assert.equal(forge.mints, before + 1);
|
assert.equal(forge.mints, before + 1);
|
||||||
} finally {
|
} finally {
|
||||||
|
|||||||
@@ -0,0 +1,332 @@
|
|||||||
|
// The forge's tools act as an account of their own, not the site administrator (novox/hq ADR 0365 §2, issue 469):
|
||||||
|
// the token the tools and the watcher use is minted for `mesh-tools`, a plain account made by the module with the
|
||||||
|
// administrator's login and given write on the mesh's repositories through one team; the administrator's own
|
||||||
|
// token by that name goes; a token kept from before, the administrator's, is minted again for the account.
|
||||||
|
//
|
||||||
|
// The forge is a fake that keeps accounts, their tokens and the organisation's teams apart, and answers as gitea
|
||||||
|
// does: an administrator may mint for another account (reqSelfOrAdmin), anyone else only for itself; a protection
|
||||||
|
// edit needs the repository's administration, which neither a plain account nor its write team holds.
|
||||||
|
|
||||||
|
import { test, after } from "node:test";
|
||||||
|
import assert from "node:assert/strict";
|
||||||
|
import { createServer, type IncomingMessage, type ServerResponse } from "node:http";
|
||||||
|
import { mkdtemp, readFile, writeFile, mkdir } from "node:fs/promises";
|
||||||
|
import { tmpdir } from "node:os";
|
||||||
|
import { join } from "node:path";
|
||||||
|
|
||||||
|
import { MintedToken } from "../dist/token.js";
|
||||||
|
import { GiteaClient } from "../dist/client.js";
|
||||||
|
import { TOOLS_ACCOUNT, TOOLS_TEAM_UNITS } from "../dist/tools-account.js";
|
||||||
|
import { repoCheckFacts } from "../dist/pulls.js";
|
||||||
|
|
||||||
|
const ADMIN = "mesh-admin";
|
||||||
|
const PASSWORD = "the-vault-minted-this";
|
||||||
|
|
||||||
|
interface Account { admin: boolean; password: string; active: boolean }
|
||||||
|
interface Token { id: number; name: string; sha1: string; user: string; scopes: string[] }
|
||||||
|
interface Team { id: number; name: string; units: any; members: Set<string> }
|
||||||
|
|
||||||
|
function fakeForge() {
|
||||||
|
const users = new Map<string, Account>([[ADMIN, { admin: true, password: PASSWORD, active: true }]]);
|
||||||
|
const tokens: Token[] = [];
|
||||||
|
const teams: Team[] = [];
|
||||||
|
const calls: string[] = [];
|
||||||
|
// What the forge does wrong when a test says so: a token mint or delete it fails, a team another process made
|
||||||
|
// between this one's look and its create (gitea answers the create 422).
|
||||||
|
const faults = { mint: false, deleteToken: false, teamRace: false };
|
||||||
|
let next = 0;
|
||||||
|
const json = (res: ServerResponse, status: number, body: unknown): void => {
|
||||||
|
res.writeHead(status, { "Content-Type": "application/json" });
|
||||||
|
res.end(body === null ? "" : JSON.stringify(body));
|
||||||
|
};
|
||||||
|
const read = (req: IncomingMessage): Promise<any> =>
|
||||||
|
new Promise((resolve) => {
|
||||||
|
let t = "";
|
||||||
|
req.on("data", (c) => (t += c));
|
||||||
|
req.on("end", () => resolve(t ? JSON.parse(t) : null));
|
||||||
|
});
|
||||||
|
const who = (req: IncomingMessage): string | null => {
|
||||||
|
const h = req.headers.authorization ?? "";
|
||||||
|
if (h.startsWith("Basic ")) {
|
||||||
|
const [u, p] = Buffer.from(h.slice(6), "base64").toString().split(":");
|
||||||
|
return users.get(u)?.password === p ? u : null;
|
||||||
|
}
|
||||||
|
if (h.startsWith("token ")) return tokens.find((t) => t.sha1 === h.slice(6))?.user ?? null;
|
||||||
|
return null;
|
||||||
|
};
|
||||||
|
const isAdmin = (u: string | null) => u !== null && users.get(u)?.admin === true;
|
||||||
|
|
||||||
|
const server = createServer(async (req, res) => {
|
||||||
|
const url = new URL(req.url ?? "/", "http://fake");
|
||||||
|
const p = url.pathname.replace(/^\/api\/v1/, "");
|
||||||
|
const me = who(req);
|
||||||
|
calls.push(`${req.method} ${p} as ${me}`);
|
||||||
|
if (me === null) return json(res, 401, { message: "unauthorized" });
|
||||||
|
|
||||||
|
let m = p.match(/^\/users\/([^/]+)\/tokens(?:\/(\d+))?$/);
|
||||||
|
if (m) {
|
||||||
|
const owner = decodeURIComponent(m[1]);
|
||||||
|
if (!(req.headers.authorization ?? "").startsWith("Basic ")) return json(res, 401, { message: "basic auth required" });
|
||||||
|
if (owner !== me && !isAdmin(me)) return json(res, 403, { message: "reqSelfOrAdmin" });
|
||||||
|
if (!users.has(owner)) return json(res, 404, { message: "user does not exist" });
|
||||||
|
if (req.method === "POST") {
|
||||||
|
if (faults.mint) return json(res, 500, { message: "the forge is not well" });
|
||||||
|
const { name, scopes } = await read(req);
|
||||||
|
if (tokens.some((t) => t.user === owner && t.name === name)) return json(res, 400, { message: "token name has already been used" });
|
||||||
|
const t = { id: ++next, name, sha1: `sha-${owner}-${next}`, user: owner, scopes };
|
||||||
|
tokens.push(t);
|
||||||
|
return json(res, 201, { id: t.id, name, sha1: t.sha1, scopes });
|
||||||
|
}
|
||||||
|
if (req.method === "GET") {
|
||||||
|
const page = Number(url.searchParams.get("page") ?? "1");
|
||||||
|
const mine = tokens.filter((t) => t.user === owner);
|
||||||
|
return json(res, 200, page === 1 ? mine.map((t) => ({ id: t.id, name: t.name, sha1: "" })) : []);
|
||||||
|
}
|
||||||
|
if (req.method === "DELETE" && m[2]) {
|
||||||
|
if (faults.deleteToken && owner === ADMIN) return json(res, 500, { message: "the forge is not well" });
|
||||||
|
const i = tokens.findIndex((t) => t.user === owner && t.id === Number(m![2]));
|
||||||
|
if (i < 0) return json(res, 404, { message: "not found" });
|
||||||
|
tokens.splice(i, 1);
|
||||||
|
return json(res, 204, null);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (req.method === "GET" && p === "/user") return json(res, 200, { login: me, is_admin: isAdmin(me) });
|
||||||
|
m = p.match(/^\/users\/([^/]+)$/);
|
||||||
|
if (m && req.method === "GET") {
|
||||||
|
const u = users.get(decodeURIComponent(m[1]));
|
||||||
|
if (!u) return json(res, 404, { message: "user does not exist" });
|
||||||
|
return json(res, 200, { login: m[1], is_admin: u.admin, active: u.active, prohibit_login: false });
|
||||||
|
}
|
||||||
|
if (p.startsWith("/admin/") && !isAdmin(me)) return json(res, 403, { message: "must be a site administrator" });
|
||||||
|
if (req.method === "POST" && p === "/admin/users") {
|
||||||
|
const b = await read(req);
|
||||||
|
if (users.has(b.username)) return json(res, 422, { message: "user already exists" });
|
||||||
|
if (!b.password || b.password.length < 8) return json(res, 422, { message: "password is too short" });
|
||||||
|
users.set(b.username, { admin: b.admin === true, password: b.password, active: true });
|
||||||
|
return json(res, 201, { login: b.username, is_admin: b.admin === true });
|
||||||
|
}
|
||||||
|
m = p.match(/^\/admin\/users\/([^/]+)$/);
|
||||||
|
if (m && req.method === "PATCH") {
|
||||||
|
const u = users.get(decodeURIComponent(m[1]));
|
||||||
|
if (!u) return json(res, 404, { message: "user does not exist" });
|
||||||
|
const b = await read(req);
|
||||||
|
if (b.admin !== undefined) u.admin = b.admin;
|
||||||
|
if (b.active !== undefined) u.active = b.active;
|
||||||
|
return json(res, 200, { login: m[1], is_admin: u.admin });
|
||||||
|
}
|
||||||
|
if (p === "/orgs/novox/teams") {
|
||||||
|
if (!isAdmin(me)) return json(res, 403, { message: "must be an owner of the organisation" });
|
||||||
|
if (req.method === "GET") return json(res, 200, teams.map((t) => ({ id: t.id, name: t.name })));
|
||||||
|
const b = await read(req);
|
||||||
|
const t = { id: ++next, name: b.name, units: b, members: new Set<string>() };
|
||||||
|
teams.push(t);
|
||||||
|
if (faults.teamRace) return json(res, 422, { message: "team already exists" });
|
||||||
|
return json(res, 201, { id: t.id, name: t.name });
|
||||||
|
}
|
||||||
|
m = p.match(/^\/teams\/(\d+)(?:\/members\/([^/]+))?$/);
|
||||||
|
if (m) {
|
||||||
|
if (!isAdmin(me)) return json(res, 403, { message: "must be an owner of the organisation" });
|
||||||
|
const t = teams.find((x) => x.id === Number(m![1]));
|
||||||
|
if (!t) return json(res, 404, { message: "team does not exist" });
|
||||||
|
if (req.method === "PATCH") { t.units = await read(req); return json(res, 200, { id: t.id }); }
|
||||||
|
if (req.method === "PUT" && m[2]) { t.members.add(decodeURIComponent(m[2])); return json(res, 204, null); }
|
||||||
|
}
|
||||||
|
// Repository administration: a site administrator, or a team member whose team administers the repository.
|
||||||
|
m = p.match(/^\/repos\/novox\/([^/]+)\/branches\/(.+)$/);
|
||||||
|
if (m && req.method === "GET") {
|
||||||
|
if (m[2] === "main") return json(res, 200, { name: "main", protected: true, effective_branch_protection_name: "main",
|
||||||
|
enable_status_check: true, status_check_contexts: ["mesh/merge-gate", "mesh/repo-check"], user_can_push: false });
|
||||||
|
if (m[2] === "feat/unguarded") return json(res, 200, { name: m[2], protected: false });
|
||||||
|
return json(res, 404, { message: "branch does not exist" });
|
||||||
|
}
|
||||||
|
// gitea's reqAdmin on the whole group: reading the rules is administration too.
|
||||||
|
m = p.match(/^\/repos\/novox\/([^/]+)\/branch_protections(?:\/.*)?$/);
|
||||||
|
if (m) {
|
||||||
|
const admins = isAdmin(me) || teams.some((t) => t.members.has(me) && t.units.permission === "admin");
|
||||||
|
if (!admins) return json(res, 403, { message: "user should be an owner or a collaborator with admin write of a repository" });
|
||||||
|
return json(res, 201, { rule_name: "main" });
|
||||||
|
}
|
||||||
|
m = p.match(/^\/repos\/novox\/([^/]+)\/statuses\/([0-9a-f]+)$/);
|
||||||
|
if (m && req.method === "POST") {
|
||||||
|
const writes = isAdmin(me) || teams.some((t) => t.members.has(me) && t.units.units_map?.["repo.code"] === "write");
|
||||||
|
if (!writes) return json(res, 403, { message: "write on code required" });
|
||||||
|
return json(res, 201, { context: (await read(req)).context, creator: { login: me } });
|
||||||
|
}
|
||||||
|
if (req.method === "GET" && p === "/repos/search") return json(res, 200, { data: [] });
|
||||||
|
return json(res, 404, { message: "no such route in the fake" });
|
||||||
|
});
|
||||||
|
return new Promise<{ url: string; users: typeof users; tokens: typeof tokens; teams: typeof teams; calls: string[];
|
||||||
|
faults: typeof faults; close(): Promise<void> }>((resolve) => {
|
||||||
|
server.listen(0, "127.0.0.1", () => {
|
||||||
|
const { port } = server.address() as { port: number };
|
||||||
|
resolve({ url: `http://127.0.0.1:${port}`, users, tokens, teams, calls, faults,
|
||||||
|
close: () => new Promise((r) => server.close(() => r())) });
|
||||||
|
});
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
const forge = await fakeForge();
|
||||||
|
after(() => forge.close());
|
||||||
|
|
||||||
|
async function environment(): Promise<NodeJS.ProcessEnv & { MESH_GITEA_STATE_DIR: string }> {
|
||||||
|
const dir = await mkdtemp(join(tmpdir(), "gitea-tools-account-"));
|
||||||
|
const passwordFile = join(dir, "admin.secret");
|
||||||
|
await writeFile(passwordFile, PASSWORD + "\n", { mode: 0o600 });
|
||||||
|
return {
|
||||||
|
MESH_GITEA_URL: forge.url,
|
||||||
|
MESH_GITEA_ADMIN_USER: ADMIN,
|
||||||
|
MESH_GITEA_ADMIN_PASSWORD_FILE: passwordFile,
|
||||||
|
MESH_GITEA_STATE_DIR: join(dir, "state"),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
/** A source as a fresh process builds it from the manifest's environment (fromEnv shares one per file). */
|
||||||
|
function fresh(env: NodeJS.ProcessEnv): GiteaClient {
|
||||||
|
return GiteaClient.fromEnv(env);
|
||||||
|
}
|
||||||
|
|
||||||
|
test("the tools' token is minted for an account of their own, which is no administrator and writes through its team", async () => {
|
||||||
|
const env = await environment();
|
||||||
|
await fresh(env).listRepos();
|
||||||
|
|
||||||
|
const account = forge.users.get(TOOLS_ACCOUNT);
|
||||||
|
assert.ok(account, "the module made the tools' account");
|
||||||
|
assert.equal(account!.admin, false, "the tools' account is no administrator");
|
||||||
|
const held = forge.tokens.filter((t) => t.name === "mesh-tools");
|
||||||
|
assert.deepEqual(held.map((t) => t.user), [TOOLS_ACCOUNT], "the one token is the tools' account's, none is the administrator's");
|
||||||
|
assert.equal(await readFile(join(env.MESH_GITEA_STATE_DIR, "token"), "utf8"), held[0].sha1 + "\n");
|
||||||
|
|
||||||
|
const team = forge.teams.find((t) => t.name === "mesh-tools");
|
||||||
|
assert.ok(team, "the team is made in the mesh's organisation");
|
||||||
|
assert.ok(team!.members.has(TOOLS_ACCOUNT));
|
||||||
|
assert.equal(team!.units.permission, "write");
|
||||||
|
assert.equal(team!.units.includes_all_repositories, true);
|
||||||
|
assert.equal(team!.units.can_create_org_repo, false, "a repository's maker would administer it");
|
||||||
|
assert.deepEqual(team!.units.units_map, TOOLS_TEAM_UNITS.units_map);
|
||||||
|
assert.ok(!Object.values(team!.units.units_map).includes("admin"));
|
||||||
|
});
|
||||||
|
|
||||||
|
test("with the tools' token, the forge writes a status and refuses a protection edit — the forge itself, not a guard", async () => {
|
||||||
|
const env = await environment();
|
||||||
|
const client = fresh(env);
|
||||||
|
await client.setCommitStatus("novox", "hq", "abc123def4567890abc123def4567890abc12345", { state: "success", context: "mesh/merge-gate", description: "ok" } as any);
|
||||||
|
await assert.rejects(
|
||||||
|
client.setBranchProtection("novox", "hq", "main", { statusChecks: ["mesh/merge-gate"] }),
|
||||||
|
/403/,
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("what a base's protection asks is read from the branch, which the tools' account may read, not from the rules' list it may not", async () => {
|
||||||
|
const env = await environment();
|
||||||
|
const client = fresh(env);
|
||||||
|
await assert.rejects(client.branchProtections("novox", "hq"), /403/);
|
||||||
|
const rule = await client.appliedProtection("novox", "hq", "main");
|
||||||
|
assert.deepEqual(rule?.status_check_contexts, ["mesh/merge-gate", "mesh/repo-check"]);
|
||||||
|
assert.equal(rule?.enable_status_check, true);
|
||||||
|
assert.equal(rule?.rule_name, "main");
|
||||||
|
assert.equal(await client.appliedProtection("novox", "hq", "feat/unguarded"), null);
|
||||||
|
assert.equal(await client.appliedProtection("novox", "hq", "no/such-branch"), null);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("the administrator's token from before goes: a kept admin token is minted again for the tools' account, and the forge drops the old one", async () => {
|
||||||
|
const env = await environment();
|
||||||
|
// What the module kept before ADR 0365: a token of the administrator's own, by the same name, its scopes beside it.
|
||||||
|
const old = { id: 900, name: "mesh-tools", sha1: "sha-admin-old", user: ADMIN, scopes: ["write:repository", "write:issue", "read:user"] };
|
||||||
|
forge.tokens.push(old);
|
||||||
|
await mkdir(env.MESH_GITEA_STATE_DIR, { recursive: true });
|
||||||
|
await writeFile(join(env.MESH_GITEA_STATE_DIR, "token"), old.sha1 + "\n", { mode: 0o600 });
|
||||||
|
await writeFile(join(env.MESH_GITEA_STATE_DIR, "token.scopes"), "read:user,write:issue,write:repository\n", { mode: 0o600 });
|
||||||
|
|
||||||
|
await fresh(env).listRepos();
|
||||||
|
|
||||||
|
assert.ok(!forge.tokens.some((t) => t.user === ADMIN), "the administrator holds no tools' token any more");
|
||||||
|
const kept = (await readFile(join(env.MESH_GITEA_STATE_DIR, "token"), "utf8")).trim();
|
||||||
|
assert.notEqual(kept, old.sha1);
|
||||||
|
assert.equal(forge.tokens.find((t) => t.sha1 === kept)?.user, TOOLS_ACCOUNT);
|
||||||
|
assert.match(await readFile(join(env.MESH_GITEA_STATE_DIR, "token.scopes"), "utf8"), / for mesh-tools\n$/);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("a tools' account someone made an administrator is made a plain account again at the next mint", async () => {
|
||||||
|
const env = await environment();
|
||||||
|
forge.users.get(TOOLS_ACCOUNT)!.admin = true;
|
||||||
|
await fresh(env).listRepos();
|
||||||
|
assert.equal(forge.users.get(TOOLS_ACCOUNT)!.admin, false);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("the tools never act as the administrator: MESH_GITEA_TOOLS_USER naming it is refused", async () => {
|
||||||
|
const env = await environment();
|
||||||
|
assert.throws(() => MintedToken.fromEnv(forge.url, { ...env, MESH_GITEA_TOOLS_USER: ADMIN }), /names the admin account/);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("the manifest hands the tools their account, and the merge-ready authors name it rather than the administrator", async () => {
|
||||||
|
const manifest = JSON.parse(await readFile(new URL("../module.json", import.meta.url), "utf8"));
|
||||||
|
const code = manifest.build.artifacts.find((a: any) => a.name === "code");
|
||||||
|
assert.equal(code.env.MESH_GITEA_TOOLS_USER, TOOLS_ACCOUNT);
|
||||||
|
const authors = String(manifest.settings["ready-authors"].default).split(/\s+/);
|
||||||
|
assert.ok(authors.includes(TOOLS_ACCOUNT));
|
||||||
|
// Kept for now (review of issue 469): a pull request the admin account marked ready before the tools acted as
|
||||||
|
// their own keeps counting as ready after the delivery.
|
||||||
|
assert.ok(authors.includes(ADMIN), "marks made by the admin account before ADR 0365 keep counting");
|
||||||
|
// The Go forge bundle reads the token the tools' account holds, never the administrator's password.
|
||||||
|
const forgeBundle = manifest.build.artifacts.find((a: any) => a.name === "forge");
|
||||||
|
assert.ok(forgeBundle.env.MESH_GITEA_TOKEN_FILE);
|
||||||
|
assert.equal(forgeBundle.env.MESH_GITEA_ADMIN_PASSWORD_FILE, undefined);
|
||||||
|
});
|
||||||
|
|
||||||
|
// Review of issue 469: what reads the rules' list is moved to the branch, and nothing leaves the tools without a token.
|
||||||
|
|
||||||
|
test("a repository check said as a warning on a head without merge-check.sh reads the base's protection from the branch", async () => {
|
||||||
|
const client = fresh(await environment());
|
||||||
|
const forgeOf = { holdsFile: async () => false, appliedProtection: client.appliedProtection.bind(client) };
|
||||||
|
const c = { owner: "novox", repo: "hq", commit: "abc123def4567890abc123def4567890abc12345", "repo-check": { verdict: "warning" } } as any;
|
||||||
|
// main requires mesh/repo-check: required, so a head that defines no check fails it for a person to see.
|
||||||
|
assert.deepEqual(await repoCheckFacts(forgeOf, c, "main"), { defined: false, required: true });
|
||||||
|
// A base nothing guards: not required, so the head's repository check passes rather than fails.
|
||||||
|
assert.deepEqual(await repoCheckFacts(forgeOf, c, "feat/unguarded"), { defined: false, required: false });
|
||||||
|
});
|
||||||
|
|
||||||
|
test("a mint that fails leaves the administrator's old token in place: the tools are never left without one", async () => {
|
||||||
|
const env = await environment();
|
||||||
|
const old = { id: 901, name: "mesh-tools", sha1: "sha-admin-kept", user: ADMIN, scopes: ["read:user"] };
|
||||||
|
forge.tokens.push(old);
|
||||||
|
forge.faults.mint = true;
|
||||||
|
try {
|
||||||
|
await assert.rejects(fresh(env).listRepos(), /500/);
|
||||||
|
} finally {
|
||||||
|
forge.faults.mint = false;
|
||||||
|
}
|
||||||
|
assert.ok(forge.tokens.some((t) => t.sha1 === old.sha1), "the admin's token was deleted before a new one was kept");
|
||||||
|
forge.tokens.splice(forge.tokens.indexOf(old), 1);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("an old token that cannot be deleted after the mint is said, the new token kept, and deleted at the next start", async () => {
|
||||||
|
const env = await environment();
|
||||||
|
const old = { id: 902, name: "mesh-tools", sha1: "sha-admin-stuck", user: ADMIN, scopes: ["read:user"] };
|
||||||
|
forge.tokens.push(old);
|
||||||
|
forge.faults.deleteToken = true;
|
||||||
|
try {
|
||||||
|
await fresh(env).listRepos();
|
||||||
|
} finally {
|
||||||
|
forge.faults.deleteToken = false;
|
||||||
|
}
|
||||||
|
const kept = (await readFile(join(env.MESH_GITEA_STATE_DIR, "token"), "utf8")).trim();
|
||||||
|
assert.equal(forge.tokens.find((t) => t.sha1 === kept)?.user, TOOLS_ACCOUNT, "the account's token is kept and works");
|
||||||
|
assert.ok(forge.tokens.some((t) => t.sha1 === old.sha1));
|
||||||
|
// The next process reuses the kept token and retires the old one it could not before.
|
||||||
|
const next = new MintedToken({ url: forge.url, admin: ADMIN, passwordFile: env.MESH_GITEA_ADMIN_PASSWORD_FILE!,
|
||||||
|
file: join(env.MESH_GITEA_STATE_DIR, "token"), account: TOOLS_ACCOUNT, log: () => {} });
|
||||||
|
assert.equal(await next.current(), kept);
|
||||||
|
assert.ok(!forge.tokens.some((t) => t.sha1 === old.sha1), "the admin's old token was deleted at the next start");
|
||||||
|
});
|
||||||
|
|
||||||
|
test("a team another process made between the look and the create is found, not a failure", async () => {
|
||||||
|
forge.teams.splice(0);
|
||||||
|
forge.faults.teamRace = true;
|
||||||
|
try {
|
||||||
|
await fresh(await environment()).listRepos();
|
||||||
|
} finally {
|
||||||
|
forge.faults.teamRace = false;
|
||||||
|
}
|
||||||
|
const team = forge.teams.find((t) => t.name === "mesh-tools");
|
||||||
|
assert.ok(team?.members.has(TOOLS_ACCOUNT));
|
||||||
|
});
|
||||||
@@ -83,20 +83,235 @@ test("each forge event is named in the module's stream and trimmed", async () =>
|
|||||||
assert.equal(eventOf("pull_request_review_approved", {}).name, "forge.other");
|
assert.equal(eventOf("pull_request_review_approved", {}).name, "forge.other");
|
||||||
});
|
});
|
||||||
|
|
||||||
test("the system webhook is made when none points at the receiver, and corrected when one does", async () => {
|
// A forge's admin hooks as 1.27.3 keeps them: POST /admin/hooks makes a system webhook only when config.is_system_webhook
|
||||||
const { ensureSystemHook } = await import("../webhook.ts");
|
// is "true", and a default webhook otherwise; the listing names one kind at a time (type=system, the default, or default).
|
||||||
|
// Repositories hold their own webhooks (a default webhook's copies), listed a page at a time.
|
||||||
|
function fakeForge(start: { id: number; url: string; system: boolean }[] = [],
|
||||||
|
repoHooks: Record<string, { id: number; url: string }[]> = {}, repoCount = 0) {
|
||||||
|
const hooks = start.map((h) => ({ ...h, events: ["push"], active: true }));
|
||||||
|
const repos = [...Object.keys(repoHooks)];
|
||||||
|
for (let i = 0; repos.length < repoCount; i++) repos.push(`novox/r${i}`);
|
||||||
|
let next = 100;
|
||||||
const calls: string[] = [];
|
const calls: string[] = [];
|
||||||
let hooks: unknown[] = [];
|
const made: any[] = [];
|
||||||
|
const page = (items: any[], q: URLSearchParams) => {
|
||||||
|
const p = Number(q.get("page") ?? 1), l = Number(q.get("limit") ?? 50);
|
||||||
|
return items.slice((p - 1) * l, p * l);
|
||||||
|
};
|
||||||
const admin = { call: async (method: string, path: string, body?: any) => {
|
const admin = { call: async (method: string, path: string, body?: any) => {
|
||||||
calls.push(`${method} ${path}`);
|
calls.push(`${method} ${path}`);
|
||||||
if (method === "GET") return { status: 200, body: hooks };
|
const u = new URL(path, "http://f");
|
||||||
if (method === "POST") { assert.ok(body.events.includes("pull_request") && body.events.includes("push")); assert.equal(body.config.secret, secret); return { status: 201, body: {} }; }
|
const repoHook = u.pathname.match(/^\/repos\/([^/]+\/[^/]+)\/hooks(?:\/(\d+))?$/);
|
||||||
return { status: 200, body: {} };
|
if (method === "GET" && u.pathname === "/repos/search") {
|
||||||
|
return { status: 200, body: { ok: true, data: page(repos.map((full_name) => ({ full_name })), u.searchParams) } };
|
||||||
|
}
|
||||||
|
if (repoHook) {
|
||||||
|
const list = repoHooks[repoHook[1]] ?? [];
|
||||||
|
if (method === "GET") return { status: 200, body: page(list.map((h) => ({ id: h.id, config: { url: h.url } })), u.searchParams) };
|
||||||
|
const i = list.findIndex((h) => h.id === Number(repoHook[2]));
|
||||||
|
if (i < 0) return { status: 404, body: {} };
|
||||||
|
list.splice(i, 1);
|
||||||
|
return { status: 204, body: null };
|
||||||
|
}
|
||||||
|
if (method === "GET") {
|
||||||
|
const type = u.searchParams.get("type") ?? "system";
|
||||||
|
const listed = hooks.filter((h) => type === "all" || h.system === (type === "system"));
|
||||||
|
return { status: 200, body: listed.map((h) => ({ id: h.id, active: h.active, events: h.events,
|
||||||
|
config: { url: h.url, content_type: "json" } })) };
|
||||||
|
}
|
||||||
|
if (method === "POST") {
|
||||||
|
made.push(body);
|
||||||
|
hooks.push({ id: next++, url: body.config.url, system: body.config.is_system_webhook === "true", events: body.events,
|
||||||
|
active: body.active });
|
||||||
|
return { status: 201, body: {} };
|
||||||
|
}
|
||||||
|
if (method === "DELETE") {
|
||||||
|
const id = Number(path.split("/").pop());
|
||||||
|
const i = hooks.findIndex((h) => h.id === id);
|
||||||
|
if (i < 0) return { status: 404, body: {} };
|
||||||
|
hooks.splice(i, 1);
|
||||||
|
return { status: 204, body: null };
|
||||||
|
}
|
||||||
|
return { status: 405, body: {} };
|
||||||
} };
|
} };
|
||||||
assert.equal(await ensureSystemHook(admin, "http://h:3047/forge-event", secret), "created");
|
return { admin, hooks, calls, made, repoHooks };
|
||||||
hooks = [{ id: 4, config: { url: "http://h:3047/forge-event" } }];
|
}
|
||||||
assert.equal(await ensureSystemHook(admin, "http://h:3047/forge-event", secret), "corrected");
|
|
||||||
assert.ok(calls.includes("PATCH /admin/hooks/4"));
|
test("the webhook is made a system webhook, which the forge sends every repository's events to", async () => {
|
||||||
|
const { ensureSystemHook } = await import("../webhook.ts");
|
||||||
|
const forge = fakeForge();
|
||||||
|
const url = "http://h:3047/forge-event";
|
||||||
|
const hook = await ensureSystemHook(forge.admin, url, secret);
|
||||||
|
assert.equal(forge.made.length, 1);
|
||||||
|
assert.equal(forge.made[0].config.is_system_webhook, "true", "without it the forge makes a default webhook, which fires for no repository");
|
||||||
|
assert.equal(forge.made[0].config.secret, secret);
|
||||||
|
assert.ok(forge.made[0].events.includes("pull_request") && forge.made[0].events.includes("push"));
|
||||||
|
assert.deepEqual(forge.hooks.map((h) => h.system), [true]);
|
||||||
|
assert.equal(hook.kind, "system");
|
||||||
|
assert.equal(hook.did, "created");
|
||||||
|
assert.equal(hook.url, url);
|
||||||
|
assert.ok(hook.events.includes("pull_request"));
|
||||||
|
});
|
||||||
|
|
||||||
|
test("default webhooks left at the receiver by an earlier start are replaced by one system webhook", async () => {
|
||||||
|
const { ensureSystemHook } = await import("../webhook.ts");
|
||||||
|
const url = "http://h:3047/forge-event";
|
||||||
|
// What the start at 23:05 left: a default webhook, invisible to the system listing, made again by every start.
|
||||||
|
const forge = fakeForge([{ id: 4, url, system: false }, { id: 5, url, system: false }, { id: 6, url: "http://other/x", system: true }]);
|
||||||
|
const hook = await ensureSystemHook(forge.admin, url, secret);
|
||||||
|
assert.deepEqual(hook.removed, { system: 0, default: 2, repository: 0, repositories: [] });
|
||||||
|
assert.equal(hook.did, "replaced");
|
||||||
|
assert.deepEqual(forge.hooks.map((h) => [h.url, h.system]), [["http://other/x", true], [url, true]], "another url is kept");
|
||||||
|
// A second start replaces the system webhook, so the secret kept now is the one the forge signs with (its edit never
|
||||||
|
// changes a secret).
|
||||||
|
const again = await ensureSystemHook(forge.admin, url, secret);
|
||||||
|
assert.deepEqual(again.removed, { system: 1, default: 0, repository: 0, repositories: [] });
|
||||||
|
assert.equal(forge.hooks.filter((h) => h.url === url).length, 1);
|
||||||
|
assert.ok(forge.calls.includes("DELETE /admin/hooks/4") && forge.calls.includes("DELETE /admin/hooks/5"));
|
||||||
|
// The new one is made before any old one is deleted, so no event falls in a gap.
|
||||||
|
const post = forge.calls.indexOf("POST /admin/hooks");
|
||||||
|
assert.ok(post >= 0 && forge.calls.findIndex((c) => c.startsWith("DELETE")) > post);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("a repository's copy of a default webhook at the receiver is deleted, on every page, and said", async () => {
|
||||||
|
const { ensureSystemHook } = await import("../webhook.ts");
|
||||||
|
const url = "http://h:3047/forge-event";
|
||||||
|
// A repository made after the default webhook got its copy, which would send its events a second time; 120
|
||||||
|
// repositories, so the copies sit beyond the first page.
|
||||||
|
const repoHooks: Record<string, { id: number; url: string }[]> = {};
|
||||||
|
for (let i = 0; i < 118; i++) repoHooks[`novox/r${i}`] = [];
|
||||||
|
repoHooks["novox/late"] = [{ id: 31, url }, { id: 32, url: "http://ci/hook" }];
|
||||||
|
repoHooks["novox/later"] = [{ id: 33, url }];
|
||||||
|
const forge = fakeForge([{ id: 4, url, system: false }], repoHooks);
|
||||||
|
const hook = await ensureSystemHook(forge.admin, url, secret);
|
||||||
|
assert.deepEqual(hook.removed, { system: 0, default: 1, repository: 2, repositories: ["novox/late", "novox/later"] });
|
||||||
|
assert.deepEqual(forge.repoHooks["novox/late"].map((h) => h.id), [32], "a repository's other webhook is kept");
|
||||||
|
assert.deepEqual(forge.repoHooks["novox/later"], []);
|
||||||
|
assert.ok(forge.calls.some((c) => /\/repos\/search\?.*page=3/.test(c)), "every page of repositories is read");
|
||||||
|
});
|
||||||
|
|
||||||
|
test("a forge that makes no system webhook is said, never taken as heard", async () => {
|
||||||
|
const { ensureSystemHook } = await import("../webhook.ts");
|
||||||
|
const forge = fakeForge();
|
||||||
|
const admin = { call: async (method: string, path: string, body?: any) => {
|
||||||
|
// A forge that ignores the flag: what 23:05 met, had the module read back.
|
||||||
|
if (method === "POST") body = { ...body, config: { ...body.config, is_system_webhook: undefined } };
|
||||||
|
return forge.admin.call(method, path, body);
|
||||||
|
} };
|
||||||
|
await assert.rejects(ensureSystemHook(admin, "http://h:3047/forge-event", secret), /names 0 new system webhooks/);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("the receiver tells what it verified and what it refused", async () => {
|
||||||
|
const { receiver, HOOK_PATH } = await import("../webhook.ts");
|
||||||
|
const heard: string[] = [], refused: string[] = [];
|
||||||
|
const server = receiver({ secret, port: 0, log: () => {}, publish: async () => {},
|
||||||
|
heard: (kind, delivery) => heard.push(`${kind} ${delivery}`), refused: (reason) => refused.push(reason) });
|
||||||
|
await new Promise<void>((r) => server.listen(0, "127.0.0.1", () => r()));
|
||||||
|
const url = `http://127.0.0.1:${(server.address() as AddressInfo).port}${HOOK_PATH}`;
|
||||||
|
try {
|
||||||
|
await fetch(url, { method: "POST", body: opened, headers: { "X-Gitea-Event": "pull_request", "X-Gitea-Signature": sign(opened), "X-Gitea-Delivery": "d-9" } });
|
||||||
|
await fetch(url, { method: "POST", body: opened, headers: { "X-Gitea-Event": "pull_request", "X-Gitea-Signature": sign(opened, "x") } });
|
||||||
|
await server.idle();
|
||||||
|
assert.deepEqual(heard, ["pull_request d-9"]);
|
||||||
|
assert.deepEqual(refused, ["its signature does not verify"]);
|
||||||
|
} finally { server.close(); }
|
||||||
|
});
|
||||||
|
|
||||||
|
test("a forge that never delivers is said loudly once, and unhealthy, when the poll finds what no event said", async () => {
|
||||||
|
const { Pace } = await import("../webhook.ts");
|
||||||
|
let now = 1_000_000;
|
||||||
|
const said: string[] = [];
|
||||||
|
const pace = new Pace(() => now, (l) => said.push(l));
|
||||||
|
const hook = { kind: "system" as const, id: 7, url: "http://h:3047/forge-event", events: ["push"], active: true,
|
||||||
|
did: "created" as const, removed: { system: 0, default: 0, repository: 0, repositories: [] } };
|
||||||
|
pace.listening("172.17.0.1", 3047, hook.url, hook);
|
||||||
|
now += 5 * 60_000;
|
||||||
|
pace.miss("novox/hq#7 at aaaaaaaa");
|
||||||
|
assert.equal(said.length, 0, "five minutes after the start is not yet silence");
|
||||||
|
assert.equal((pace.report() as any).healthy, true);
|
||||||
|
pace.refused("it is unsigned", "172.17.0.3");
|
||||||
|
assert.equal((pace.report() as any).healthy, false, "refused, and none verified");
|
||||||
|
now += 6 * 60_000;
|
||||||
|
pace.miss("the merge of novox/hq#7");
|
||||||
|
pace.miss("novox/hq#8 at bbbbbbbb");
|
||||||
|
assert.equal(said.filter((l) => /NOT HEARD: no delivery ever verified/.test(l)).length, 1, "said once");
|
||||||
|
let r = pace.report() as any;
|
||||||
|
assert.equal(r.healthy, false);
|
||||||
|
assert.equal(r.state, "unhealthy");
|
||||||
|
assert.match(r.why, /never delivered/);
|
||||||
|
assert.equal(r.polls.found_unsaid, 3);
|
||||||
|
assert.equal(r.refused.count, 1);
|
||||||
|
assert.equal(r.refused.last.reason, "it is unsigned");
|
||||||
|
assert.equal(r.hook.kind, "system");
|
||||||
|
assert.equal(r.receiver.port, 3047);
|
||||||
|
// The first delivery clears it.
|
||||||
|
now += 1000;
|
||||||
|
pace.heard("push", "d-1");
|
||||||
|
r = pace.report() as any;
|
||||||
|
assert.equal(r.healthy, true);
|
||||||
|
assert.equal(r.verified.count, 1);
|
||||||
|
assert.equal(r.verified.last.event, "push");
|
||||||
|
assert.equal(r.polls.every_seconds, 300);
|
||||||
|
assert.match(said.join("\n"), /heard again/);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("a forge whose events stop after one delivery is said loudly once per silent spell, and refusals are unhealthy", async () => {
|
||||||
|
const { Pace } = await import("../webhook.ts");
|
||||||
|
let now = 1_000_000;
|
||||||
|
const said: string[] = [];
|
||||||
|
const pace = new Pace(() => now, (l) => said.push(l));
|
||||||
|
const hook = { kind: "system" as const, id: 7, url: "http://h:3047/forge-event", events: ["push"], active: true,
|
||||||
|
did: "created" as const, removed: { system: 0, default: 0, repository: 0, repositories: [] } };
|
||||||
|
pace.listening("172.17.0.1", 3047, hook.url, hook);
|
||||||
|
now += 60_000;
|
||||||
|
pace.heard("push", "d-1");
|
||||||
|
// The hook is deleted by hand: nothing more comes, and the poll finds what no event said.
|
||||||
|
now += 11 * 60_000;
|
||||||
|
pace.miss("novox/hq#7 at aaaaaaaa");
|
||||||
|
assert.equal(said.filter((l) => /NOT HEARD/.test(l)).length, 0, "one find alone may be the poll beating an event");
|
||||||
|
pace.miss("the merge of novox/hq#7");
|
||||||
|
pace.miss("novox/hq#8 at bbbbbbbb");
|
||||||
|
assert.equal(said.filter((l) => /NOT HEARD/.test(l)).length, 1, "said once");
|
||||||
|
assert.match(said.find((l) => /NOT HEARD/.test(l)) ?? "", /since the last at/);
|
||||||
|
assert.equal((pace.report() as any).healthy, false);
|
||||||
|
// A delivery ends the spell; the next spell is said again.
|
||||||
|
now += 1000;
|
||||||
|
pace.heard("push", "d-2");
|
||||||
|
assert.equal((pace.report() as any).healthy, true);
|
||||||
|
now += 11 * 60_000;
|
||||||
|
pace.miss("novox/hq#9 at cccccccc");
|
||||||
|
pace.miss("novox/hq#10 at dddddddd");
|
||||||
|
assert.equal(said.filter((l) => /NOT HEARD/.test(l)).length, 2, "a second silent spell is said again");
|
||||||
|
// Every delivery refused after a verified one (a secret that no longer matches) is unhealthy at once.
|
||||||
|
now += 1000;
|
||||||
|
pace.heard("push", "d-3");
|
||||||
|
now += 1000;
|
||||||
|
pace.refused("its signature does not verify", "172.17.0.3");
|
||||||
|
pace.refused("its signature does not verify", "172.17.0.3");
|
||||||
|
const r = pace.report() as any;
|
||||||
|
assert.equal(r.healthy, false);
|
||||||
|
assert.equal(r.refused.since_verified, 2);
|
||||||
|
assert.match(r.why, /2 deliveries refused since the last verified one.*does not verify/);
|
||||||
|
now += 1000;
|
||||||
|
pace.heard("push", "d-4");
|
||||||
|
assert.equal((pace.report() as any).healthy, true, "a verified delivery clears it");
|
||||||
|
});
|
||||||
|
|
||||||
|
test("without a receiver by configuration the polls alone watch, healthy; a receiver that failed is unhealthy", async () => {
|
||||||
|
const { Pace } = await import("../webhook.ts");
|
||||||
|
let now = 1_000_000;
|
||||||
|
const said: string[] = [];
|
||||||
|
const quiet = new Pace(() => now, (l) => said.push(l));
|
||||||
|
quiet.unheard("MESH_GITEA_HOOK_URL unset", false);
|
||||||
|
now += 20 * 60_000;
|
||||||
|
quiet.miss("novox/hq#7 at aaaaaaaa");
|
||||||
|
assert.equal(said.length, 0, "no deliveries were expected");
|
||||||
|
assert.equal((quiet.report() as any).healthy, true);
|
||||||
|
const failed = new Pace(() => now, () => {});
|
||||||
|
failed.unheard("the forge's system webhook could not be made: 403");
|
||||||
|
const r = failed.report() as any;
|
||||||
|
assert.equal(r.healthy, false);
|
||||||
|
assert.match(r.why, /403/);
|
||||||
});
|
});
|
||||||
|
|
||||||
test("the polls look every 30 s until events are heard, every five minutes while they are, and 30 s again on a miss", async () => {
|
test("the polls look every 30 s until events are heard, every five minutes while they are, and 30 s again on a miss", async () => {
|
||||||
|
|||||||
+208
-78
@@ -12,7 +12,20 @@
|
|||||||
// hands a token's value out exactly once, so a token not kept is a token lost;
|
// hands a token's value out exactly once, so a token not kept is a token lost;
|
||||||
// - re-minted when the forge rejects it (401) or the kept file is gone. The one case that is not
|
// - re-minted when the forge rejects it (401) or the kept file is gone. The one case that is not
|
||||||
// a fault: the forge's data was restored from a predecessor and the token the file names never
|
// a fault: the forge's data was restored from a predecessor and the token the file names never
|
||||||
// existed there.
|
// existed there;
|
||||||
|
// - minted by one process at a time. The runtime starts each of the bundle's entrypoints as a process
|
||||||
|
// of its own (novox/hq ADR 0193): the watcher (index.js) and the tools (tools/index.js) are two
|
||||||
|
// processes over one kept file. Two that minted at once left the forge two tokens by one name, which
|
||||||
|
// gitea then refuses to delete by name — for ever, and every forge tool with it (the 2026-10-14
|
||||||
|
// outage). So a mint holds a lock file beside the kept token, and reads the kept file again once it
|
||||||
|
// holds it: what another process minted meanwhile is reused, not minted over;
|
||||||
|
// - replaced by id: an old token by this name is found in the account's list and deleted by its id,
|
||||||
|
// every one of them, so a name held twice is cleared rather than refused.
|
||||||
|
//
|
||||||
|
// - minted for the tools' account, not the administrator (novox/hq ADR 0365 §2, issue 469): the administrator's
|
||||||
|
// login mints it for `mesh-tools` (tools-account.ts), after making that account and its team, and deletes any
|
||||||
|
// token by this name the administrator itself still holds — the token the tools used before. A token kept
|
||||||
|
// for another account than the one asked (the scopes line says whose) is minted again at the next start.
|
||||||
//
|
//
|
||||||
// An explicitly configured token still wins, and is never minted over: if it is rejected, that is
|
// An explicitly configured token still wins, and is never minted over: if it is rejected, that is
|
||||||
// reported, not repaired — somebody chose it.
|
// reported, not repaired — somebody chose it.
|
||||||
@@ -20,8 +33,10 @@
|
|||||||
// The token is never logged. Lines say that one was minted, reused or renewed, and where it is
|
// The token is never logged. Lines say that one was minted, reused or renewed, and where it is
|
||||||
// kept; never what it is.
|
// kept; never what it is.
|
||||||
|
|
||||||
import { chmodSync, mkdirSync, readFileSync, renameSync, writeFileSync } from "node:fs";
|
import { chmodSync, closeSync, existsSync, mkdirSync, openSync, readFileSync, renameSync, rmSync, statSync, unlinkSync, writeFileSync, writeSync } from "node:fs";
|
||||||
|
import { randomBytes } from "node:crypto";
|
||||||
import { dirname, join } from "node:path";
|
import { dirname, join } from "node:path";
|
||||||
|
import { setUpToolsAccount, TOOLS_ACCOUNT, type AdminCall } from "./tools-account.js";
|
||||||
|
|
||||||
/** The name the token carries in the forge's own list — one per mesh runtime, found by name. */
|
/** The name the token carries in the forge's own list — one per mesh runtime, found by name. */
|
||||||
export const TOKEN_NAME = "mesh-tools";
|
export const TOKEN_NAME = "mesh-tools";
|
||||||
@@ -39,7 +54,7 @@ export const TOKEN_NAME = "mesh-tools";
|
|||||||
* `gitea_api` (novox/hq 229); whoever may call a tool includes agents, and the forge's administration is the
|
* `gitea_api` (novox/hq 229); whoever may call a tool includes agents, and the forge's administration is the
|
||||||
* operator's. The provisioner settles users with the admin account's own login (GiteaAdmin), never this token.
|
* operator's. The provisioner settles users with the admin account's own login (GiteaAdmin), never this token.
|
||||||
* A token kept with a different list is minted again with this one at the next start (the scopes are kept
|
* A token kept with a different list is minted again with this one at the next start (the scopes are kept
|
||||||
* beside it), and the forge drops the old one by name — so the scope a kept token had does not outlive this.
|
* beside it), and the old one is deleted from the forge by its id — so the scope a kept token had does not outlive this.
|
||||||
*
|
*
|
||||||
* A token kept from before a scope was added lacks it: the forge answers such a call with
|
* A token kept from before a scope was added lacks it: the forge answers such a call with
|
||||||
* `403 token does not have at least one of required scope(s)`, and the client treats that like a
|
* `403 token does not have at least one of required scope(s)`, and the client treats that like a
|
||||||
@@ -95,42 +110,37 @@ export interface MintedTokenOptions {
|
|||||||
readonly file: string;
|
readonly file: string;
|
||||||
readonly name?: string;
|
readonly name?: string;
|
||||||
readonly scopes?: readonly string[];
|
readonly scopes?: readonly string[];
|
||||||
|
/**
|
||||||
|
* The account the token is minted for: the tools' own (novox/hq ADR 0365 §2). The admin login mints it, after
|
||||||
|
* `setUp` made the account, and the admin's own tokens by this name are deleted. Left out, the token is the
|
||||||
|
* admin's own, as it was before.
|
||||||
|
*/
|
||||||
|
readonly account?: string;
|
||||||
|
/** Run with the admin's password before each mint for `account`: makes the account and its team (tools-account.ts). */
|
||||||
|
readonly setUp?: (password: string) => Promise<void>;
|
||||||
readonly log?: (line: string) => void;
|
readonly log?: (line: string) => void;
|
||||||
readonly fetch?: typeof fetch;
|
readonly fetch?: typeof fetch;
|
||||||
/** The clock, in milliseconds; a test sets it. */
|
|
||||||
readonly now?: () => number;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* **How long a failed mint is not tried again** (novox/hq issue 739). Every call that finds no token mints, and a
|
|
||||||
* mint is two or three calls over basic auth, each one hashing the admin password on the forge. A mint that keeps
|
|
||||||
* failing, once per tool call, is a loop the forge pays for in CPU: on 2026-10-14 it ran several times a minute
|
|
||||||
* on the control-node while it was already overloaded. Within this window the last failure is answered again,
|
|
||||||
* without asking the forge.
|
|
||||||
*/
|
|
||||||
export const MINT_RETRY_AFTER_MS = 30_000;
|
|
||||||
|
|
||||||
/** The token the module mints for itself, kept in its state and renewed when the forge rejects it. */
|
/** The token the module mints for itself, kept in its state and renewed when the forge rejects it. */
|
||||||
export class MintedToken implements TokenSource {
|
export class MintedToken implements TokenSource {
|
||||||
private held: string | null = null;
|
private held: string | null = null;
|
||||||
private readFile = false;
|
private readFile = false;
|
||||||
private inflight: Promise<string> | null = null;
|
private inflight: Promise<string> | null = null;
|
||||||
private failed: { at: number; error: unknown } | null = null;
|
|
||||||
private readonly name: string;
|
private readonly name: string;
|
||||||
private readonly scopes: readonly string[];
|
private readonly scopes: readonly string[];
|
||||||
|
private readonly account: string;
|
||||||
private readonly log: (line: string) => void;
|
private readonly log: (line: string) => void;
|
||||||
private readonly fetchImpl: typeof fetch;
|
private readonly fetchImpl: typeof fetch;
|
||||||
|
|
||||||
constructor(private readonly opts: MintedTokenOptions) {
|
constructor(private readonly opts: MintedTokenOptions) {
|
||||||
this.name = opts.name ?? TOKEN_NAME;
|
this.name = opts.name ?? TOKEN_NAME;
|
||||||
this.scopes = opts.scopes ?? TOKEN_SCOPES;
|
this.scopes = opts.scopes ?? TOKEN_SCOPES;
|
||||||
|
this.account = opts.account ?? opts.admin;
|
||||||
this.log = opts.log ?? ((line) => console.log(`[gitea] ${line}`));
|
this.log = opts.log ?? ((line) => console.log(`[gitea] ${line}`));
|
||||||
this.fetchImpl = opts.fetch ?? fetch;
|
this.fetchImpl = opts.fetch ?? fetch;
|
||||||
this.now = opts.now ?? Date.now;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private readonly now: () => number;
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Build from the runtime's environment: the forge's URL, the admin login and password file the
|
* Build from the runtime's environment: the forge's URL, the admin login and password file the
|
||||||
* manifest hands the runtime, and the module's state directory (MESH_GITEA_STATE_DIR, a directory
|
* manifest hands the runtime, and the module's state directory (MESH_GITEA_STATE_DIR, a directory
|
||||||
@@ -139,12 +149,9 @@ export class MintedToken implements TokenSource {
|
|||||||
*/
|
*/
|
||||||
static fromEnv(url: string, env: NodeJS.ProcessEnv = process.env): MintedToken {
|
static fromEnv(url: string, env: NodeJS.ProcessEnv = process.env): MintedToken {
|
||||||
const opts = MintedToken.optionsFromEnv(url, env);
|
const opts = MintedToken.optionsFromEnv(url, env);
|
||||||
// One source per kept file in a process. The watcher and the tools entrypoint both build a
|
// One source per kept file in a process: two sources in one process would each renew on a 401.
|
||||||
// client in the same runtime; two sources over one file would each renew on a 401 and drop the
|
// This covers one process only. The watcher and the tools are two processes (the runtime starts
|
||||||
// other's token by name, forever. Shared, a renewal is one renewal. Across processes it is not shared: the
|
// each entrypoint as its own), and those are kept apart by the mint's lock file (mint, below).
|
||||||
// runtime serves the watcher (index.js) and the tools (tools/index.js) as separate bundles, so two sources over
|
|
||||||
// one file do exist. renew() reads the kept file before it mints, and a race that still leaves two tokens by
|
|
||||||
// the name is undone by dropping each by its id (doMint; novox/hq issue 739).
|
|
||||||
const shared = MintedToken.shared.get(opts.file);
|
const shared = MintedToken.shared.get(opts.file);
|
||||||
if (shared) return shared;
|
if (shared) return shared;
|
||||||
const source = new MintedToken(opts);
|
const source = new MintedToken(opts);
|
||||||
@@ -166,7 +173,17 @@ export class MintedToken implements TokenSource {
|
|||||||
if (missing.length) {
|
if (missing.length) {
|
||||||
throw new Error(`no Gitea token, and nothing to mint one with — set MESH_GITEA_TOKEN, or ${missing.join(", ")}`);
|
throw new Error(`no Gitea token, and nothing to mint one with — set MESH_GITEA_TOKEN, or ${missing.join(", ")}`);
|
||||||
}
|
}
|
||||||
return { url, admin: admin!, passwordFile: passwordFile!, file: join(stateDir!, "token") };
|
// The tools' own account, never the administrator (novox/hq ADR 0365 §2): MESH_GITEA_TOOLS_USER names it.
|
||||||
|
const account = env.MESH_GITEA_TOOLS_USER || TOOLS_ACCOUNT;
|
||||||
|
if (account === admin) throw new Error(`MESH_GITEA_TOOLS_USER names the admin account "${admin}"; the tools act as an account of their own (novox/hq ADR 0365)`);
|
||||||
|
return {
|
||||||
|
url, admin: admin!, passwordFile: passwordFile!, file: join(stateDir!, "token"), account,
|
||||||
|
// GiteaAdmin is loaded when first used: client.ts imports this file.
|
||||||
|
setUp: async (password) => {
|
||||||
|
const { GiteaAdmin } = await import("./client.js");
|
||||||
|
await setUpToolsAccount(new GiteaAdmin(url, admin!, password), account, () => randomBytes(24).toString("base64url"));
|
||||||
|
},
|
||||||
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
async current(): Promise<string> {
|
async current(): Promise<string> {
|
||||||
@@ -177,11 +194,13 @@ export class MintedToken implements TokenSource {
|
|||||||
if (kept !== null && this.keptScopes() === this.scopesLine()) {
|
if (kept !== null && this.keptScopes() === this.scopesLine()) {
|
||||||
this.held = kept;
|
this.held = kept;
|
||||||
this.log(`reusing the token kept at ${this.opts.file}`);
|
this.log(`reusing the token kept at ${this.opts.file}`);
|
||||||
|
// A retirement of the admin's old token that failed after the mint is tried again, once a process.
|
||||||
|
if (existsSync(this.retireFile)) await this.locked(() => this.retireAdmin()).catch((err) => this.log(`${err}`));
|
||||||
return kept;
|
return kept;
|
||||||
}
|
}
|
||||||
if (kept !== null) return this.mint("the kept token was minted with other scopes than the tools take now — minting one with them");
|
if (kept !== null) return this.mint("the kept token was minted with other scopes than the tools take now — minting one with them", kept);
|
||||||
}
|
}
|
||||||
return this.mint(this.failed ? "no token held since the last mint failed — minting one" : "no token kept — minting one");
|
return this.mint("no token kept — minting one", null);
|
||||||
}
|
}
|
||||||
|
|
||||||
async renew(rejected: string): Promise<string> {
|
async renew(rejected: string): Promise<string> {
|
||||||
@@ -195,7 +214,7 @@ export class MintedToken implements TokenSource {
|
|||||||
return kept;
|
return kept;
|
||||||
}
|
}
|
||||||
this.held = null;
|
this.held = null;
|
||||||
return this.mint("the forge rejected the kept token — minting a fresh one");
|
return this.mint("the forge rejected the kept token — minting a fresh one", rejected);
|
||||||
}
|
}
|
||||||
|
|
||||||
/** What the forge's scoped tokens say when a kept token predates a scope the tools now need. */
|
/** What the forge's scoped tokens say when a kept token predates a scope the tools now need. */
|
||||||
@@ -203,28 +222,95 @@ export class MintedToken implements TokenSource {
|
|||||||
return status === 403 && /required scope/i.test(body);
|
return status === 403 && /required scope/i.test(body);
|
||||||
}
|
}
|
||||||
|
|
||||||
/** One mint at a time: concurrent first calls share it, rather than each minting its own. */
|
/**
|
||||||
private mint(why: string): Promise<string> {
|
* One mint at a time: concurrent first calls in this process share it, and a mint in another process
|
||||||
|
* on this machine is waited for (the lock file). Once the lock is held, the kept file is read again:
|
||||||
|
* a token there that is not `stale` (the one rejected, or kept with other scopes) and carries the
|
||||||
|
* scopes asked for was minted by the process that held the lock before — it is reused.
|
||||||
|
*/
|
||||||
|
private mint(why: string, stale: string | null): Promise<string> {
|
||||||
if (this.inflight === null) {
|
if (this.inflight === null) {
|
||||||
// A mint that failed a moment ago is not asked of the forge again yet (novox/hq issue 739): its error stands.
|
|
||||||
if (this.failed !== null && this.now() - this.failed.at < MINT_RETRY_AFTER_MS) return Promise.reject(this.failed.error);
|
|
||||||
this.log(why);
|
this.log(why);
|
||||||
this.inflight = this.doMint().then(
|
this.inflight = this.locked(async () => {
|
||||||
(token) => {
|
const kept = this.read();
|
||||||
this.failed = null;
|
if (kept !== null && kept !== stale && this.keptScopes() === this.scopesLine()) {
|
||||||
return token;
|
this.held = kept;
|
||||||
},
|
this.log(`another process minted it meanwhile — reusing the token kept at ${this.opts.file}`);
|
||||||
(err) => {
|
return kept;
|
||||||
this.failed = { at: this.now(), error: err };
|
}
|
||||||
throw err;
|
return this.doMint();
|
||||||
},
|
}).finally(() => {
|
||||||
).finally(() => {
|
|
||||||
this.inflight = null;
|
this.inflight = null;
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
return this.inflight;
|
return this.inflight;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** How long a mint may hold the lock before another process takes it as left behind, and how long one waits. */
|
||||||
|
static readonly LOCK_STALE_MS = 120_000;
|
||||||
|
static readonly LOCK_WAIT_MS = 150_000;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Run `work` holding the lock file beside the kept token: created exclusively, holding this process's
|
||||||
|
* pid, the time and a nonce; removed after, only when it is still ours. A lock whose process is gone,
|
||||||
|
* or older than LOCK_STALE_MS, was left by a process that died mid-mint and is taken over — never
|
||||||
|
* waited on for ever, which would be the same lock-out by another road.
|
||||||
|
*/
|
||||||
|
private async locked<T>(work: () => Promise<T>): Promise<T> {
|
||||||
|
const lock = `${this.opts.file}.lock`;
|
||||||
|
mkdirSync(dirname(this.opts.file), { recursive: true, mode: 0o700 });
|
||||||
|
const mine = `${process.pid} ${Date.now()} ${randomBytes(8).toString("hex")}\n`;
|
||||||
|
const deadline = Date.now() + MintedToken.LOCK_WAIT_MS;
|
||||||
|
for (;;) {
|
||||||
|
try {
|
||||||
|
const fd = openSync(lock, "wx", 0o600);
|
||||||
|
try { writeSync(fd, mine); } finally { closeSync(fd); }
|
||||||
|
break;
|
||||||
|
} catch (err) {
|
||||||
|
if ((err as NodeJS.ErrnoException).code !== "EEXIST") {
|
||||||
|
throw new Error(`cannot take the mint lock at ${lock}: ${(err as Error).message}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (this.leftBehind(lock)) {
|
||||||
|
this.log(`the mint lock at ${lock} was left by a process that is gone — taking it over`);
|
||||||
|
try { unlinkSync(lock); } catch { /* another waiter took it over first */ }
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if (Date.now() > deadline) {
|
||||||
|
throw new Error(`another process has held the mint lock at ${lock} for over ${MintedToken.LOCK_WAIT_MS / 1000}s — not minting beside it`);
|
||||||
|
}
|
||||||
|
await new Promise((r) => setTimeout(r, 50 + Math.floor(Math.random() * 50)));
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
return await work();
|
||||||
|
} finally {
|
||||||
|
try {
|
||||||
|
if (readFileSync(lock, "utf8") === mine) unlinkSync(lock);
|
||||||
|
} catch { /* already gone */ }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Whether the lock's holder is gone: its pid answers no signal, or it is older than LOCK_STALE_MS. */
|
||||||
|
private leftBehind(lock: string): boolean {
|
||||||
|
let text: string;
|
||||||
|
let age: number;
|
||||||
|
try {
|
||||||
|
text = readFileSync(lock, "utf8");
|
||||||
|
age = Date.now() - statSync(lock).mtimeMs;
|
||||||
|
} catch {
|
||||||
|
return false; // released meanwhile: the next attempt takes it
|
||||||
|
}
|
||||||
|
if (age > MintedToken.LOCK_STALE_MS) return true;
|
||||||
|
const pid = Number(text.split(" ")[0]);
|
||||||
|
if (!Number.isInteger(pid) || pid <= 0) return age > 1_000; // half-written, or not ours: give it a moment
|
||||||
|
try {
|
||||||
|
process.kill(pid, 0);
|
||||||
|
return false;
|
||||||
|
} catch (err) {
|
||||||
|
return (err as NodeJS.ErrnoException).code === "ESRCH";
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
private read(): string | null {
|
private read(): string | null {
|
||||||
try {
|
try {
|
||||||
const token = readFileSync(this.opts.file, "utf8").replace(/\n$/, "");
|
const token = readFileSync(this.opts.file, "utf8").replace(/\n$/, "");
|
||||||
@@ -235,9 +321,10 @@ export class MintedToken implements TokenSource {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/** The scopes as kept beside the token: sorted, one line. */
|
/** The scopes as kept beside the token: sorted, one line, and whose it is when it is not the admin's. */
|
||||||
private scopesLine(): string {
|
private scopesLine(): string {
|
||||||
return [...this.scopes].sort().join(",");
|
const scopes = [...this.scopes].sort().join(",");
|
||||||
|
return this.account === this.opts.admin ? scopes : `${scopes} for ${this.account}`;
|
||||||
}
|
}
|
||||||
|
|
||||||
/** The scopes the kept token was minted with, "" when not kept (a token kept before they were). */
|
/** The scopes the kept token was minted with, "" when not kept (a token kept before they were). */
|
||||||
@@ -260,17 +347,20 @@ export class MintedToken implements TokenSource {
|
|||||||
renameSync(`${this.opts.file}.scopes.tmp`, `${this.opts.file}.scopes`);
|
renameSync(`${this.opts.file}.scopes.tmp`, `${this.opts.file}.scopes`);
|
||||||
}
|
}
|
||||||
|
|
||||||
private async doMint(): Promise<string> {
|
/** One call to the forge's API with the admin login, the password read now. */
|
||||||
let password: string;
|
private password(): string {
|
||||||
try {
|
try {
|
||||||
password = readFileSync(this.opts.passwordFile, "utf8").replace(/\n$/, "");
|
return readFileSync(this.opts.passwordFile, "utf8").replace(/\n$/, "");
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
throw new Error(`cannot read the admin password at ${this.opts.passwordFile}: ${(err as Error).message}`);
|
throw new Error(`cannot read the admin password at ${this.opts.passwordFile}: ${(err as Error).message}`);
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private adminCall(password = this.password()): AdminCall {
|
||||||
const authorization = "Basic " + Buffer.from(`${this.opts.admin}:${password}`).toString("base64");
|
const authorization = "Basic " + Buffer.from(`${this.opts.admin}:${password}`).toString("base64");
|
||||||
const tokens = `${this.opts.url.replace(/\/+$/, "")}/api/v1/users/${encodeURIComponent(this.opts.admin)}/tokens`;
|
const api = `${this.opts.url.replace(/\/+$/, "")}/api/v1`;
|
||||||
const call = async (method: string, path = "", body?: unknown): Promise<{ status: number; body: any }> => {
|
const admin: AdminCall = async (method, path, body) => {
|
||||||
const res = await this.fetchImpl(tokens + path, {
|
const res = await this.fetchImpl(api + path, {
|
||||||
method,
|
method,
|
||||||
headers: { "Content-Type": "application/json", Authorization: authorization },
|
headers: { "Content-Type": "application/json", Authorization: authorization },
|
||||||
...(body === undefined ? {} : { body: JSON.stringify(body) }),
|
...(body === undefined ? {} : { body: JSON.stringify(body) }),
|
||||||
@@ -282,56 +372,96 @@ export class MintedToken implements TokenSource {
|
|||||||
}
|
}
|
||||||
return { status: res.status, body: parsed };
|
return { status: res.status, body: parsed };
|
||||||
};
|
};
|
||||||
|
return admin;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Where a retirement of the admin's old token that failed is remembered, to be tried again (retireAdmin). */
|
||||||
|
private get retireFile(): string {
|
||||||
|
return `${this.opts.file}.retire-admin`;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The admin's own token by this name goes: what the tools used before must not outlive the move (novox/hq ADR
|
||||||
|
* 0365 §2). Run after the account's token is minted and kept, so a failure here never leaves the tools without
|
||||||
|
* a token: it is said, remembered beside the kept token, and tried again when a process next reads that token.
|
||||||
|
*/
|
||||||
|
private async retireAdmin(admin: AdminCall = this.adminCall()): Promise<void> {
|
||||||
|
if (this.account === this.opts.admin) return;
|
||||||
|
try {
|
||||||
|
const retired = await this.dropNamed(admin, `/users/${encodeURIComponent(this.opts.admin)}/tokens`, this.opts.admin);
|
||||||
|
if (retired > 0) this.log(`deleted ${retired} token(s) named "${this.name}" the admin account "${this.opts.admin}" held: the tools act as "${this.account}" now`);
|
||||||
|
rmSync(this.retireFile, { force: true });
|
||||||
|
} catch (err) {
|
||||||
|
writeFileSync(this.retireFile, `${new Date().toISOString()}\n`, { mode: 0o600 });
|
||||||
|
this.log(`FAILED to delete the token(s) named "${this.name}" the admin account "${this.opts.admin}" holds — tried again at the next start: ${(err as Error).message}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private async doMint(): Promise<string> {
|
||||||
|
const password = this.password();
|
||||||
|
const admin = this.adminCall(password);
|
||||||
|
const tokensOf = (account: string) => `/users/${encodeURIComponent(account)}/tokens`;
|
||||||
|
const call = (method: string, path = "", body?: unknown) => admin(method, tokensOf(this.account) + path, body);
|
||||||
|
|
||||||
|
if (this.account !== this.opts.admin) {
|
||||||
|
// The tools' account and its team first (novox/hq ADR 0365 §2): a failure here fails the mint, tried again
|
||||||
|
// at the next call.
|
||||||
|
const me = await admin("GET", "/user");
|
||||||
|
if (me.status === 401 || me.status === 403) throw new AdminRefused(this.opts.admin, me.status);
|
||||||
|
await this.opts.setUp?.(password);
|
||||||
|
}
|
||||||
|
|
||||||
let res = await call("POST", "", { name: this.name, scopes: this.scopes });
|
let res = await call("POST", "", { name: this.name, scopes: this.scopes });
|
||||||
if (res.status === 401 || res.status === 403) throw new AdminRefused(this.opts.admin, res.status);
|
if (res.status === 401 || res.status === 403) throw new AdminRefused(this.opts.admin, res.status);
|
||||||
if (res.status === 400 || res.status === 422) {
|
if (res.status === 400 || res.status === 422) {
|
||||||
// The forge still holds a token by this name whose value we no longer have — the kept file
|
// The forge still holds a token by this name whose value we no longer have — the kept file
|
||||||
// went while the forge's data stayed. It is ours to replace: drop it by name and mint again.
|
// went while the forge's data stayed, or two processes once minted at once and left two. It is
|
||||||
this.log(`the forge already holds a token named "${this.name}" — replacing it`);
|
// ours to replace: find every one by this name in the account's list and delete each by its id
|
||||||
const dropped = await call("DELETE", `/${encodeURIComponent(this.name)}`);
|
// (gitea refuses a delete by a name held twice: "multiple matches for token name"), then mint.
|
||||||
if (dropped.status === 422) {
|
await this.dropNamed(admin, tokensOf(this.account), this.account, true);
|
||||||
// More than one token carries the name (novox/hq issue 739): two processes minted at once, and the forge's
|
|
||||||
// check that a name is unused is not atomic. The forge refuses a delete by such a name for good, so every
|
|
||||||
// one of them is dropped by its id — all are ours, and none of their values is kept.
|
|
||||||
await this.dropEveryNamed(call, dropped);
|
|
||||||
} else if (dropped.status !== 204 && dropped.status !== 404) {
|
|
||||||
throw new Error(`Gitea DELETE /users/${this.opts.admin}/tokens/${this.name}: ${dropped.status} ${detail(dropped.body)}`);
|
|
||||||
}
|
|
||||||
res = await call("POST", "", { name: this.name, scopes: this.scopes });
|
res = await call("POST", "", { name: this.name, scopes: this.scopes });
|
||||||
}
|
}
|
||||||
if (res.status !== 201 && res.status !== 200) {
|
if (res.status !== 201 && res.status !== 200) {
|
||||||
throw new Error(`Gitea POST /users/${this.opts.admin}/tokens: ${res.status} ${detail(res.body)}`);
|
throw new Error(`Gitea POST /users/${this.account}/tokens: ${res.status} ${detail(res.body)}`);
|
||||||
}
|
}
|
||||||
const token = typeof res.body?.sha1 === "string" ? res.body.sha1 : null;
|
const token = typeof res.body?.sha1 === "string" ? res.body.sha1 : null;
|
||||||
if (!token) throw new Error(`Gitea POST /users/${this.opts.admin}/tokens: ${res.status} but no token in the reply`);
|
if (!token) throw new Error(`Gitea POST /users/${this.account}/tokens: ${res.status} but no token in the reply`);
|
||||||
|
|
||||||
this.keep(token);
|
this.keep(token);
|
||||||
this.held = token;
|
this.held = token;
|
||||||
this.log(`minted a token for "${this.opts.admin}" (${this.scopes.join(", ")}), kept at ${this.opts.file}`);
|
this.log(`minted a token for "${this.account}" (${this.scopes.join(", ")}), kept at ${this.opts.file}`);
|
||||||
|
// Only now, with the account's own token kept, does the admin's old one go.
|
||||||
|
await this.retireAdmin(admin);
|
||||||
return token;
|
return token;
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Drop, by id, every token of the admin account that carries this name; the forge listed them, page by page. */
|
/**
|
||||||
private async dropEveryNamed(call: (method: string, path?: string) => Promise<{ status: number; body: any }>,
|
* Delete every token by this name an account holds, each by its id (gitea refuses a delete by a name held twice:
|
||||||
refused: { status: number; body: any }): Promise<void> {
|
* "multiple matches for token name"); answers how many. `replacing` says the line for a mint that found its name
|
||||||
const ids: number[] = [];
|
* taken.
|
||||||
for (let page = 1; page <= 20; page++) {
|
*/
|
||||||
const listed = await call("GET", `?page=${page}&limit=50`);
|
private async dropNamed(admin: AdminCall, tokens: string, account: string, replacing = false): Promise<number> {
|
||||||
|
const held: number[] = [];
|
||||||
|
for (let page = 1; page <= 100; page++) {
|
||||||
|
const listed = await admin("GET", `${tokens}?page=${page}&limit=50`);
|
||||||
if (listed.status !== 200 || !Array.isArray(listed.body)) {
|
if (listed.status !== 200 || !Array.isArray(listed.body)) {
|
||||||
throw new Error(`Gitea DELETE /users/${this.opts.admin}/tokens/${this.name}: ${refused.status} ${detail(refused.body)}; ` +
|
throw new Error(`Gitea GET /users/${account}/tokens: ${listed.status} ${detail(listed.body)}`);
|
||||||
`and its tokens could not be listed to drop them one by one: ${listed.status} ${detail(listed.body)}`);
|
|
||||||
}
|
}
|
||||||
for (const t of listed.body) if (t?.name === this.name && Number.isInteger(t?.id)) ids.push(t.id);
|
if (listed.body.length === 0) break;
|
||||||
if (listed.body.length < 50) break;
|
for (const t of listed.body) if (t?.name === this.name && typeof t.id === "number") held.push(t.id);
|
||||||
}
|
}
|
||||||
this.log(`the forge holds ${ids.length} tokens named "${this.name}" — dropping each by its id`);
|
if (replacing) {
|
||||||
for (const id of ids) {
|
this.log(held.length === 1
|
||||||
const dropped = await call("DELETE", `/${id}`);
|
? `the forge already holds a token named "${this.name}" — replacing it`
|
||||||
|
: `the forge holds ${held.length} tokens named "${this.name}" — deleting each by its id and minting one`);
|
||||||
|
}
|
||||||
|
for (const id of held) {
|
||||||
|
const dropped = await admin("DELETE", `${tokens}/${id}`);
|
||||||
if (dropped.status !== 204 && dropped.status !== 404) {
|
if (dropped.status !== 204 && dropped.status !== 404) {
|
||||||
throw new Error(`Gitea DELETE /users/${this.opts.admin}/tokens/${id}: ${dropped.status} ${detail(dropped.body)}`);
|
throw new Error(`Gitea DELETE /users/${account}/tokens/${id} (named ${this.name}): ${dropped.status} ${detail(dropped.body)}`);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
return held.length;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,52 @@
|
|||||||
|
// The forge's tools act as an account of their own, not as the site administrator (novox/hq ADR 0365 §2,
|
||||||
|
// issue 469).
|
||||||
|
//
|
||||||
|
// The administrator's token reached everything the forge holds, so the tools' boundary rested on their own guards
|
||||||
|
// (guard.ts): a protection edit, say, the forge itself would have taken. Now the tools' token is minted for a plain
|
||||||
|
// account, `mesh-tools`, which is no administrator and writes to the mesh's repositories through one team of the
|
||||||
|
// mesh's organisation:
|
||||||
|
//
|
||||||
|
// - **the account**: made by the module with the administrator's login (that login sets the forge up and does
|
||||||
|
// nothing else), never an administrator — one found made an administrator is made a plain account again. Its
|
||||||
|
// password is random, sent once and kept nowhere: the account is reached only by the tokens the administrator
|
||||||
|
// mints for it (gitea lets an administrator mint for another account: `reqSelfOrAdmin`).
|
||||||
|
// - **the team**: `mesh-tools` in the organisation `novox`, on every repository of it, writing code (pushes to a
|
||||||
|
// branch, commit statuses, merges), issues (notes, comments, labels) and pull requests. No administration of a
|
||||||
|
// repository, so the forge refuses the account a protection edit, a repository's settings and its deletion;
|
||||||
|
// no repository made in the organisation, since its maker would be that repository's administrator.
|
||||||
|
//
|
||||||
|
// Every step is idempotent (GiteaAdmin's, client.ts), and run with the administrator's login before each mint of the tools' token, so a team
|
||||||
|
// or a membership lost behind the module's back is made again at the next mint.
|
||||||
|
|
||||||
|
/** The account the tools act as. */
|
||||||
|
export const TOOLS_ACCOUNT = "mesh-tools";
|
||||||
|
/** The organisation whose repositories are the mesh's, and the team through which the tools' account writes there. */
|
||||||
|
export const TOOLS_ORG = "novox";
|
||||||
|
export const TOOLS_TEAM = "mesh-tools";
|
||||||
|
|
||||||
|
/** What the team grants: write on code, issues and pull requests of every repository of the organisation, and no more. */
|
||||||
|
export const TOOLS_TEAM_UNITS = {
|
||||||
|
permission: "write",
|
||||||
|
units_map: { "repo.code": "write", "repo.issues": "write", "repo.pulls": "write" },
|
||||||
|
includes_all_repositories: true,
|
||||||
|
can_create_org_repo: false,
|
||||||
|
} as const;
|
||||||
|
|
||||||
|
/** One call to the forge's API with the administrator's login: a path under /api/v1, the status and body answered. */
|
||||||
|
export type AdminCall = (method: string, path: string, body?: unknown) => Promise<{ status: number; body: any }>;
|
||||||
|
|
||||||
|
/** What setting the account up needs of the forge's admin API: GiteaAdmin's own (client.ts), shared with the
|
||||||
|
* package provisioner, so a create race (422) and a user made by hand are handled once. */
|
||||||
|
export interface ToolsForge {
|
||||||
|
ensureUser(username: string, password: string, email: string, plain?: boolean): Promise<void>;
|
||||||
|
ensureTeamWith(org: string, team: string, units: Record<string, unknown>): Promise<number>;
|
||||||
|
addUserToTeam(teamId: number, username: string): Promise<void>;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Make the tools' account, plain, a member of its team on every repository of the organisation. Its password is
|
||||||
|
* set afresh each time and kept nowhere: the account is reached only by the tokens the administrator mints. */
|
||||||
|
export async function setUpToolsAccount(forge: ToolsForge, account = TOOLS_ACCOUNT, password: () => string): Promise<void> {
|
||||||
|
await forge.ensureUser(account, password(), `${account}@noreply.localhost`, true);
|
||||||
|
const team = await forge.ensureTeamWith(TOOLS_ORG, TOOLS_TEAM, { name: TOOLS_TEAM, ...TOOLS_TEAM_UNITS });
|
||||||
|
await forge.addUserToTeam(team, account);
|
||||||
|
}
|
||||||
@@ -10,7 +10,7 @@
|
|||||||
|
|
||||||
import { registerModuleTools, type ToolDefinition } from "@novox/mesh-sdk/tools";
|
import { registerModuleTools, type ToolDefinition } from "@novox/mesh-sdk/tools";
|
||||||
import { emit } from "@novox/mesh-sdk/events";
|
import { emit } from "@novox/mesh-sdk/events";
|
||||||
import { GiteaClient, readyAuthors, type BranchProtection } from "../client.js";
|
import { GiteaAdmin, GiteaClient, readyAuthors, type BranchProtection } from "../client.js";
|
||||||
import { appendNote, run, viewBody, viewComment } from "../delivery.js";
|
import { appendNote, run, viewBody, viewComment } from "../delivery.js";
|
||||||
import { containedFrom, isCommit, mergeableUnder, newestByContext } from "../checks.js";
|
import { containedFrom, isCommit, mergeableUnder, newestByContext } from "../checks.js";
|
||||||
import {
|
import {
|
||||||
@@ -59,13 +59,13 @@ function namesChecked(tools: ToolDefinition[]): ToolDefinition[] {
|
|||||||
}));
|
}));
|
||||||
}
|
}
|
||||||
|
|
||||||
export function getGiteaTools(gitea: GiteaClient): ToolDefinition[] {
|
export function getGiteaTools(gitea: GiteaClient, admin?: () => GiteaAdmin): ToolDefinition[] {
|
||||||
return namesChecked(allGiteaTools(gitea));
|
return namesChecked(allGiteaTools(gitea, admin));
|
||||||
}
|
}
|
||||||
|
|
||||||
// gitea_merge_pull_request is served from the watcher's bundle (merge.ts, index.ts), beside the merge check's one
|
// gitea_merge_pull_request is served from the watcher's bundle (merge.ts, index.ts), beside the merge check's one
|
||||||
// writer (novox/hq issue 456, ADR 0323).
|
// writer (novox/hq issue 456, ADR 0323).
|
||||||
function allGiteaTools(gitea: GiteaClient): ToolDefinition[] {
|
function allGiteaTools(gitea: GiteaClient, admin?: () => GiteaAdmin): ToolDefinition[] {
|
||||||
return [
|
return [
|
||||||
// ---- Repositories ----
|
// ---- Repositories ----
|
||||||
{
|
{
|
||||||
@@ -401,8 +401,11 @@ function allGiteaTools(gitea: GiteaClient): ToolDefinition[] {
|
|||||||
},
|
},
|
||||||
run: async (args) => {
|
run: async (args) => {
|
||||||
const owner = String(args.owner), repo = String(args.repo);
|
const owner = String(args.owner), repo = String(args.repo);
|
||||||
if (!args.branch) return { rules: (await gitea.branchProtections(owner, repo)).map(summarised) };
|
// The one read made as the forge's administrator (novox/hq ADR 0375, issue 772): the rules' list is the
|
||||||
const rule = await gitea.branchProtection(owner, repo, String(args.branch));
|
// repository administrator's, which the tools' account is not. No other tool uses this account.
|
||||||
|
const rules = await (admin ? admin() : gitea).branchProtections(owner, repo);
|
||||||
|
if (!args.branch) return { rules: rules.map(summarised) };
|
||||||
|
const rule = rules.find((p) => (p.rule_name ?? p.branch_name) === String(args.branch)) ?? null;
|
||||||
return { branch: String(args.branch), rule: rule ? summarised(rule) : null };
|
return { branch: String(args.branch), rule: rule ? summarised(rule) : null };
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
@@ -504,7 +507,7 @@ function allGiteaTools(gitea: GiteaClient): ToolDefinition[] {
|
|||||||
base = base || "main";
|
base = base || "main";
|
||||||
let rule: BranchProtection | null = null, unreadable: string | undefined;
|
let rule: BranchProtection | null = null, unreadable: string | undefined;
|
||||||
try {
|
try {
|
||||||
rule = await gitea.branchProtection(owner, repo, base);
|
rule = await gitea.appliedProtection(owner, repo, base);
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
unreadable = err instanceof Error ? err.message : String(err);
|
unreadable = err instanceof Error ? err.message : String(err);
|
||||||
}
|
}
|
||||||
@@ -589,8 +592,12 @@ function allGiteaTools(gitea: GiteaClient): ToolDefinition[] {
|
|||||||
const { owner, repo, branch, newBranch } = judged.contents;
|
const { owner, repo, branch, newBranch } = judged.contents;
|
||||||
const found = await gitea.api<{ default_branch?: string }>(`/repos/${checkName("owner", owner)}/${checkName("name", repo)}`);
|
const found = await gitea.api<{ default_branch?: string }>(`/repos/${checkName("owner", owner)}/${checkName("name", repo)}`);
|
||||||
const defaultBranch = String(found?.default_branch ?? "main");
|
const defaultBranch = String(found?.default_branch ?? "main");
|
||||||
const refused = judgeContentsWrite(newBranch ?? branch ?? defaultBranch, defaultBranch,
|
// The protection the forge applies to the branch written, read from the branch: the rules' own list is a
|
||||||
await gitea.branchProtections(owner, repo));
|
// repository administrator's, which the tools' account is not (novox/hq ADR 0365). A branch not made yet is
|
||||||
|
// left to the forge, which refuses a write that would make a branch a rule guards.
|
||||||
|
const target = newBranch ?? branch ?? defaultBranch;
|
||||||
|
const applied = target === defaultBranch ? null : await gitea.appliedProtection(owner, repo, target);
|
||||||
|
const refused = judgeContentsWrite(target, defaultBranch, applied ? [applied] : []);
|
||||||
if (refused) throw new Refused(refused);
|
if (refused) throw new Refused(refused);
|
||||||
}
|
}
|
||||||
// The path judged is the path sent: normalised, with only the allow-listed query keys (guard.ts).
|
// The path judged is the path sent: normalised, with only the allow-listed query keys (guard.ts).
|
||||||
@@ -611,7 +618,7 @@ function allGiteaTools(gitea: GiteaClient): ToolDefinition[] {
|
|||||||
// way, gitea contributes none rather than failing the whole runtime, and says why.
|
// way, gitea contributes none rather than failing the whole runtime, and says why.
|
||||||
registerModuleTools("gitea", (env) => {
|
registerModuleTools("gitea", (env) => {
|
||||||
try {
|
try {
|
||||||
return getGiteaTools(GiteaClient.fromEnv(env));
|
return getGiteaTools(GiteaClient.fromEnv(env), () => GiteaAdmin.fromEnv(env));
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
console.log(`[gitea] no tools — ${err instanceof Error ? err.message : String(err)}`);
|
console.log(`[gitea] no tools — ${err instanceof Error ? err.message : String(err)}`);
|
||||||
return [];
|
return [];
|
||||||
|
|||||||
+241
-36
@@ -140,8 +140,11 @@ export interface ReceiverOptions {
|
|||||||
publish: (event: ForgeEvent) => Promise<void>;
|
publish: (event: ForgeEvent) => Promise<void>;
|
||||||
/** Acts on it at once, beside publishing it (the announcer). */
|
/** Acts on it at once, beside publishing it (the announcer). */
|
||||||
act?: (event: ForgeEvent) => Promise<void>;
|
act?: (event: ForgeEvent) => Promise<void>;
|
||||||
/** Told of every verified delivery, so the polls know events are heard. */
|
/** Told of every verified delivery — its kind (X-Gitea-Event) and its delivery id — so the polls know events are
|
||||||
heard?: () => void;
|
* heard, and the module can say when it last heard one. */
|
||||||
|
heard?: (kind: string, delivery: string) => void;
|
||||||
|
/** Told of every refused delivery, with the reason and where it came from. */
|
||||||
|
refused?: (reason: string, from: string) => void;
|
||||||
log?: (line: string) => void;
|
log?: (line: string) => void;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -158,6 +161,10 @@ export const REMEMBERED = 2000;
|
|||||||
*/
|
*/
|
||||||
export function receiver(opts: ReceiverOptions): Server & { idle(): Promise<void> } {
|
export function receiver(opts: ReceiverOptions): Server & { idle(): Promise<void> } {
|
||||||
const log = opts.log ?? ((l) => console.log(`[gitea] ${l}`));
|
const log = opts.log ?? ((l) => console.log(`[gitea] ${l}`));
|
||||||
|
const refuse = (from: string, reason: string): void => {
|
||||||
|
log(`refused a forge event from ${from}: ${reason}`);
|
||||||
|
opts.refused?.(reason, from);
|
||||||
|
};
|
||||||
const seen = new Set<string>();
|
const seen = new Set<string>();
|
||||||
let queue: Promise<void> = Promise.resolve();
|
let queue: Promise<void> = Promise.resolve();
|
||||||
const server = createServer((req, res) => {
|
const server = createServer((req, res) => {
|
||||||
@@ -168,14 +175,14 @@ export function receiver(opts: ReceiverOptions): Server & { idle(): Promise<void
|
|||||||
const from = req.socket.remoteAddress ?? "?";
|
const from = req.socket.remoteAddress ?? "?";
|
||||||
const signature = String(req.headers["x-gitea-signature"] ?? "").trim();
|
const signature = String(req.headers["x-gitea-signature"] ?? "").trim();
|
||||||
if (!signature) {
|
if (!signature) {
|
||||||
log(`refused a forge event from ${from}: it is unsigned`);
|
refuse(from, "it is unsigned");
|
||||||
res.writeHead(401, { Connection: "close" });
|
res.writeHead(401, { Connection: "close" });
|
||||||
res.end("unsigned\n");
|
res.end("unsigned\n");
|
||||||
req.destroy();
|
req.destroy();
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
if (Number(req.headers["content-length"] ?? 0) > MAX_BODY) {
|
if (Number(req.headers["content-length"] ?? 0) > MAX_BODY) {
|
||||||
log(`refused a forge event from ${from}: its body is over ${MAX_BODY} bytes`);
|
refuse(from, `its body is over ${MAX_BODY} bytes`);
|
||||||
res.writeHead(413, { Connection: "close" });
|
res.writeHead(413, { Connection: "close" });
|
||||||
res.end("too large\n");
|
res.end("too large\n");
|
||||||
req.destroy();
|
req.destroy();
|
||||||
@@ -189,7 +196,7 @@ export function receiver(opts: ReceiverOptions): Server & { idle(): Promise<void
|
|||||||
if (size > MAX_BODY) {
|
if (size > MAX_BODY) {
|
||||||
if (!over) {
|
if (!over) {
|
||||||
over = true;
|
over = true;
|
||||||
log(`refused a forge event from ${from}: its body is over ${MAX_BODY} bytes`);
|
refuse(from, `its body is over ${MAX_BODY} bytes`);
|
||||||
res.writeHead(413, { Connection: "close" });
|
res.writeHead(413, { Connection: "close" });
|
||||||
res.end("too large\n");
|
res.end("too large\n");
|
||||||
req.destroy();
|
req.destroy();
|
||||||
@@ -200,7 +207,7 @@ export function receiver(opts: ReceiverOptions): Server & { idle(): Promise<void
|
|||||||
if (over) return;
|
if (over) return;
|
||||||
const body = Buffer.concat(chunks);
|
const body = Buffer.concat(chunks);
|
||||||
if (!verified(opts.secret, body, signature)) {
|
if (!verified(opts.secret, body, signature)) {
|
||||||
log(`refused a forge event from ${from}: its signature does not verify`);
|
refuse(from, "its signature does not verify");
|
||||||
res.statusCode = 401;
|
res.statusCode = 401;
|
||||||
return res.end("signature does not verify\n");
|
return res.end("signature does not verify\n");
|
||||||
}
|
}
|
||||||
@@ -208,11 +215,13 @@ export function receiver(opts: ReceiverOptions): Server & { idle(): Promise<void
|
|||||||
try {
|
try {
|
||||||
payload = JSON.parse(body.toString("utf8"));
|
payload = JSON.parse(body.toString("utf8"));
|
||||||
} catch {
|
} catch {
|
||||||
|
refuse(from, "it is signed but not JSON");
|
||||||
res.statusCode = 400;
|
res.statusCode = 400;
|
||||||
return res.end("not JSON\n");
|
return res.end("not JSON\n");
|
||||||
}
|
}
|
||||||
opts.heard?.();
|
|
||||||
const delivery = String(req.headers["x-gitea-delivery"] ?? "");
|
const delivery = String(req.headers["x-gitea-delivery"] ?? "");
|
||||||
|
const kind = String(req.headers["x-gitea-event"] ?? "other");
|
||||||
|
opts.heard?.(kind, delivery);
|
||||||
if (delivery && seen.has(delivery)) {
|
if (delivery && seen.has(delivery)) {
|
||||||
res.statusCode = 200;
|
res.statusCode = 200;
|
||||||
return res.end("already heard\n");
|
return res.end("already heard\n");
|
||||||
@@ -221,7 +230,7 @@ export function receiver(opts: ReceiverOptions): Server & { idle(): Promise<void
|
|||||||
seen.add(delivery);
|
seen.add(delivery);
|
||||||
if (seen.size > REMEMBERED) seen.delete(seen.values().next().value as string);
|
if (seen.size > REMEMBERED) seen.delete(seen.values().next().value as string);
|
||||||
}
|
}
|
||||||
const event = eventOf(String(req.headers["x-gitea-event"] ?? "other"), payload);
|
const event = eventOf(kind, payload);
|
||||||
res.statusCode = 200;
|
res.statusCode = 200;
|
||||||
res.end("queued\n");
|
res.end("queued\n");
|
||||||
queue = queue.then(async () => {
|
queue = queue.then(async () => {
|
||||||
@@ -242,27 +251,113 @@ export interface HookAdmin {
|
|||||||
call(method: string, path: string, body?: unknown): Promise<{ status: number; body: any }>;
|
call(method: string, path: string, body?: unknown): Promise<{ status: number; body: any }>;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** The forge's one system webhook as ensured at start: read back from the forge after it was made. */
|
||||||
|
export interface HookState {
|
||||||
|
/** Always "system": a hook the forge's listing of system webhooks names, so it fires for every repository. */
|
||||||
|
kind: "system";
|
||||||
|
id: number;
|
||||||
|
url: string;
|
||||||
|
events: string[];
|
||||||
|
active: boolean;
|
||||||
|
content_type?: string;
|
||||||
|
/** Made where none was, or made in place of the ones there (system, default, or a repository's) at the receiver's url. */
|
||||||
|
did: "created" | "replaced";
|
||||||
|
/** The hooks at the receiver's url it took the place of, by kind: a repository's is a copy of a default webhook, which
|
||||||
|
* would have sent that repository's events a second time. */
|
||||||
|
removed: { system: number; default: number; repository: number; repositories: string[] };
|
||||||
|
}
|
||||||
|
|
||||||
|
/** How long without a verified delivery — since the start, or since the last one — a forge is said silent once the poll
|
||||||
|
* finds what no event said. */
|
||||||
|
export const SILENT_AFTER = 10 * 60_000;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* **How fast the polls look** (issue 729): every five minutes only while the forge's events are heard — a verified
|
* **How fast the polls look, and whether the forge's events are heard** (issue 729): every five minutes only while the
|
||||||
* delivery within the last ten minutes, and none missed since. Until the first delivery, after ten minutes without one,
|
* forge's events are heard — a verified delivery within the last ten minutes, and none missed since. Until the first
|
||||||
* or once the poll found something no event said, every thirty seconds.
|
* delivery, after ten minutes without one, or once the poll found something no event said, every thirty seconds.
|
||||||
|
*
|
||||||
|
* It also keeps what `gitea_events` says: where the receiver listens, the hook as ensured, the verified and refused
|
||||||
|
* deliveries, and the polls' finds no event said. **A forge whose events stop is said** (issue 729): once the receiver
|
||||||
|
* listens and the hook is ensured, when nothing verified for ten minutes — since the start, or since the last delivery —
|
||||||
|
* and the poll has found at least two heads or merges no event said in that quiet (one could be the poll beating an
|
||||||
|
* event by a second), it is logged loudly once per silent spell, and the module's events are said unhealthy until a
|
||||||
|
* delivery verifies. Deliveries refused since the last verified one are said unhealthy too.
|
||||||
*/
|
*/
|
||||||
export class Pace {
|
export class Pace {
|
||||||
private last = 0;
|
private last = 0;
|
||||||
private missed = 0;
|
private missed = 0;
|
||||||
|
private readonly started: number;
|
||||||
private readonly now: () => number;
|
private readonly now: () => number;
|
||||||
private readonly log: (l: string) => void;
|
private readonly log: (l: string) => void;
|
||||||
|
private where: { host: string; port: number; url: string } | null = null;
|
||||||
|
private hook: HookState | null = null;
|
||||||
|
private notHeard: string | null = null;
|
||||||
|
private notHeardFails = false;
|
||||||
|
private verifiedCount = 0;
|
||||||
|
private lastVerified: { at: number; event: string; delivery: string } | null = null;
|
||||||
|
private refusedCount = 0;
|
||||||
|
private refusedSpell = 0;
|
||||||
|
private lastRefused: { at: number; reason: string; from: string } | null = null;
|
||||||
|
private missCount = 0;
|
||||||
|
private lastMiss: { at: number; what: string } | null = null;
|
||||||
|
private silentSince = 0;
|
||||||
|
private quietMisses = 0;
|
||||||
constructor(now: () => number = Date.now, log: (l: string) => void = (l) => console.error(`[gitea] ${l}`)) {
|
constructor(now: () => number = Date.now, log: (l: string) => void = (l) => console.error(`[gitea] ${l}`)) {
|
||||||
this.now = now;
|
this.now = now;
|
||||||
this.log = log;
|
this.log = log;
|
||||||
|
this.started = now();
|
||||||
}
|
}
|
||||||
heard(): void {
|
/** The receiver listens, and the hook at it was ensured: from now on, deliveries are expected. */
|
||||||
|
listening(host: string, port: number, url: string, hook: HookState): void {
|
||||||
|
this.where = { host, port, url };
|
||||||
|
this.hook = hook;
|
||||||
|
this.notHeard = null;
|
||||||
|
}
|
||||||
|
/** No events are listened for, and why: by configuration (`fails` false), or because the receiver could not listen
|
||||||
|
* or the hook could not be ensured (`fails` true, said unhealthy). The polls alone watch. */
|
||||||
|
unheard(why: string, fails = true): void {
|
||||||
|
this.notHeard = why;
|
||||||
|
this.notHeardFails = fails;
|
||||||
|
}
|
||||||
|
heard(event = "", delivery = ""): void {
|
||||||
this.last = this.now();
|
this.last = this.now();
|
||||||
|
this.verifiedCount++;
|
||||||
|
this.lastVerified = { at: this.last, event, delivery };
|
||||||
|
this.refusedSpell = 0;
|
||||||
|
this.quietMisses = 0;
|
||||||
|
if (this.silentSince) this.log(`the forge's events are heard again: ${event || "a delivery"} verified`);
|
||||||
|
this.silentSince = 0;
|
||||||
|
}
|
||||||
|
refused(reason: string, from: string): void {
|
||||||
|
this.refusedCount++;
|
||||||
|
this.refusedSpell++;
|
||||||
|
this.lastRefused = { at: this.now(), reason, from };
|
||||||
}
|
}
|
||||||
/** The poll found a head or merge no forge event announced. */
|
/** The poll found a head or merge no forge event announced. */
|
||||||
miss(what: string): void {
|
miss(what: string): void {
|
||||||
|
const t = this.now();
|
||||||
if (this.hearing()) this.log(`THE FORGE'S EVENTS MISSED ${what}: the poll found it, so the polls look every 30 s again until the next event`);
|
if (this.hearing()) this.log(`THE FORGE'S EVENTS MISSED ${what}: the poll found it, so the polls look every 30 s again until the next event`);
|
||||||
this.missed = this.now();
|
this.missed = t;
|
||||||
|
this.missCount++;
|
||||||
|
this.lastMiss = { at: t, what };
|
||||||
|
// Silent: nothing verified for SILENT_AFTER — never since the start, or not since the last delivery (a hook deleted by
|
||||||
|
// hand, every delivery refused) — while the poll finds what no event said. Said once per silent spell.
|
||||||
|
const quietSince = this.last || this.started;
|
||||||
|
this.quietMisses++;
|
||||||
|
if (this.expecting() && t - quietSince >= SILENT_AFTER && this.quietMisses >= 2 && !this.silentSince) {
|
||||||
|
this.silentSince = t;
|
||||||
|
const minutes = Math.round((t - quietSince) / 60_000);
|
||||||
|
const refusedSince = this.refusedSinceHeard();
|
||||||
|
this.log(`THE FORGE'S EVENTS ARE NOT HEARD: ${this.last ? `no delivery verified since the last at ` +
|
||||||
|
`${new Date(this.last).toISOString()}` : "no delivery ever verified since the start"}, ${minutes} min, from the ` +
|
||||||
|
`system webhook at ${this.hook?.url}${refusedSince ? `; ${refusedSince} refused since, the last because ` +
|
||||||
|
`${this.lastRefused?.reason}` : ""}; the poll found ${what} — the polls alone watch the forge, every 30 s ` +
|
||||||
|
`(novox/hq issue 729)`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
/** Whether a delivery was refused after the last verified one (or with none verified). */
|
||||||
|
private refusedSinceHeard(): number {
|
||||||
|
return this.lastRefused && this.lastRefused.at >= this.last ? this.refusedSpell : 0;
|
||||||
}
|
}
|
||||||
hearing(): boolean {
|
hearing(): boolean {
|
||||||
const t = this.now();
|
const t = this.now();
|
||||||
@@ -271,36 +366,146 @@ export class Pace {
|
|||||||
every(): number {
|
every(): number {
|
||||||
return this.hearing() ? 5 * 60_000 : 30_000;
|
return this.hearing() ? 5 * 60_000 : 30_000;
|
||||||
}
|
}
|
||||||
|
private expecting(): boolean {
|
||||||
|
return this.where !== null && this.hook !== null;
|
||||||
|
}
|
||||||
|
/** What `gitea_events` answers: healthy, unhealthy or waits (ADR 0283's shape), with every fact behind it. */
|
||||||
|
report(): Record<string, unknown> {
|
||||||
|
const t = this.now();
|
||||||
|
const iso = (ms: number): string => new Date(ms).toISOString();
|
||||||
|
let state: "healthy" | "unhealthy" | "waits" = "healthy";
|
||||||
|
let why: string;
|
||||||
|
const refusedSince = this.refusedSinceHeard();
|
||||||
|
if (this.silentSince) {
|
||||||
|
state = "unhealthy";
|
||||||
|
why = `${this.last ? `no delivery verified since the last at ${iso(this.last)}` : `the forge has never delivered an ` +
|
||||||
|
`event since the start at ${iso(this.started)}`}, while the poll found ${this.missCount} head(s) or merge(s) no ` +
|
||||||
|
`event said, the last ${this.lastMiss?.what} at ${iso(this.lastMiss?.at ?? t)}`;
|
||||||
|
} else if (refusedSince) {
|
||||||
|
state = "unhealthy";
|
||||||
|
why = `${refusedSince} deliveries refused ${this.last ? `since the last verified one at ${iso(this.last)}` :
|
||||||
|
"and none verified"}, the last at ${iso(this.lastRefused?.at ?? t)} because ${this.lastRefused?.reason}`;
|
||||||
|
} else if (!this.expecting()) {
|
||||||
|
why = this.notHeard ? `no events are listened for, so the polls alone watch the forge: ${this.notHeard}` :
|
||||||
|
"the receiver is not listening yet";
|
||||||
|
if (this.notHeard && this.notHeardFails) state = "unhealthy";
|
||||||
|
} else if (this.hearing()) {
|
||||||
|
why = `the forge's events are heard: ${this.verifiedCount} verified, the last at ${iso(this.last)}`;
|
||||||
|
} else if (this.last === 0) {
|
||||||
|
why = `no delivery heard yet; the forge is said silent if the poll finds what no event said ${SILENT_AFTER / 60_000} min after the start`;
|
||||||
|
} else {
|
||||||
|
why = `the last delivery was at ${iso(this.last)}; the polls look every 30 s until the next`;
|
||||||
|
}
|
||||||
|
return {
|
||||||
|
healthy: state !== "unhealthy", state, why,
|
||||||
|
started: iso(this.started),
|
||||||
|
receiver: this.where ? { listening: true, ...this.where } : { listening: false, why: this.notHeard ?? "not started" },
|
||||||
|
hook: this.hook,
|
||||||
|
verified: { count: this.verifiedCount, last: this.lastVerified && { ...this.lastVerified, at: iso(this.lastVerified.at) } },
|
||||||
|
refused: { count: this.refusedCount, since_verified: refusedSince, last: this.lastRefused && { ...this.lastRefused, at: iso(this.lastRefused.at) } },
|
||||||
|
polls: { every_seconds: this.every() / 1000, hearing: this.hearing(), found_unsaid: this.missCount,
|
||||||
|
last_found: this.lastMiss && { ...this.lastMiss, at: iso(this.lastMiss.at) } },
|
||||||
|
silent: this.silentSince ? { since: iso(this.silentSince) } : null,
|
||||||
|
};
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Every event family the forge (1.27) can send a webhook. */
|
/** Every event family the forge (1.27) can send a webhook. */
|
||||||
export const HOOK_EVENTS = ["create", "delete", "fork", "push", "issues", "issue_comment", "pull_request",
|
export const HOOK_EVENTS = ["create", "delete", "fork", "push", "issues", "issue_comment", "pull_request",
|
||||||
"pull_request_review_request", "wiki", "repository", "release", "package", "status", "workflow_run", "workflow_job"];
|
"pull_request_review_request", "wiki", "repository", "release", "package", "status", "workflow_run", "workflow_job"];
|
||||||
|
|
||||||
/**
|
async function hooksAt(admin: HookAdmin, type: "system" | "default", url: string): Promise<any[]> {
|
||||||
* **One system webhook, for every event, at the receiver** (issue 729): created when none points there, corrected
|
const listed = await admin.call("GET", `/admin/hooks?type=${type}&limit=50`);
|
||||||
* when one does but differs (events, active, content type), and its secret set again each start — the forge never
|
if (listed.status !== 200) throw new Error(`the forge's ${type} webhooks could not be listed: ${listed.status}`);
|
||||||
* hands a secret back, so a corrected secret cannot be told from a kept one. Answers what it did.
|
return (Array.isArray(listed.body) ? listed.body : []).filter((h) => h?.config?.url === url);
|
||||||
*/
|
}
|
||||||
export async function ensureSystemHook(admin: HookAdmin, url: string, secret: string): Promise<"created" | "corrected"> {
|
|
||||||
// POST /admin/hooks makes a system webhook, and its options take no flag for it (the forge's 1.27.3 API schema). The
|
/** Every page of a listing that answers an array (or `{ data: [] }`, the repository search), PAGE at a time. */
|
||||||
// forge has no "every event" word: each family is named, and a family name (issues, pull_request) takes all of it.
|
const PAGE = 50;
|
||||||
const want = { type: "gitea", active: true, events: HOOK_EVENTS, branch_filter: "*",
|
async function everyPage(admin: HookAdmin, path: string, what: string): Promise<any[]> {
|
||||||
config: { url, content_type: "json", secret } };
|
const all: any[] = [];
|
||||||
const listed = await admin.call("GET", "/admin/hooks?type=system&limit=50");
|
for (let page = 1; ; page++) {
|
||||||
if (listed.status !== 200) throw new Error(`the forge's system webhooks could not be listed: ${listed.status}`);
|
const r = await admin.call("GET", `${path}${path.includes("?") ? "&" : "?"}page=${page}&limit=${PAGE}`);
|
||||||
const hooks: any[] = Array.isArray(listed.body) ? listed.body : [];
|
if (r.status !== 200) throw new Error(`${what} could not be listed: ${r.status}`);
|
||||||
const ours = hooks.filter((h) => h?.config?.url === url);
|
const items: any[] = Array.isArray(r.body) ? r.body : Array.isArray(r.body?.data) ? r.body.data : [];
|
||||||
if (ours.length === 0) {
|
all.push(...items);
|
||||||
const made = await admin.call("POST", "/admin/hooks", want);
|
if (items.length < PAGE) return all;
|
||||||
if (made.status !== 201 && made.status !== 200) throw new Error(`the forge's system webhook could not be made: ${made.status}`);
|
|
||||||
return "created";
|
|
||||||
}
|
}
|
||||||
for (const extra of ours.slice(1)) await admin.call("DELETE", `/admin/hooks/${extra.id}`);
|
}
|
||||||
const fixed = await admin.call("PATCH", `/admin/hooks/${ours[0].id}`, { active: true, events: HOOK_EVENTS, branch_filter: "*",
|
|
||||||
config: { url, content_type: "json", secret } });
|
/** Every repository's own webhooks at the receiver's url: copies of a default webhook made at it. */
|
||||||
if (fixed.status !== 200) throw new Error(`the forge's system webhook could not be corrected: ${fixed.status}`);
|
async function repoHooksAt(admin: HookAdmin, url: string): Promise<{ repo: string; id: number }[]> {
|
||||||
return "corrected";
|
const found: { repo: string; id: number }[] = [];
|
||||||
|
for (const repo of await everyPage(admin, "/repos/search?sort=id", "the forge's repositories")) {
|
||||||
|
const full = String(repo?.full_name ?? "");
|
||||||
|
if (!full.includes("/")) continue;
|
||||||
|
const hooks = await everyPage(admin, `/repos/${full}/hooks`, `${full}'s webhooks`);
|
||||||
|
for (const h of hooks) if (h?.config?.url === url) found.push({ repo: full, id: Number(h.id) });
|
||||||
|
}
|
||||||
|
return found;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* **One system webhook, for every event, at the receiver** (issue 729), made each start and read back.
|
||||||
|
*
|
||||||
|
* The forge (1.27.3, routers/api/v1/utils/hook.go addHook) makes a *default* webhook from POST /admin/hooks unless
|
||||||
|
* `config.is_system_webhook` is "true": a default webhook fires for no repository and is only copied into repositories
|
||||||
|
* made after it, and the listing of system webhooks does not name it — so without the flag every start made another
|
||||||
|
* one and no event came. Its edit (editHook) never changes a webhook's secret nor its kind, so a hook already there is
|
||||||
|
* not corrected but replaced: the new system webhook is made first, with the secret kept now, so no event falls in a
|
||||||
|
* gap; then every other hook at the receiver's url is deleted — system, default, and each repository's copy of a
|
||||||
|
* default one, which would send that repository's events twice. The forge's listing of system webhooks must then name
|
||||||
|
* exactly one at the url, or this fails, saying so.
|
||||||
|
*/
|
||||||
|
export async function ensureSystemHook(admin: HookAdmin, url: string, secret: string): Promise<HookState> {
|
||||||
|
// The forge has no "every event" word: each family is named, and a family name (issues, pull_request) takes all of it.
|
||||||
|
const want = { type: "gitea", active: true, events: HOOK_EVENTS, branch_filter: "*",
|
||||||
|
config: { url, content_type: "json", secret, is_system_webhook: "true" } };
|
||||||
|
const before = new Set((await hooksAt(admin, "system", url)).map((h) => Number(h.id)));
|
||||||
|
const made = await admin.call("POST", "/admin/hooks", want);
|
||||||
|
if (made.status !== 201 && made.status !== 200) throw new Error(`the forge's system webhook could not be made: ${made.status}`);
|
||||||
|
const fresh = (await hooksAt(admin, "system", url)).filter((h) => !before.has(Number(h.id)));
|
||||||
|
if (fresh.length !== 1) {
|
||||||
|
throw new Error(`the forge names ${fresh.length} new system webhooks at ${url} after one was made, not one: its events ` +
|
||||||
|
`are not known to be sent`);
|
||||||
|
}
|
||||||
|
const ours = fresh[0];
|
||||||
|
const defaults = await hooksAt(admin, "default", url);
|
||||||
|
const gone = async (path: string): Promise<void> => {
|
||||||
|
const r = await admin.call("DELETE", path);
|
||||||
|
if (r.status !== 204 && r.status !== 200 && r.status !== 404) throw new Error(`the forge's webhook ${path} could not be deleted: ${r.status}`);
|
||||||
|
};
|
||||||
|
for (const id of before) await gone(`/admin/hooks/${id}`);
|
||||||
|
for (const h of defaults) await gone(`/admin/hooks/${h.id}`);
|
||||||
|
const copies = await repoHooksAt(admin, url);
|
||||||
|
for (const c of copies) await gone(`/repos/${c.repo}/hooks/${c.id}`);
|
||||||
|
const kept = await hooksAt(admin, "system", url);
|
||||||
|
if (kept.length !== 1 || Number(kept[0].id) !== Number(ours.id)) {
|
||||||
|
throw new Error(`the forge names ${kept.length} system webhooks at ${url} once the others were deleted, not the one ` +
|
||||||
|
`made (${ours.id})`);
|
||||||
|
}
|
||||||
|
const removed = { system: before.size, default: defaults.length, repository: copies.length,
|
||||||
|
repositories: [...new Set(copies.map((c) => c.repo))] };
|
||||||
|
return { kind: "system", id: Number(ours.id), url, events: Array.isArray(ours.events) ? ours.events : [],
|
||||||
|
active: !!ours.active, content_type: ours.config?.content_type,
|
||||||
|
did: removed.system + removed.default + removed.repository > 0 ? "replaced" : "created", removed };
|
||||||
|
}
|
||||||
|
|
||||||
|
/** The `gitea_events` tool: what the module knows of the forge's events, from its Pace. */
|
||||||
|
export function eventsTool(pace: Pace): { name: string; description: string; input: Record<string, unknown>;
|
||||||
|
run: (args: Readonly<Record<string, unknown>>) => Promise<unknown> } {
|
||||||
|
return {
|
||||||
|
name: "gitea_events",
|
||||||
|
description: "Whether the forge's events reach the mesh (novox/hq issue 729): where the receiver listens, the " +
|
||||||
|
"system webhook as ensured at the start (its id, url, events, kind, and what it replaced), how many deliveries " +
|
||||||
|
"verified and the last one's time and event, how many were refused and the last reason, how often the polls look " +
|
||||||
|
"and whether events are heard, and how many heads or merges the poll found that no event said. Unhealthy when the " +
|
||||||
|
"forge has delivered nothing that verified for ten minutes (since the start or since the last delivery) while the " +
|
||||||
|
"poll found what no event said, when deliveries were refused since the last verified one, or when the receiver " +
|
||||||
|
"could not listen or the webhook could not be made.",
|
||||||
|
input: {},
|
||||||
|
run: async () => pace.report(),
|
||||||
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|||||||
@@ -0,0 +1,52 @@
|
|||||||
|
# kernel
|
||||||
|
|
||||||
|
The kernel's run-time settings on a machine (novox/hq to-be 42 Phase 1, row 11). Today it holds one:
|
||||||
|
**transparent huge pages**, set per machine. The rest of the row (the kernel package, microcode, boot
|
||||||
|
entries) is not built yet.
|
||||||
|
|
||||||
|
## Why (novox/hq issue 746)
|
||||||
|
|
||||||
|
With transparent huge pages `always`, the page cache of a file mapped by a program is collapsed into
|
||||||
|
2 MiB folios (khugepaged), and a small write into such a folio dirties all of it: writeback then writes
|
||||||
|
2 MiB. journald writes its journal files through mmap, and every journal reader (fail2ban's journal
|
||||||
|
jails among them) maps them too. Measured on the control node on 2026-10-14: journald's cgroup had
|
||||||
|
written 2.5 TB in 12 days while the journal grew by about 25-30 KB/s, fail2ban's cgroup 63 GB in 4 h
|
||||||
|
while its own files did not grow, both holding hundreds of MiB of file pages in huge folios; the bursts
|
||||||
|
saturated the mirrored spinning root disk that holds the bus store and the store, and the controller
|
||||||
|
lost its lease four times in 30 minutes. With `madvise`, only memory a program asks to have in huge pages
|
||||||
|
(`madvise(MADV_HUGEPAGE)`) gets them; a database or a JVM that wants them still asks.
|
||||||
|
|
||||||
|
## What it owns
|
||||||
|
|
||||||
|
| | what | notes |
|
||||||
|
|---|---|---|
|
||||||
|
| setting | `transparent-huge-pages` | `always`, `madvise` or `never`. Default `madvise`, the value the mesh chose for every node (ADR 0381), not what the distribution's kernel boots with (`always`); a machine that needs another value is given it as its own setting, with its reason |
|
||||||
|
| file | `${dir:state}/huge-pages` | the script, written from the setting: `apply` writes the mode to `/sys/kernel/mm/transparent_hugepage/enabled` and reads it back (a value that does not take fails the unit); `restore` gives back the mode found before the module first set it, kept in `${dir:state}/found` |
|
||||||
|
| file | `/etc/systemd/system/mesh-kernel-huge-pages.service` | oneshot, early in boot (`sysinit.target`, before the services that fill the page cache); reloading it runs `apply`, stopping it runs `restore`, which also removes `found`. Sandboxed: the file system read-only but for the module's state (`/sys` stays writable), no home, no network, no new privileges; the journal's rotation goes over its local socket, which a private network leaves reachable |
|
||||||
|
| service | `mesh-kernel-huge-pages.service` | running and enabled; **reloaded** when the script (so the setting) changes, so a new value is set directly and never passes back through the found one; restarted only when the unit changes; its health is the unit's |
|
||||||
|
|
||||||
|
The per-size controls (`hugepages-<n>kB/enabled`) are left alone: on the machines read, 2048 kB says
|
||||||
|
`inherit` (it follows the mode) and every other size `never`. `kernel_huge_pages` shows each one.
|
||||||
|
|
||||||
|
**A folio already in the page cache keeps its size.** A new mode applies to pages faulted or collapsed
|
||||||
|
from then on. The active journal files would keep their 2 MiB folios until archived, so `apply` rotates
|
||||||
|
the journal (`journalctl --rotate`) whenever it moves the mode away from `always`: new files are mapped
|
||||||
|
under the new mode. A failed rotation is said in the unit's journal and does not fail the setting. No
|
||||||
|
rotation is made while the machine is still starting (`systemctl is-system-running` says `initializing`
|
||||||
|
or `starting`): nothing is in huge pages yet, and the persistent journal opens after this unit.
|
||||||
|
|
||||||
|
## Tools
|
||||||
|
|
||||||
|
| tool | | answers |
|
||||||
|
|---|---|---|
|
||||||
|
| `kernel_huge_pages` | r | the mode, defrag and shared-memory mode; each per-size control and what `inherit` resolves to; huge pages by kind from `/proc/meminfo` (MiB); the units holding the most file pages in huge folios (`memory.stat` `file_thp`) with their dirty and writeback bytes, khugepaged's collapses and the bytes their cgroup wrote (`io.stat`, the most any one device saw: a mirror and its members each count the same write) |
|
||||||
|
|
||||||
|
## Tests
|
||||||
|
|
||||||
|
`go test ./...`: the tool over a stand-in tree holding the control node's own files of 2026-10-14; the
|
||||||
|
manifest's shape (the default, the order, the unit and its sandbox, reload, restart and health); and the script as the mesh
|
||||||
|
writes it, run with `sh` against a stand-in directory: it sets and reads back, keeps what it found once,
|
||||||
|
rotates the journal once and not for `always` nor while the machine starts, goes straight to a new value on a
|
||||||
|
reload without rewriting what it found, gives back what it found and then keeps nothing, refuses a value the kernel
|
||||||
|
does not take before writing anything, fails when the write does not take, and says when the kernel has
|
||||||
|
no transparent huge pages.
|
||||||
@@ -0,0 +1,241 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"path"
|
||||||
|
"sort"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
)
|
||||||
|
|
||||||
|
// THPDir is where the kernel says and takes its transparent huge pages.
|
||||||
|
const THPDir = "sys/kernel/mm/transparent_hugepage"
|
||||||
|
|
||||||
|
// CgroupRoot is the unified cgroup hierarchy.
|
||||||
|
const CgroupRoot = "sys/fs/cgroup"
|
||||||
|
|
||||||
|
// HugePagesAnswer is what kernel_huge_pages answers.
|
||||||
|
type HugePagesAnswer struct {
|
||||||
|
// Mode is the word the kernel brackets in enabled: always, madvise or never.
|
||||||
|
Mode string `json:"mode"`
|
||||||
|
Defrag string `json:"defrag,omitempty"`
|
||||||
|
Shmem string `json:"shmem_enabled,omitempty"`
|
||||||
|
PerSize []Size `json:"per_size,omitempty"`
|
||||||
|
// Meminfo is the machine's huge pages by kind, in MiB.
|
||||||
|
Meminfo map[string]float64 `json:"meminfo_mib,omitempty"`
|
||||||
|
Units []Unit `json:"units"`
|
||||||
|
Said []string `json:"said"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// Size is one per-size control (hugepages-<n>kB/enabled).
|
||||||
|
type Size struct {
|
||||||
|
Size string `json:"size"`
|
||||||
|
Enabled string `json:"enabled"`
|
||||||
|
// Means is what inherit resolves to: the top-level mode.
|
||||||
|
Means string `json:"means"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// Unit is one cgroup holding file pages in huge folios.
|
||||||
|
type Unit struct {
|
||||||
|
Cgroup string `json:"cgroup"`
|
||||||
|
FileTHP int64 `json:"file_thp_bytes"`
|
||||||
|
FileDirty int64 `json:"file_dirty_bytes"`
|
||||||
|
FileWriteback int64 `json:"file_writeback_bytes"`
|
||||||
|
FileMapped int64 `json:"file_mapped_bytes"`
|
||||||
|
CollapsedByKhug int64 `json:"thp_collapse_alloc"`
|
||||||
|
Written int64 `json:"written_bytes"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// bracketed is the word a sysfs choice file brackets: "always [madvise] never" is madvise.
|
||||||
|
func bracketed(text string) string {
|
||||||
|
open := strings.IndexByte(text, '[')
|
||||||
|
end := strings.IndexByte(text, ']')
|
||||||
|
if open < 0 || end < open {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
return text[open+1 : end]
|
||||||
|
}
|
||||||
|
|
||||||
|
// HugePages reads the mode, the per-size controls, meminfo and the top units by file_thp.
|
||||||
|
func (m *Machine) HugePages(units int) (*HugePagesAnswer, error) {
|
||||||
|
enabled, err := m.read(THPDir + "/enabled")
|
||||||
|
if err != nil {
|
||||||
|
if os.IsNotExist(err) {
|
||||||
|
return nil, fmt.Errorf("this kernel has no /%s/enabled: it was built without transparent huge pages", THPDir)
|
||||||
|
}
|
||||||
|
return nil, fmt.Errorf("reading /%s/enabled: %w", THPDir, err)
|
||||||
|
}
|
||||||
|
a := &HugePagesAnswer{Mode: bracketed(enabled), Units: []Unit{}}
|
||||||
|
if s, err := m.read(THPDir + "/defrag"); err == nil {
|
||||||
|
a.Defrag = bracketed(s)
|
||||||
|
}
|
||||||
|
if s, err := m.read(THPDir + "/shmem_enabled"); err == nil {
|
||||||
|
a.Shmem = bracketed(s)
|
||||||
|
}
|
||||||
|
a.Said = append(a.Said, fmt.Sprintf("transparent huge pages: %s, defrag %s", a.Mode, orUnknown(a.Defrag)))
|
||||||
|
|
||||||
|
sizes, _ := m.dirs(THPDir)
|
||||||
|
for _, d := range sizes {
|
||||||
|
if !strings.HasPrefix(d, "hugepages-") {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
s, err := m.read(THPDir + "/" + d + "/enabled")
|
||||||
|
if err != nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
sz := Size{Size: strings.TrimPrefix(d, "hugepages-"), Enabled: bracketed(s)}
|
||||||
|
sz.Means = sz.Enabled
|
||||||
|
if sz.Enabled == "inherit" {
|
||||||
|
sz.Means = a.Mode
|
||||||
|
}
|
||||||
|
a.PerSize = append(a.PerSize, sz)
|
||||||
|
}
|
||||||
|
sort.Slice(a.PerSize, func(i, j int) bool { return kib(a.PerSize[i].Size) < kib(a.PerSize[j].Size) })
|
||||||
|
var on []string
|
||||||
|
for _, s := range a.PerSize {
|
||||||
|
if s.Means != "never" {
|
||||||
|
on = append(on, s.Size+" "+s.Means)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(a.PerSize) > 0 {
|
||||||
|
a.Said = append(a.Said, "sizes not never: "+orNone(on))
|
||||||
|
}
|
||||||
|
|
||||||
|
if mi, err := m.read("proc/meminfo"); err == nil {
|
||||||
|
a.Meminfo = meminfo(mi)
|
||||||
|
a.Said = append(a.Said, fmt.Sprintf("huge pages held: anonymous %.0f MiB, shared memory %.0f MiB, file %.0f MiB (mapped at 2 MiB %.0f MiB)",
|
||||||
|
a.Meminfo["AnonHugePages"], a.Meminfo["ShmemHugePages"], a.Meminfo["FileHugePages"], a.Meminfo["FilePmdMapped"]))
|
||||||
|
}
|
||||||
|
|
||||||
|
if units > 0 {
|
||||||
|
all := m.cgroups()
|
||||||
|
sort.Slice(all, func(i, j int) bool {
|
||||||
|
if all[i].FileTHP != all[j].FileTHP {
|
||||||
|
return all[i].FileTHP > all[j].FileTHP
|
||||||
|
}
|
||||||
|
return all[i].Cgroup < all[j].Cgroup
|
||||||
|
})
|
||||||
|
if len(all) > units {
|
||||||
|
all = all[:units]
|
||||||
|
}
|
||||||
|
a.Units = all
|
||||||
|
for _, u := range all {
|
||||||
|
a.Said = append(a.Said, fmt.Sprintf("%s: file in huge folios %s, dirty %s, writeback %s, written %s since it started",
|
||||||
|
u.Cgroup, mib(u.FileTHP), mib(u.FileDirty), mib(u.FileWriteback), mib(u.Written)))
|
||||||
|
}
|
||||||
|
if len(all) == 0 {
|
||||||
|
a.Said = append(a.Said, "no unit holds file pages in huge folios")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return a, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// cgroups walks the leaf services and scopes (to three levels below the root) that hold file_thp.
|
||||||
|
func (m *Machine) cgroups() []Unit {
|
||||||
|
var out []Unit
|
||||||
|
var walk func(rel string, depth int)
|
||||||
|
walk = func(rel string, depth int) {
|
||||||
|
children, _ := m.dirs(path.Join(CgroupRoot, rel))
|
||||||
|
for _, c := range children {
|
||||||
|
p := path.Join(rel, c)
|
||||||
|
if strings.HasSuffix(c, ".slice") {
|
||||||
|
if depth < 3 {
|
||||||
|
walk(p, depth+1)
|
||||||
|
}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if !strings.HasSuffix(c, ".service") && !strings.HasSuffix(c, ".scope") {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
stat, err := m.read(path.Join(CgroupRoot, p, "memory.stat"))
|
||||||
|
if err != nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
kv := memoryStat(stat)
|
||||||
|
if kv["file_thp"] == 0 {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
u := Unit{Cgroup: p, FileTHP: kv["file_thp"], FileDirty: kv["file_dirty"], FileWriteback: kv["file_writeback"],
|
||||||
|
FileMapped: kv["file_mapped"], CollapsedByKhug: kv["thp_collapse_alloc"]}
|
||||||
|
if io, err := m.read(path.Join(CgroupRoot, p, "io.stat")); err == nil {
|
||||||
|
u.Written = mostWritten(io)
|
||||||
|
}
|
||||||
|
out = append(out, u)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
walk("", 0)
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// memoryStat reads memory.stat's "key value" lines.
|
||||||
|
func memoryStat(text string) map[string]int64 {
|
||||||
|
kv := map[string]int64{}
|
||||||
|
for _, l := range strings.Split(text, "\n") {
|
||||||
|
f := strings.Fields(l)
|
||||||
|
if len(f) != 2 {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if n, err := strconv.ParseInt(f[1], 10, 64); err == nil {
|
||||||
|
kv[f[0]] = n
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return kv
|
||||||
|
}
|
||||||
|
|
||||||
|
// mostWritten is the most bytes any one device of io.stat saw written: a mirror's member disks and the
|
||||||
|
// mirror itself each count the same write, so the lines are never summed.
|
||||||
|
func mostWritten(text string) int64 {
|
||||||
|
var most int64
|
||||||
|
for _, l := range strings.Split(text, "\n") {
|
||||||
|
for _, f := range strings.Fields(l) {
|
||||||
|
if v, ok := strings.CutPrefix(f, "wbytes="); ok {
|
||||||
|
if n, err := strconv.ParseInt(v, 10, 64); err == nil && n > most {
|
||||||
|
most = n
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return most
|
||||||
|
}
|
||||||
|
|
||||||
|
// meminfo reads the huge page lines of /proc/meminfo, in MiB.
|
||||||
|
func meminfo(text string) map[string]float64 {
|
||||||
|
want := map[string]bool{"AnonHugePages": true, "ShmemHugePages": true, "ShmemPmdMapped": true, "FileHugePages": true, "FilePmdMapped": true}
|
||||||
|
out := map[string]float64{}
|
||||||
|
for _, l := range strings.Split(text, "\n") {
|
||||||
|
k, v, ok := strings.Cut(l, ":")
|
||||||
|
if !ok || !want[k] {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
f := strings.Fields(v)
|
||||||
|
if len(f) == 0 {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if n, err := strconv.ParseFloat(f[0], 64); err == nil {
|
||||||
|
out[k] = n / 1024
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
func kib(size string) int {
|
||||||
|
n, _ := strconv.Atoi(strings.TrimSuffix(size, "kB"))
|
||||||
|
return n
|
||||||
|
}
|
||||||
|
|
||||||
|
func mib(b int64) string { return fmt.Sprintf("%.1f MiB", float64(b)/(1<<20)) }
|
||||||
|
|
||||||
|
func orUnknown(s string) string {
|
||||||
|
if s == "" {
|
||||||
|
return "unknown"
|
||||||
|
}
|
||||||
|
return s
|
||||||
|
}
|
||||||
|
|
||||||
|
func orNone(list []string) string {
|
||||||
|
if len(list) == 0 {
|
||||||
|
return "none"
|
||||||
|
}
|
||||||
|
return strings.Join(list, ", ")
|
||||||
|
}
|
||||||
@@ -0,0 +1,113 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
// Over a stand-in tree holding novox's own files of 2026-10-14 23:25 (read through node-reads for
|
||||||
|
// novox/hq issue 746): the mode, the per-size controls, meminfo's huge page lines, and fail2ban's and
|
||||||
|
// journald's memory.stat and io.stat, cut to the lines the tool reads.
|
||||||
|
|
||||||
|
import (
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func tree(t *testing.T, files map[string]string) *Machine {
|
||||||
|
t.Helper()
|
||||||
|
root := t.TempDir()
|
||||||
|
for p, content := range files {
|
||||||
|
full := filepath.Join(root, p)
|
||||||
|
if err := os.MkdirAll(filepath.Dir(full), 0o755); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(full, []byte(content), 0o644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return &Machine{Root: root}
|
||||||
|
}
|
||||||
|
|
||||||
|
func novox(t *testing.T) *Machine {
|
||||||
|
thp := "sys/kernel/mm/transparent_hugepage/"
|
||||||
|
cg := "sys/fs/cgroup/system.slice/"
|
||||||
|
files := map[string]string{
|
||||||
|
thp + "enabled": "[always] madvise never\n",
|
||||||
|
thp + "defrag": "always defer defer+madvise [madvise] never\n",
|
||||||
|
thp + "shmem_enabled": "always within_size [advise] never deny force\n",
|
||||||
|
thp + "hugepages-2048kB/enabled": "always [inherit] madvise never\n",
|
||||||
|
thp + "hugepages-64kB/enabled": "always inherit madvise [never]\n",
|
||||||
|
thp + "hugepages-1024kB/enabled": "always inherit madvise [never]\n",
|
||||||
|
"proc/meminfo": "MemTotal: 65748808 kB\nAnonHugePages: 3579904 kB\nShmemHugePages: 4096 kB\n" +
|
||||||
|
"ShmemPmdMapped: 4096 kB\nFileHugePages: 1693696 kB\nFilePmdMapped: 882688 kB\nHugepagesize: 2048 kB\n",
|
||||||
|
cg + "fail2ban.service/memory.stat": "anon 21815296\nfile 204623872\nfile_mapped 130736128\nfile_dirty 27262976\n" +
|
||||||
|
"file_writeback 4096\nanon_thp 0\nfile_thp 197132288\nthp_fault_alloc 0\nthp_collapse_alloc 550\n",
|
||||||
|
cg + "fail2ban.service/io.stat": "9:0 \n8:16 rbytes=3223552 wbytes=67723415552 rios=240 wios=281159 dbytes=0 dios=0\n" +
|
||||||
|
"8:0 rbytes=6152192 wbytes=67723415552 rios=832 wios=281159 dbytes=0 dios=0\n" +
|
||||||
|
"9:2 rbytes=9375744 wbytes=67723415552 rios=1072 wios=281159 dbytes=0 dios=0\n",
|
||||||
|
cg + "systemd-journald.service/memory.stat": "anon 3227648\nfile 1032015872\nfile_mapped 131928064\nfile_dirty 622592\n" +
|
||||||
|
"file_writeback 159744\nanon_thp 2097152\nfile_thp 379584512\nthp_fault_alloc 0\nthp_collapse_alloc 11741\n",
|
||||||
|
cg + "systemd-journald.service/io.stat": "9:0 \n8:16 rbytes=55209984 wbytes=2498768035840 rios=1317 wios=24719004 dbytes=0 dios=0\n" +
|
||||||
|
"8:0 rbytes=72953856 wbytes=2498768039936 rios=2194 wios=24719005 dbytes=0 dios=0\n" +
|
||||||
|
"9:2 rbytes=128163840 wbytes=2498724843520 rios=3511 wios=24707873 dbytes=0 dios=0\n",
|
||||||
|
cg + "sshd.service/memory.stat": "anon 1000\nfile_thp 0\n",
|
||||||
|
}
|
||||||
|
return tree(t, files)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNovoxBeforeTheChange(t *testing.T) {
|
||||||
|
a, err := novox(t).HugePages(10)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if a.Mode != "always" || a.Defrag != "madvise" || a.Shmem != "advise" {
|
||||||
|
t.Fatalf("modes: %+v", a)
|
||||||
|
}
|
||||||
|
if len(a.PerSize) != 3 || a.PerSize[0].Size != "64kB" || a.PerSize[2].Size != "2048kB" ||
|
||||||
|
a.PerSize[2].Enabled != "inherit" || a.PerSize[2].Means != "always" || a.PerSize[0].Means != "never" {
|
||||||
|
t.Fatalf("per size: %+v", a.PerSize)
|
||||||
|
}
|
||||||
|
if a.Meminfo["FileHugePages"] != 1654 || a.Meminfo["FilePmdMapped"] != 862 {
|
||||||
|
t.Fatalf("meminfo: %v", a.Meminfo)
|
||||||
|
}
|
||||||
|
if len(a.Units) != 2 || a.Units[0].Cgroup != "system.slice/systemd-journald.service" ||
|
||||||
|
a.Units[1].Cgroup != "system.slice/fail2ban.service" {
|
||||||
|
t.Fatalf("units: %+v", a.Units)
|
||||||
|
}
|
||||||
|
// The mirror and its two members each count the write: the most any one saw, never their sum.
|
||||||
|
if a.Units[1].Written != 67723415552 || a.Units[1].FileTHP != 197132288 || a.Units[1].CollapsedByKhug != 550 {
|
||||||
|
t.Fatalf("fail2ban: %+v", a.Units[1])
|
||||||
|
}
|
||||||
|
said := strings.Join(a.Said, "\n")
|
||||||
|
for _, want := range []string{"transparent huge pages: always, defrag madvise", "sizes not never: 2048kB always",
|
||||||
|
"file 1654 MiB (mapped at 2 MiB 862 MiB)", "system.slice/fail2ban.service: file in huge folios 188.0 MiB"} {
|
||||||
|
if !strings.Contains(said, want) {
|
||||||
|
t.Errorf("said lacks %q:\n%s", want, said)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestUnitsAreBoundedAndNoneIsSaid(t *testing.T) {
|
||||||
|
a, err := novox(t).HugePages(1)
|
||||||
|
if err != nil || len(a.Units) != 1 {
|
||||||
|
t.Fatalf("%v %+v", err, a)
|
||||||
|
}
|
||||||
|
b, err := tree(t, map[string]string{"sys/kernel/mm/transparent_hugepage/enabled": "always [madvise] never\n"}).HugePages(10)
|
||||||
|
if err != nil || b.Mode != "madvise" || len(b.Units) != 0 || !strings.Contains(strings.Join(b.Said, "\n"), "no unit holds") {
|
||||||
|
t.Fatalf("%v %+v", err, b)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAKernelWithoutHugePagesIsAnError(t *testing.T) {
|
||||||
|
if _, err := tree(t, map[string]string{"proc/meminfo": ""}).HugePages(10); err == nil ||
|
||||||
|
!strings.Contains(err.Error(), "built without transparent huge pages") {
|
||||||
|
t.Fatalf("%v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTheArgumentIsBounded(t *testing.T) {
|
||||||
|
if _, err := whole(map[string]any{"units": -1.0}, "units", 10, 0, 100); err == nil {
|
||||||
|
t.Error("a negative count was accepted")
|
||||||
|
}
|
||||||
|
if n, _ := whole(map[string]any{"units": 1000.0}, "units", 10, 0, 100); n != 100 {
|
||||||
|
t.Errorf("not bounded: %d", n)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,65 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
// The machine as the tools see it: the files they read, under a root that tests replace with a stand-in
|
||||||
|
// tree. Everything read here is readable by the runtime's account; nothing is escalated.
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"io/fs"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Machine reads this machine's kernel files under Root ("/" on the machine).
|
||||||
|
type Machine struct {
|
||||||
|
Root string
|
||||||
|
}
|
||||||
|
|
||||||
|
// ThisMachine is the machine the runtime launched this bundle on.
|
||||||
|
func ThisMachine() *Machine { return &Machine{Root: "/"} }
|
||||||
|
|
||||||
|
func (m *Machine) path(p string) string { return filepath.Join(m.Root, p) }
|
||||||
|
|
||||||
|
func (m *Machine) read(p string) (string, error) {
|
||||||
|
b, err := os.ReadFile(m.path(p))
|
||||||
|
return string(b), err
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *Machine) dirs(p string) ([]string, error) {
|
||||||
|
entries, err := os.ReadDir(m.path(p))
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
var out []string
|
||||||
|
for _, e := range entries {
|
||||||
|
if e.IsDir() || e.Type()&fs.ModeSymlink != 0 {
|
||||||
|
out = append(out, e.Name())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// whole is a whole-number argument with a default, kept within bounds.
|
||||||
|
func whole(args map[string]any, key string, def, least, most int) (int, error) {
|
||||||
|
raw, present := args[key]
|
||||||
|
if !present || raw == nil {
|
||||||
|
return def, nil
|
||||||
|
}
|
||||||
|
f, ok := raw.(float64)
|
||||||
|
if !ok || f != float64(int(f)) {
|
||||||
|
return 0, fmt.Errorf("%s must be a whole number", key)
|
||||||
|
}
|
||||||
|
n := int(f)
|
||||||
|
if n < least {
|
||||||
|
return 0, fmt.Errorf("%s must be at least %d", key, least)
|
||||||
|
}
|
||||||
|
if n > most {
|
||||||
|
n = most
|
||||||
|
}
|
||||||
|
return n, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// schema is a tool's input: its properties.
|
||||||
|
func schema(properties map[string]any) map[string]any {
|
||||||
|
return map[string]any{"type": "object", "properties": properties}
|
||||||
|
}
|
||||||
@@ -0,0 +1,50 @@
|
|||||||
|
// The kernel module's tools bundle (novox/hq to-be 42 Phase 1 row 11, issue 746).
|
||||||
|
//
|
||||||
|
// Served by the node's runtime over MCP on stdio through the Go SDK (ADR 0188, ADR 0193). It only reads:
|
||||||
|
// the transparent huge pages the kernel runs with, and which units hold file pages in huge folios, which
|
||||||
|
// is what a small write to a mapped file costs a whole huge page of writeback for (issue 746).
|
||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
|
||||||
|
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||||
|
)
|
||||||
|
|
||||||
|
// binaryName is what the build names this bundle's executable: the manifest's `binary`.
|
||||||
|
const binaryName = "kernel-tools"
|
||||||
|
|
||||||
|
func main() {
|
||||||
|
// An empty name serves as the module the runtime names (MESH_SERVED_MODULE): kernel.
|
||||||
|
if err := stdio.Serve("", tools(ThisMachine())); err != nil {
|
||||||
|
fmt.Fprintln(os.Stderr, err)
|
||||||
|
os.Exit(1)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func tools(m *Machine) []stdio.Tool {
|
||||||
|
return []stdio.Tool{
|
||||||
|
{
|
||||||
|
Name: "kernel_huge_pages",
|
||||||
|
Description: "Transparent huge pages on this machine: the mode in force (always, madvise or never) and defrag, " +
|
||||||
|
"each per-size control with what inherit resolves to, the machine's huge pages by kind (anonymous, " +
|
||||||
|
"shared memory, file, file mapped at 2 MiB) from /proc/meminfo, and the units holding the most file " +
|
||||||
|
"pages in huge folios (memory.stat file_thp) with their dirty and writeback bytes and the bytes their " +
|
||||||
|
"cgroup has written (io.stat, the most any one device saw). A unit with file_thp that writes far more " +
|
||||||
|
"than its files grow is paying a whole huge page of writeback per small write (novox/hq issue 746). " +
|
||||||
|
"Replaces cat /sys/kernel/mm/transparent_hugepage/enabled, grep Huge /proc/meminfo and reading " +
|
||||||
|
"memory.stat by hand. (r)",
|
||||||
|
Input: schema(map[string]any{
|
||||||
|
"units": map[string]any{"type": "integer", "description": "how many units to name, the most file_thp first (default 10, 0 for none, at most 100)"},
|
||||||
|
}),
|
||||||
|
Run: func(args map[string]any) (any, error) {
|
||||||
|
n, err := whole(args, "units", 10, 0, 100)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return m.HugePages(n)
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,239 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
// The module's shape (novox/hq issue 746): the setting's default is madvise, the value the mesh chose
|
||||||
|
// (ADR 0381), not what the kernel boots with; the script is written into the
|
||||||
|
// module's own state and run by a sandboxed oneshot unit that is reloaded (applied again) when the script,
|
||||||
|
// and so the setting, changes and restarted only when the unit does; and the script itself sets, verifies,
|
||||||
|
// gives back and refuses as it says.
|
||||||
|
|
||||||
|
import (
|
||||||
|
"os"
|
||||||
|
"os/exec"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestTheDefaultIsMadviseAndTheUnitFollowsTheSetting(t *testing.T) {
|
||||||
|
m := manifest(t)
|
||||||
|
s := m.Settings["transparent-huge-pages"]
|
||||||
|
if s["kind"] != "preference" || s["default"] != "madvise" || s["why"] == "" {
|
||||||
|
t.Fatalf("the setting: %v", s)
|
||||||
|
}
|
||||||
|
var ids []string
|
||||||
|
for _, r := range m.Resources {
|
||||||
|
ids = append(ids, r["id"].(string))
|
||||||
|
}
|
||||||
|
if strings.Join(ids, " ") != "state huge-pages unit apply" {
|
||||||
|
t.Fatalf("order: %v", ids)
|
||||||
|
}
|
||||||
|
script := m.resource(t, "huge-pages")
|
||||||
|
// The script runs as root, so the setting it is written from is set at the terminal alone (issue 339).
|
||||||
|
if script["path"] != "${dir:state}/huge-pages" || script["mode"] != "0755" || script["trusted"] != true ||
|
||||||
|
!strings.Contains(script["content"].(string), "want='${setting:transparent-huge-pages}'") {
|
||||||
|
t.Fatalf("the script: %v", script["path"])
|
||||||
|
}
|
||||||
|
unit := m.resource(t, "unit")["content"].(string)
|
||||||
|
for _, line := range []string{"Type=oneshot", "RemainAfterExit=yes", "ExecStart=${dir:state}/huge-pages apply",
|
||||||
|
"ExecReload=${dir:state}/huge-pages apply", "ExecStop=${dir:state}/huge-pages restore", "WantedBy=sysinit.target",
|
||||||
|
"DefaultDependencies=no", "ProtectSystem=strict", "ReadWritePaths=${dir:state}", "ProtectHome=yes", "PrivateNetwork=yes",
|
||||||
|
"NoNewPrivileges=yes"} {
|
||||||
|
if !strings.Contains(unit, line+"\n") {
|
||||||
|
t.Errorf("the unit lacks %q", line)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
apply := m.resource(t, "apply")
|
||||||
|
if apply["unit"] != "mesh-kernel-huge-pages.service" || apply["state"] != "running" || apply["boot"] != "enabled" {
|
||||||
|
t.Fatalf("apply: %v", apply)
|
||||||
|
}
|
||||||
|
// A new value reloads (apply again); only a new unit restarts, whose stop would pass back through the
|
||||||
|
// mode the machine was found with.
|
||||||
|
if on := apply["restart-on"].([]any); len(on) != 1 || on[0] != "unit" {
|
||||||
|
t.Fatalf("restart-on: %v", on)
|
||||||
|
}
|
||||||
|
if on := apply["reload-on"].([]any); len(on) != 1 || on[0] != "huge-pages" {
|
||||||
|
t.Fatalf("reload-on: %v", on)
|
||||||
|
}
|
||||||
|
if h := apply["health"].(map[string]any); h["kind"] != "unit" {
|
||||||
|
t.Fatalf("health: %v", h)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// bed is the script as the mesh writes it for one value of the setting, a stand-in transparent_hugepage
|
||||||
|
// directory, the module's state, and a PATH whose journalctl records that it was asked to rotate.
|
||||||
|
type bed struct {
|
||||||
|
script, sys, state, rotated, path, bin string
|
||||||
|
}
|
||||||
|
|
||||||
|
func newBed(t *testing.T, want, found string) bed {
|
||||||
|
return newBedIn(t, t.TempDir(), want, found)
|
||||||
|
}
|
||||||
|
|
||||||
|
// newBedIn writes the script for want into dir; a second bed in the same dir shares the machine and the
|
||||||
|
// state, as a reload with a new value does.
|
||||||
|
func newBedIn(t *testing.T, dir, want, found string) bed {
|
||||||
|
t.Helper()
|
||||||
|
b := bed{script: filepath.Join(dir, "huge-pages"), sys: filepath.Join(dir, "thp"), state: filepath.Join(dir, "state"),
|
||||||
|
rotated: filepath.Join(dir, "rotated")}
|
||||||
|
content := manifest(t).resource(t, "huge-pages")["content"].(string)
|
||||||
|
content = strings.ReplaceAll(content, "${setting:transparent-huge-pages}", want)
|
||||||
|
content = strings.ReplaceAll(content, "${dir:state}", b.state)
|
||||||
|
if strings.Contains(content, "${") {
|
||||||
|
t.Fatalf("a substitution the test does not make is left in the script")
|
||||||
|
}
|
||||||
|
bin := filepath.Join(dir, "bin")
|
||||||
|
for _, d := range []string{b.sys, b.state, bin} {
|
||||||
|
if err := os.MkdirAll(d, 0o755); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
write(t, b.script, content, 0o755)
|
||||||
|
if found != "" {
|
||||||
|
write(t, filepath.Join(b.sys, "enabled"), found, 0o644)
|
||||||
|
}
|
||||||
|
write(t, filepath.Join(bin, "journalctl"), "#!/bin/sh\necho \"$@\" >>"+b.rotated+"\n", 0o755)
|
||||||
|
b.bin = bin
|
||||||
|
b.booted(t, "running")
|
||||||
|
b.path = bin + string(os.PathListSeparator) + os.Getenv("PATH")
|
||||||
|
return b
|
||||||
|
}
|
||||||
|
|
||||||
|
// booted makes the service manager say state for is-system-running.
|
||||||
|
func (b bed) booted(t *testing.T, state string) {
|
||||||
|
t.Helper()
|
||||||
|
write(t, filepath.Join(b.bin, "systemctl"), "#!/bin/sh\n[ \"$1\" = is-system-running ] && echo "+state+"\n", 0o755)
|
||||||
|
}
|
||||||
|
|
||||||
|
func write(t *testing.T, p, content string, mode os.FileMode) {
|
||||||
|
t.Helper()
|
||||||
|
if err := os.WriteFile(p, []byte(content), mode); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (b bed) run(t *testing.T, verb string) (string, error) {
|
||||||
|
t.Helper()
|
||||||
|
cmd := exec.Command("sh", b.script, verb, b.sys, b.state)
|
||||||
|
cmd.Env = append(os.Environ(), "PATH="+b.path)
|
||||||
|
out, err := cmd.CombinedOutput()
|
||||||
|
return string(out), err
|
||||||
|
}
|
||||||
|
|
||||||
|
func (b bed) file(t *testing.T, p string) string {
|
||||||
|
t.Helper()
|
||||||
|
raw, err := os.ReadFile(p)
|
||||||
|
if err != nil {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
return string(raw)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestApplySetsTheModeRotatesTheJournalOnceAndRestoreGivesBackWhatWasFound(t *testing.T) {
|
||||||
|
// novox's file on 2026-10-14, read through node-reads.
|
||||||
|
b := newBed(t, "madvise", "[always] madvise never\n")
|
||||||
|
out, err := b.run(t, "apply")
|
||||||
|
if err != nil || !strings.Contains(out, "always -> madvise") {
|
||||||
|
t.Fatalf("apply: %v %s", err, out)
|
||||||
|
}
|
||||||
|
if got := b.file(t, filepath.Join(b.sys, "enabled")); got != "madvise\n" {
|
||||||
|
t.Fatalf("written: %q", got)
|
||||||
|
}
|
||||||
|
if got := b.file(t, filepath.Join(b.state, "found")); got != "always\n" {
|
||||||
|
t.Fatalf("found: %q", got)
|
||||||
|
}
|
||||||
|
if got := b.file(t, b.rotated); got != "--rotate\n" {
|
||||||
|
t.Fatalf("rotation: %q", got)
|
||||||
|
}
|
||||||
|
// Applied again (a restart with the same value): nothing written, nothing rotated, found kept.
|
||||||
|
if out, err := b.run(t, "apply"); err != nil || !strings.Contains(out, "madvise, as set") {
|
||||||
|
t.Fatalf("again: %v %s", err, out)
|
||||||
|
}
|
||||||
|
if got := b.file(t, b.rotated); got != "--rotate\n" {
|
||||||
|
t.Fatalf("rotated again: %q", got)
|
||||||
|
}
|
||||||
|
if out, err := b.run(t, "restore"); err != nil || !strings.Contains(out, "back to always") {
|
||||||
|
t.Fatalf("restore: %v %s", err, out)
|
||||||
|
}
|
||||||
|
if got := b.file(t, filepath.Join(b.sys, "enabled")); got != "always\n" {
|
||||||
|
t.Fatalf("restored: %q", got)
|
||||||
|
}
|
||||||
|
// Nothing of the module is kept once it is stopped.
|
||||||
|
if _, err := os.Stat(filepath.Join(b.state, "found")); !os.IsNotExist(err) {
|
||||||
|
t.Fatalf("found kept after restore: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAReloadWithANewValueGoesStraightThereAndNeverRewritesFound(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
b := newBedIn(t, dir, "madvise", "[always] madvise never\n")
|
||||||
|
if out, err := b.run(t, "apply"); err != nil {
|
||||||
|
t.Fatalf("apply: %v %s", err, out)
|
||||||
|
}
|
||||||
|
// The setting changes to never: the mesh rewrites the script and reloads the unit, which applies again.
|
||||||
|
n := newBedIn(t, dir, "never", "")
|
||||||
|
out, err := n.run(t, "apply")
|
||||||
|
if err != nil || !strings.Contains(out, "madvise -> never") {
|
||||||
|
t.Fatalf("reload: %v %s", err, out)
|
||||||
|
}
|
||||||
|
if got := n.file(t, filepath.Join(n.state, "found")); got != "always\n" {
|
||||||
|
t.Fatalf("found rewritten by a reload: %q", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNoRotationWhileTheMachineIsStarting(t *testing.T) {
|
||||||
|
for _, state := range []string{"initializing", "starting"} {
|
||||||
|
b := newBed(t, "madvise", "[always] madvise never\n")
|
||||||
|
b.booted(t, state)
|
||||||
|
out, err := b.run(t, "apply")
|
||||||
|
if err != nil || !strings.Contains(out, "still starting") {
|
||||||
|
t.Fatalf("%s: %v %s", state, err, out)
|
||||||
|
}
|
||||||
|
if got := b.file(t, filepath.Join(b.sys, "enabled")); got != "madvise\n" {
|
||||||
|
t.Fatalf("%s: not set: %q", state, got)
|
||||||
|
}
|
||||||
|
if got := b.file(t, b.rotated); got != "" {
|
||||||
|
t.Fatalf("%s: rotated at boot: %q", state, got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAlwaysIsSetWithoutRotating(t *testing.T) {
|
||||||
|
b := newBed(t, "always", "always [madvise] never\n")
|
||||||
|
if out, err := b.run(t, "apply"); err != nil || !strings.Contains(out, "madvise -> always") {
|
||||||
|
t.Fatalf("apply: %v %s", err, out)
|
||||||
|
}
|
||||||
|
if got := b.file(t, b.rotated); got != "" {
|
||||||
|
t.Fatalf("rotated for always: %q", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAValueTheKernelDoesNotTakeIsRefusedBeforeAnythingIsWritten(t *testing.T) {
|
||||||
|
b := newBed(t, "sometimes", "[always] madvise never\n")
|
||||||
|
out, err := b.run(t, "apply")
|
||||||
|
if err == nil || !strings.Contains(out, "takes always, madvise or never") {
|
||||||
|
t.Fatalf("accepted: %v %s", err, out)
|
||||||
|
}
|
||||||
|
if got := b.file(t, filepath.Join(b.sys, "enabled")); got != "[always] madvise never\n" {
|
||||||
|
t.Fatalf("written anyway: %q", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAWriteThatDoesNotTakeFailsTheUnit(t *testing.T) {
|
||||||
|
if os.Getuid() == 0 {
|
||||||
|
t.Skip("root writes a read-only file")
|
||||||
|
}
|
||||||
|
b := newBed(t, "madvise", "[always] madvise never\n")
|
||||||
|
if err := os.Chmod(filepath.Join(b.sys, "enabled"), 0o444); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if out, err := b.run(t, "apply"); err == nil {
|
||||||
|
t.Fatalf("a refused write passed: %s", out)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAKernelWithoutHugePagesIsSaid(t *testing.T) {
|
||||||
|
b := newBed(t, "madvise", "")
|
||||||
|
if out, err := b.run(t, "apply"); err == nil || !strings.Contains(out, "built without transparent huge pages") {
|
||||||
|
t.Fatalf("%v %s", err, out)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,79 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"os"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
type resource map[string]any
|
||||||
|
|
||||||
|
type manifestShape struct {
|
||||||
|
Module string `json:"module"`
|
||||||
|
Capabilities []string `json:"capabilities"`
|
||||||
|
Settings map[string]map[string]any `json:"settings"`
|
||||||
|
Tools []string `json:"tools"`
|
||||||
|
Resources []resource `json:"resources"`
|
||||||
|
Build struct {
|
||||||
|
Artifacts []map[string]any `json:"artifacts"`
|
||||||
|
} `json:"build"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func manifest(t *testing.T) manifestShape {
|
||||||
|
t.Helper()
|
||||||
|
raw, err := os.ReadFile("../../module.json")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var m manifestShape
|
||||||
|
if err := json.Unmarshal(raw, &m); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return m
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m manifestShape) resource(t *testing.T, id string) resource {
|
||||||
|
t.Helper()
|
||||||
|
for _, r := range m.Resources {
|
||||||
|
if r["id"] == id {
|
||||||
|
return r
|
||||||
|
}
|
||||||
|
}
|
||||||
|
t.Fatalf("no resource %s", id)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestToolsAreTheManifests holds the served tools and the manifest's list to one another, and the
|
||||||
|
// bundle to the shape the builder compiles and the runtime loads.
|
||||||
|
func TestToolsAreTheManifests(t *testing.T) {
|
||||||
|
m := manifest(t)
|
||||||
|
names := map[string]bool{}
|
||||||
|
for _, tool := range tools(&Machine{Root: t.TempDir()}) {
|
||||||
|
if names[tool.Name] {
|
||||||
|
t.Errorf("%s is served twice", tool.Name)
|
||||||
|
}
|
||||||
|
names[tool.Name] = true
|
||||||
|
}
|
||||||
|
for _, want := range m.Tools {
|
||||||
|
if !names[want] {
|
||||||
|
t.Errorf("the manifest lists %s and the bundle does not serve it", want)
|
||||||
|
}
|
||||||
|
delete(names, want)
|
||||||
|
}
|
||||||
|
if len(names) != 0 {
|
||||||
|
t.Errorf("served and not listed: %v", names)
|
||||||
|
}
|
||||||
|
var tools map[string]any
|
||||||
|
for _, a := range m.Build.Artifacts {
|
||||||
|
if a["name"] == "tools" {
|
||||||
|
tools = a
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if tools == nil || tools["kind"] != "bundle" || tools["language"] != "go" || tools["system"] != "arch" ||
|
||||||
|
tools["from"] != "cmd/"+binaryName || tools["binary"] != binaryName {
|
||||||
|
t.Fatalf("the tools artifact: %v", tools)
|
||||||
|
}
|
||||||
|
if loads, _ := tools["loads"].([]any); len(loads) != 1 || loads[0] != binaryName {
|
||||||
|
t.Fatalf("loads: %v", tools["loads"])
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,5 @@
|
|||||||
|
module kernel
|
||||||
|
|
||||||
|
go 1.22
|
||||||
|
|
||||||
|
require git.novox.be/novox/mesh-sdk/go v0.1.7
|
||||||
@@ -0,0 +1,2 @@
|
|||||||
|
git.novox.be/novox/mesh-sdk/go v0.1.7 h1:C0sTQmtTiyYH7bnqZb7PusXnqA37gKuT7Nqjn9gG47w=
|
||||||
|
git.novox.be/novox/mesh-sdk/go v0.1.7/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY=
|
||||||
@@ -0,0 +1,71 @@
|
|||||||
|
{
|
||||||
|
"module": "kernel",
|
||||||
|
"version": "1",
|
||||||
|
"capabilities": [
|
||||||
|
"service-manager"
|
||||||
|
],
|
||||||
|
"settings": {
|
||||||
|
"transparent-huge-pages": {
|
||||||
|
"kind": "preference",
|
||||||
|
"default": "madvise",
|
||||||
|
"why": "madvise is the value the mesh chose for every node (novox/hq ADR 0381), not the always the distribution's kernel boots with; a machine that needs another value is given it as its own setting; madvise gives huge pages only to programs that ask for them, which stops a small write to a mapped file (the journal) dirtying and writing back a whole 2 MiB page on a machine whose disks are slow (novox/hq issue 746)"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"tools": [
|
||||||
|
"kernel_huge_pages"
|
||||||
|
],
|
||||||
|
"resources": [
|
||||||
|
{
|
||||||
|
"id": "state",
|
||||||
|
"type": "directory",
|
||||||
|
"mode": "0700",
|
||||||
|
"place": "."
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "huge-pages",
|
||||||
|
"type": "file",
|
||||||
|
"path": "${dir:state}/huge-pages",
|
||||||
|
"mode": "0755",
|
||||||
|
"trusted": true,
|
||||||
|
"content": "#!/bin/sh\n# Transparent huge pages as the mesh sets them on this machine (module kernel, novox/hq issue 746).\n# Written by the mesh from this machine's setting transparent-huge-pages and run by\n# mesh-kernel-huge-pages.service: `apply` at boot and whenever the setting changes, `restore` when the\n# unit stops (the module unassigned, or the machine shutting down). Replaced on every push; change the\n# setting, never this file. A change of the setting reloads the unit, which runs `apply` again: the mode\n# moves straight to the new value and never passes back through the one the machine was found with.\n#\n# huge-pages apply|restore [<the kernel's transparent_hugepage directory> [<this module's state>]]\n#\n# The two directories are arguments only so the module's tests can run this against a stand-in tree.\nset -eu\n\nwant='${setting:transparent-huge-pages}'\nsys=/sys/kernel/mm/transparent_hugepage\nstate='${dir:state}'\n[ $# -ge 2 ] && sys=$2\n[ $# -ge 3 ] && state=$3\n\ncase \"$want\" in\nalways | madvise | never) ;;\n*)\n\techo \"the setting transparent-huge-pages is '$want'; the kernel takes always, madvise or never\" >&2\n\texit 2\n\t;;\nesac\n[ -f \"$sys/enabled\" ] || {\n\techo \"this kernel has no $sys/enabled: it was built without transparent huge pages, so there is nothing to set\" >&2\n\texit 1\n}\n\n# The mode in force is the word the kernel brackets: \"always [madvise] never\". A file holding one bare\n# word (the stand-in tree of the tests) is read as that word.\nmode() { sed -n -e 's/.*\\[\\([a-z]*\\)\\].*/\\1/p' -e t -e 's/^ *\\([a-z]*\\) *$/\\1/p' \"$sys/enabled\"; }\n\n[ $# -ge 1 ] || set -- usage\ncase \"$1\" in\napply)\n\tnow=$(mode)\n\t# What the machine had before the mesh first set it, kept once, so `restore` gives it back.\n\t[ -s \"$state/found\" ] || printf '%s\\n' \"$now\" >\"$state/found\"\n\tif [ \"$now\" = \"$want\" ]; then\n\t\techo \"transparent huge pages: $want, as set\"\n\t\texit 0\n\tfi\n\tprintf '%s\\n' \"$want\" >\"$sys/enabled\"\n\tgot=$(mode)\n\t[ \"$got\" = \"$want\" ] || {\n\t\techo \"wrote $want to $sys/enabled and the kernel says $got\" >&2\n\t\texit 1\n\t}\n\techo \"transparent huge pages: $now -> $want\"\n\t# A file folio already in the page cache keeps its size: the active journal files, mapped by journald\n\t# and by every reader, stay in 2 MiB folios until they are archived (issue 746). Rotating starts new\n\t# files that are mapped under the new mode. A failed rotation is said and does not fail the setting.\n\t# At boot nothing is in huge folios yet, and the persistent journal opens only after this unit: no rotation\n\t# while the service manager is still bringing the machine up.\n\tbooting=$(systemctl is-system-running 2>/dev/null || true)\n\tcase \"$booting\" in\n\tinitializing | starting) echo \"the machine is still starting: nothing to rotate yet\" ;;\n\t*)\n\t\tif [ \"$want\" != always ] && command -v journalctl >/dev/null 2>&1; then\n\t\t\tjournalctl --rotate || echo \"the journal could not be rotated: its active files keep their huge pages until they are archived\" >&2\n\t\tfi\n\t\t;;\n\tesac\n\t;;\nrestore)\n\tfound=$(cat \"$state/found\" 2>/dev/null || true)\n\tcase \"$found\" in\n\talways | madvise | never)\n\t\tprintf '%s\\n' \"$found\" >\"$sys/enabled\"\n\t\techo \"transparent huge pages: back to $found, as the machine had them\"\n\t\t;;\n\t*) echo \"transparent huge pages: nothing recorded as found, left at $(mode)\" ;;\n\tesac\n\t# Unassigned or shut down: nothing of the module is kept; the next start records what it finds again.\n\trm -f \"$state/found\"\n\t;;\n*)\n\techo \"usage: huge-pages apply|restore [<transparent_hugepage directory> [<state directory>]]\" >&2\n\texit 2\n\t;;\nesac\n"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "unit",
|
||||||
|
"type": "file",
|
||||||
|
"path": "/etc/systemd/system/mesh-kernel-huge-pages.service",
|
||||||
|
"mode": "0644",
|
||||||
|
"content": "[Unit]\nDescription=Transparent huge pages as the mesh sets them on this machine\n# Module kernel, novox/hq issue 746. Starting this unit sets the mode from the setting\n# transparent-huge-pages; reloading it (the setting changed) sets the new mode; stopping it gives back the\n# mode the machine had before. It runs early in boot, before the services that fill the page cache.\nDefaultDependencies=no\nAfter=local-fs.target\nBefore=sysinit.target shutdown.target\nConflicts=shutdown.target\n\n[Service]\nType=oneshot\nRemainAfterExit=yes\nExecStart=${dir:state}/huge-pages apply\nExecReload=${dir:state}/huge-pages apply\nExecStop=${dir:state}/huge-pages restore\n# It writes one file under /sys and its own state, and asks the journal daemon to rotate over its socket.\nProtectSystem=strict\nReadWritePaths=${dir:state}\nProtectHome=yes\nPrivateNetwork=yes\nNoNewPrivileges=yes\n\n[Install]\nWantedBy=sysinit.target\n"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "apply",
|
||||||
|
"type": "service",
|
||||||
|
"unit": "mesh-kernel-huge-pages.service",
|
||||||
|
"state": "running",
|
||||||
|
"boot": "enabled",
|
||||||
|
"restart-on": [
|
||||||
|
"unit"
|
||||||
|
],
|
||||||
|
"reload-on": [
|
||||||
|
"huge-pages"
|
||||||
|
],
|
||||||
|
"health": {
|
||||||
|
"kind": "unit"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"build": {
|
||||||
|
"artifacts": [
|
||||||
|
{
|
||||||
|
"name": "tools",
|
||||||
|
"kind": "bundle",
|
||||||
|
"language": "go",
|
||||||
|
"system": "arch",
|
||||||
|
"from": "cmd/kernel-tools",
|
||||||
|
"binary": "kernel-tools",
|
||||||
|
"loads": [
|
||||||
|
"kernel-tools"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -144,7 +144,9 @@ func (h *Holder) accounted(w Walk) bool {
|
|||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
|
|
||||||
// held is whether a live delivery follows the walk, or a live delivery it answers follows a newer one.
|
// held is whether a live delivery follows the walk, or a live delivery it answers follows a newer one. A walk whose
|
||||||
|
// word only stops that lapsed give (lapsedOn, novox/hq issue 753) is not held: it stays under the orphans' timer, so
|
||||||
|
// past the grace without its word it is listed by `stalled`, saying why it was not given.
|
||||||
func (h *Holder) heldWalk(w Walk) bool {
|
func (h *Holder) heldWalk(w Walk) bool {
|
||||||
for _, d := range h.deliveries {
|
for _, d := range h.deliveries {
|
||||||
if d.State.Final() {
|
if d.State.Final() {
|
||||||
@@ -292,12 +294,28 @@ func (h *Holder) orphanLines(now time.Time) []StalledLine {
|
|||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
w := s.walk
|
w := s.walk
|
||||||
|
if lapsed, behind := h.lapsedOf(w); len(lapsed) > 0 || len(behind) > 0 {
|
||||||
|
// Owed its word by stops that lapsed, and not given it past the grace (review of issue 753): said, why.
|
||||||
|
says := fmt.Sprintf("the walk %s of %s at %s waits for this owner's word, which the stopped deliveries whose "+
|
||||||
|
"merge it answers owe it (novox/hq issue 753), and it has not started", w.ID, w.Repository, shortOf(w.Commit, 8))
|
||||||
|
if why := h.lapsedSaid(w); why != "" {
|
||||||
|
says += ": " + why
|
||||||
|
}
|
||||||
|
out = append(out, StalledLine{ID: IDOf(w.Repository, w.Commit), State: Published, Walk: w.ID,
|
||||||
|
For: now.Sub(first).Round(time.Second).String(), Bound: matchGrace.String(),
|
||||||
|
H2: "none: its word is given at its turn — the operator's past the bound", Says: says})
|
||||||
|
continue
|
||||||
|
}
|
||||||
out = append(out, StalledLine{ID: IDOf(w.Repository, w.Commit), State: Unheld, Walk: w.ID,
|
out = append(out, StalledLine{ID: IDOf(w.Repository, w.Commit), State: Unheld, Walk: w.ID,
|
||||||
For: now.Sub(first).Round(time.Second).String(), Bound: matchGrace.String(),
|
For: now.Sub(first).Round(time.Second).String(), Bound: matchGrace.String(),
|
||||||
H2: "none: no delivery holds the walk, so nothing gives it its word — the operator's",
|
H2: "none: no delivery holds the walk, so nothing gives it its word — the operator's",
|
||||||
Says: fmt.Sprintf("the walk %s of %s at %s waits for this owner's word, and no delivery that has not ended "+
|
Says: fmt.Sprintf("the walk %s of %s at %s waits for this owner's word, and no delivery that has not ended "+
|
||||||
"follows it or is answered by it: `mesh-delivery.show` for the deliveries of %s, and `plans go %s --why …` "+
|
"follows it or is answered by it: `mesh-delivery.show` for the deliveries of %s, and `plans go %s --why …` "+
|
||||||
"starts it by hand", w.ID, w.Repository, shortOf(w.Commit, 8), w.Repository, w.ID)})
|
"starts it by hand", w.ID, w.Repository, shortOf(w.Commit, 8), w.Repository, w.ID)})
|
||||||
|
out[len(out)-1].Says += h.endedSaid(w)
|
||||||
|
if hold := h.coverHeld(id); hold != "" {
|
||||||
|
out[len(out)-1].Says += "; it is held back meanwhile: " + hold
|
||||||
|
}
|
||||||
}
|
}
|
||||||
return out
|
return out
|
||||||
}
|
}
|
||||||
@@ -344,6 +362,7 @@ func (h *Holder) forgetWalks(now time.Time) {
|
|||||||
}
|
}
|
||||||
case !w.Open() && now.Sub(s.at) > keptWalkFor:
|
case !w.Open() && now.Sub(s.at) > keptWalkFor:
|
||||||
delete(h.walks, id)
|
delete(h.walks, id)
|
||||||
|
delete(h.deliverErr, id)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -356,6 +375,168 @@ type turnWant struct {
|
|||||||
whys []string
|
whys []string
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// endedSaid names the ended deliveries a waiting walk answers or that follow it, with their states, so a walk no live
|
||||||
|
// delivery holds says what it carries instead of only that nothing holds it (novox/hq issue 753); empty for none.
|
||||||
|
func (h *Holder) endedSaid(w Walk) string {
|
||||||
|
var ended []string
|
||||||
|
for _, d := range h.deliveries {
|
||||||
|
if !d.State.Final() {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
_, answers := walkAnswers(w, d)
|
||||||
|
if answers || d.Walk != nil && d.Walk.ID == w.ID {
|
||||||
|
ended = append(ended, fmt.Sprintf("%s (%s)", d.ID, d.State))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(ended) == 0 {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
sort.Strings(ended)
|
||||||
|
return "; every delivery it carries ended: " + strings.Join(ended, ", ")
|
||||||
|
}
|
||||||
|
|
||||||
|
// lapsedOn are the deliveries, by id, whose word an open walk waiting for this owner's word takes from a stop that
|
||||||
|
// lapsed (novox/hq issue 753): each was stopped by its group for a member delivered since, so its own change waits for
|
||||||
|
// a later walk of its repository that contains its merge (issue 747), and this walk answers that merge — it names it,
|
||||||
|
// or is made at it — and has not left it out. Such a walk is the one that delivery waits for: it gives it its word, so
|
||||||
|
// a walk whose every other delivery ended is never left without one. A walk at the merge of a delivery stopped for
|
||||||
|
// itself, or by a person, gets nothing from it. Under the lock.
|
||||||
|
//
|
||||||
|
// In its group's order (review of issue 753): a member gives its word only once every member ordered before it reached
|
||||||
|
// the machines, or is itself a stop that lapsed whose merge this same walk answers — the walk then delivers both, in
|
||||||
|
// the group's order among its tiers, as members sharing one walk do. A member behind one still undelivered gives
|
||||||
|
// nothing; lapsedBehind names that one.
|
||||||
|
func (h *Holder) lapsedOn(w Walk) []string {
|
||||||
|
out, _ := h.lapsedOf(w)
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// lapsedOf is lapsedOn, and, by delivery, the member before it in its group that holds back the word of a stop that
|
||||||
|
// lapsed whose merge the walk answers. Under the lock.
|
||||||
|
func (h *Holder) lapsedOf(w Walk) ([]string, map[string]string) {
|
||||||
|
if w.Release != nil || w.IsBatch() || !w.Open() || !w.Seen().Waits {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
var out []string
|
||||||
|
behind := map[string]string{}
|
||||||
|
for _, d := range h.deliveries {
|
||||||
|
if !h.lapsedAnswered(w, d) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if before := h.lapsedBehind(w, d); before != "" {
|
||||||
|
behind[d.ID] = before
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
out = append(out, d.ID)
|
||||||
|
}
|
||||||
|
sort.Strings(out)
|
||||||
|
return out, behind
|
||||||
|
}
|
||||||
|
|
||||||
|
// lapsedAnswered is whether a delivery's stop lapsed and the walk answers its merge, not left out.
|
||||||
|
func (h *Holder) lapsedAnswered(w Walk, d *Delivery) bool {
|
||||||
|
if d.MergedAs == "" || !h.stopLapsed(d) {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
if _, ok := walkAnswers(w, d); !ok {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
_, left := w.leftOut(d.Repository, d.MergedAs)
|
||||||
|
return !left
|
||||||
|
}
|
||||||
|
|
||||||
|
// lapsedBehind is the member ordered before a delivery in its group that has not reached the machines and that this
|
||||||
|
// walk does not deliver with it, with its state; empty for none.
|
||||||
|
func (h *Holder) lapsedBehind(w Walk, d *Delivery) string {
|
||||||
|
g := h.groups[d.Group]
|
||||||
|
if d.Group == "" || g == nil {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
members := h.membersOf(g)
|
||||||
|
byID := map[string]*Delivery{}
|
||||||
|
for _, m := range members {
|
||||||
|
byID[m.ID] = m
|
||||||
|
}
|
||||||
|
for _, m := range h.inOrder(g, members) {
|
||||||
|
if m.ID == d.ID {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
o := byID[m.ID]
|
||||||
|
if o == nil || memberReached(o) || h.lapsedAnswered(w, o) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
return fmt.Sprintf("%s (%s)", o.ID, o.State)
|
||||||
|
}
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
|
// lapsedSaid is why a walk whose word stops that lapsed give still waits: the members of their groups not yet
|
||||||
|
// delivered before them, what the last tick held it for, or the controller's refusal of the word. Under the lock.
|
||||||
|
func (h *Holder) lapsedSaid(w Walk) string {
|
||||||
|
var parts []string
|
||||||
|
_, behind := h.lapsedOf(w)
|
||||||
|
ids := make([]string, 0, len(behind))
|
||||||
|
for id := range behind {
|
||||||
|
ids = append(ids, id)
|
||||||
|
}
|
||||||
|
sort.Strings(ids)
|
||||||
|
for _, id := range ids {
|
||||||
|
parts = append(parts, fmt.Sprintf("%s waits for %s, ordered before it in its group", id, behind[id]))
|
||||||
|
}
|
||||||
|
if why := h.waitsWhy[w.ID]; why != "" {
|
||||||
|
parts = append(parts, "not let go yet: "+why)
|
||||||
|
}
|
||||||
|
if err := h.deliverErr[w.ID]; err != "" {
|
||||||
|
parts = append(parts, "the controller refused its word: "+err)
|
||||||
|
}
|
||||||
|
lapsed, _ := h.lapsedOf(w)
|
||||||
|
for _, id := range lapsed {
|
||||||
|
if d := h.deliveries[id]; d != nil && strings.Contains(d.LetGoWhy, lapsedWord(w.ID)) {
|
||||||
|
parts = append(parts, fmt.Sprintf("its word was given for %s at %s, and the controller has not started it",
|
||||||
|
id, d.LetGo.Local().Format("15:04:05")))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return strings.Join(parts, "; ")
|
||||||
|
}
|
||||||
|
|
||||||
|
// lapsedOwed is whether a walk's word is owed by stops that lapsed: given now, or once a member before one is delivered.
|
||||||
|
func (h *Holder) lapsedOwed(w Walk) bool {
|
||||||
|
lapsed, behind := h.lapsedOf(w)
|
||||||
|
return len(lapsed) > 0 || len(behind) > 0
|
||||||
|
}
|
||||||
|
|
||||||
|
// lapsedTurns registers, for every open walk waiting for this owner's word, the turn of each delivery whose stop
|
||||||
|
// lapsed and whose merge it answers (lapsedOn, novox/hq issue 753). The word is then given as for any walk: once,
|
||||||
|
// when every live delivery it carries is at its turn too and it carries no stopped change still held back (words).
|
||||||
|
// Given once per walk: the walk's id is in the delivery's LetGoWhy once its word was given. Under the lock.
|
||||||
|
func (h *Holder) lapsedTurns() {
|
||||||
|
for id, s := range h.walks {
|
||||||
|
for _, did := range h.lapsedOn(s.walk) {
|
||||||
|
d := h.deliveries[did]
|
||||||
|
if strings.Contains(d.LetGoWhy, lapsedWord(id)) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if h.turning == nil {
|
||||||
|
h.turning = map[string]*turnWant{}
|
||||||
|
}
|
||||||
|
want := h.turning[id]
|
||||||
|
if want == nil {
|
||||||
|
want = &turnWant{}
|
||||||
|
h.turning[id] = want
|
||||||
|
}
|
||||||
|
if !containsString(want.ids, d.ID) {
|
||||||
|
want.ids = append(want.ids, d.ID)
|
||||||
|
want.whys = append(want.whys, fmt.Sprintf("%s: %s its merge %s, stopped by its group for %s, whose change "+
|
||||||
|
"was delivered since; it waits for a later walk of %s that contains its merge, and this is that "+
|
||||||
|
"walk (novox/hq issue 753)", d.ID, lapsedWord(id), shortOf(d.MergedAs, 8), d.stoppedFor(), d.Repository))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// lapsedWord is how the word a lapsed stop gives a walk names that walk.
|
||||||
|
func lapsedWord(walk string) string { return "the walk " + walk + " answers" }
|
||||||
|
|
||||||
// words are the asks of the word of every walk whose turn came for every live delivery it carries: one ask per
|
// words are the asks of the word of every walk whose turn came for every live delivery it carries: one ask per
|
||||||
// walk, however many deliveries it carries, and none while one of them is not at its turn (ADR 0276 decision 5).
|
// walk, however many deliveries it carries, and none while one of them is not at its turn (ADR 0276 decision 5).
|
||||||
// Under the lock.
|
// Under the lock.
|
||||||
@@ -388,6 +569,9 @@ func (h *Holder) words() []func() {
|
|||||||
if stopped, _ := h.carriesStopped(walk); len(stopped) > 0 {
|
if stopped, _ := h.carriesStopped(walk); len(stopped) > 0 {
|
||||||
h.waitsWhy[walk] = "it carries " + strings.Join(stopped, ", ") + ", whose delivery was stopped: it is " +
|
h.waitsWhy[walk] = "it carries " + strings.Join(stopped, ", ") + ", whose delivery was stopped: it is " +
|
||||||
"asked to leave that merge out first"
|
"asked to leave that merge out first"
|
||||||
|
if hold := h.coverHeld(walk); hold != "" {
|
||||||
|
h.waitsWhy[walk] = hold
|
||||||
|
}
|
||||||
h.Logf("[mesh-delivery] the walk %s is not let go: it carries %s, whose delivery was stopped", walk,
|
h.Logf("[mesh-delivery] the walk %s is not let go: it carries %s, whose delivery was stopped", walk,
|
||||||
strings.Join(stopped, ", "))
|
strings.Join(stopped, ", "))
|
||||||
continue
|
continue
|
||||||
@@ -403,9 +587,14 @@ func (h *Holder) words() []func() {
|
|||||||
defer h.mu.Unlock()
|
defer h.mu.Unlock()
|
||||||
delete(h.asking, walk)
|
delete(h.asking, walk)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
if h.deliverErr == nil {
|
||||||
|
h.deliverErr = map[string]string{}
|
||||||
|
}
|
||||||
|
h.deliverErr[walk] = err.Error() // said by `stalled` past the grace (review of novox/hq issue 753)
|
||||||
h.Logf("[mesh-delivery] the walk %s could not be let go yet: %v", walk, err)
|
h.Logf("[mesh-delivery] the walk %s could not be let go yet: %v", walk, err)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
delete(h.deliverErr, walk)
|
||||||
for _, id := range ids {
|
for _, id := range ids {
|
||||||
d := h.deliveries[id]
|
d := h.deliveries[id]
|
||||||
if d == nil {
|
if d == nil {
|
||||||
@@ -479,9 +668,28 @@ func (h *Holder) WalksWaiting() []StalledLine {
|
|||||||
switch {
|
switch {
|
||||||
case len(held) > 0:
|
case len(held) > 0:
|
||||||
out = append(out, held...)
|
out = append(out, held...)
|
||||||
|
case h.coverHeld(id) != "":
|
||||||
|
out = append(out, StalledLine{ID: IDOf(w.Repository, w.Commit), State: Published, Walk: id, For: waited,
|
||||||
|
Bound: coverAskBound.String(), H2: "none: it is asked to leave the merge out past the bound",
|
||||||
|
Says: "held back: " + h.coverHeld(id)})
|
||||||
case h.waitsWhy[id] != "":
|
case h.waitsWhy[id] != "":
|
||||||
out = append(out, StalledLine{ID: IDOf(w.Repository, w.Commit), State: Published, Walk: id, For: waited,
|
out = append(out, StalledLine{ID: IDOf(w.Repository, w.Commit), State: Published, Walk: id, For: waited,
|
||||||
Bound: "0s", H2: "none: it goes at its turn", Says: "not let go yet: " + h.waitsWhy[id]})
|
Bound: "0s", H2: "none: it goes at its turn", Says: "not let go yet: " + h.waitsWhy[id]})
|
||||||
|
case !live && h.lapsedOwed(w):
|
||||||
|
// Its word comes from the stopped deliveries whose merge it answers (novox/hq issue 753): said, never
|
||||||
|
// unheld, while the controller has not taken it, with what holds it.
|
||||||
|
lapsed, _ := h.lapsedOf(w)
|
||||||
|
says := "its word is owed by the stopped deliveries whose merge it answers, each stopped for a member " +
|
||||||
|
"delivered since (novox/hq issue 753)"
|
||||||
|
if len(lapsed) > 0 {
|
||||||
|
says = "its word is given for " + strings.Join(lapsed, ", ") + ", stopped for a member delivered since, " +
|
||||||
|
"whose merge it answers (novox/hq issue 753)"
|
||||||
|
}
|
||||||
|
if why := h.lapsedSaid(w); why != "" {
|
||||||
|
says += "; " + why
|
||||||
|
}
|
||||||
|
out = append(out, StalledLine{ID: IDOf(w.Repository, w.Commit), State: Published, Walk: id, For: waited,
|
||||||
|
Bound: "0s", H2: "none: it goes at its turn", Says: says})
|
||||||
case !live:
|
case !live:
|
||||||
says := "no delivery that has not ended follows it or is answered by it, so nothing gives it its word: " +
|
says := "no delivery that has not ended follows it or is answered by it, so nothing gives it its word: " +
|
||||||
"`mesh-delivery.show` for the deliveries of " + w.Repository
|
"`mesh-delivery.show` for the deliveries of " + w.Repository
|
||||||
@@ -492,7 +700,7 @@ func (h *Holder) WalksWaiting() []StalledLine {
|
|||||||
"), yet the walk moves " + strings.Join(walkModules(w), ", ") + " — nothing gives it its word"
|
"), yet the walk moves " + strings.Join(walkModules(w), ", ") + " — nothing gives it its word"
|
||||||
}
|
}
|
||||||
out = append(out, StalledLine{ID: IDOf(w.Repository, w.Commit), State: Unheld, Walk: id, For: waited,
|
out = append(out, StalledLine{ID: IDOf(w.Repository, w.Commit), State: Unheld, Walk: id, For: waited,
|
||||||
Bound: "0s", H2: "none: no delivery holds the walk — the operator's", Says: says})
|
Bound: "0s", H2: "none: no delivery holds the walk — the operator's", Says: says + h.endedSaid(w)})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
sort.Slice(out, func(i, j int) bool {
|
sort.Slice(out, func(i, j int) bool {
|
||||||
|
|||||||
@@ -150,6 +150,9 @@ func statusOf(d *Delivery) (string, string) {
|
|||||||
case Delivered:
|
case Delivered:
|
||||||
return "success", "delivered"
|
return "success", "delivered"
|
||||||
case Failed:
|
case Failed:
|
||||||
|
if d.coveredStop() {
|
||||||
|
return "success", clip(d.coveredStopSaid(), 140) // its change reached the machines (novox/hq issue 747)
|
||||||
|
}
|
||||||
return "failure", clip("failed: "+why, 140)
|
return "failure", clip("failed: "+why, 140)
|
||||||
case Superseded:
|
case Superseded:
|
||||||
return "warning", clip("superseded: "+why, 140)
|
return "warning", clip("superseded: "+why, 140)
|
||||||
@@ -178,7 +181,7 @@ func ViewBody(d *Delivery, g *Group) string {
|
|||||||
var b strings.Builder
|
var b strings.Builder
|
||||||
fmt.Fprintf(&b, "%s\n**Delivery** `%s` — **%s** since %s\n\n", viewMarker, d.ID, d.State, d.Since.UTC().Format(time.RFC3339))
|
fmt.Fprintf(&b, "%s\n**Delivery** `%s` — **%s** since %s\n\n", viewMarker, d.ID, d.State, d.Since.UTC().Format(time.RFC3339))
|
||||||
if d.coveredStop() {
|
if d.coveredStop() {
|
||||||
fmt.Fprintf(&b, "**Covered** after its stop: a later merge that contains it was delivered: %s\n\n", d.CoveredBy)
|
fmt.Fprintf(&b, "**Covered**: %s\n\n", d.coveredStopSaid())
|
||||||
}
|
}
|
||||||
if d.HeldWhy != "" && d.State == Held {
|
if d.HeldWhy != "" && d.State == Held {
|
||||||
fmt.Fprintf(&b, "Held for a person: %s — `mesh-delivery.release` with why lets it go on.\n\n", d.HeldWhy)
|
fmt.Fprintf(&b, "Held for a person: %s — `mesh-delivery.release` with why lets it go on.\n\n", d.HeldWhy)
|
||||||
|
|||||||
@@ -106,6 +106,8 @@ type fakeController struct {
|
|||||||
moves map[string][]string
|
moves map[string][]string
|
||||||
plans int
|
plans int
|
||||||
down bool
|
down bool
|
||||||
|
// refuseDeliver refuses every word, as a controller that will not take it does (review of novox/hq issue 753).
|
||||||
|
refuseDeliver bool
|
||||||
// onCheck runs as a check is asked, before it is answered: what the forge held at that moment.
|
// onCheck runs as a check is asked, before it is answered: what the forge held at that moment.
|
||||||
onCheck func()
|
onCheck func()
|
||||||
// planned is what Plan answers instead of "builds app", when set.
|
// planned is what Plan answers instead of "builds app", when set.
|
||||||
@@ -182,6 +184,9 @@ func (c *fakeController) Deliver(walk, why string) error {
|
|||||||
if c.down {
|
if c.down {
|
||||||
return errors.New("down")
|
return errors.New("down")
|
||||||
}
|
}
|
||||||
|
if c.refuseDeliver {
|
||||||
|
return errors.New("the controller refuses the word")
|
||||||
|
}
|
||||||
w, known := c.walks[walk]
|
w, known := c.walks[walk]
|
||||||
if !known || w.Delivery == nil || w.Delivery.Go != nil {
|
if !known || w.Delivery == nil || w.Delivery.Go != nil {
|
||||||
return fmt.Errorf("%s waits for nobody", walk)
|
return fmt.Errorf("%s waits for nobody", walk)
|
||||||
|
|||||||
@@ -60,6 +60,8 @@ type Holder struct {
|
|||||||
waitsWhy map[string]string
|
waitsWhy map[string]string
|
||||||
// asking are the walks whose word is being asked right now, outside the lock: not asked twice meanwhile.
|
// asking are the walks whose word is being asked right now, outside the lock: not asked twice meanwhile.
|
||||||
asking map[string]bool
|
asking map[string]bool
|
||||||
|
// deliverErr is, by walk, the controller's last refusal of its word, until it is given (review of issue 753).
|
||||||
|
deliverErr map[string]string
|
||||||
// leaving are, by walk, the merges a stop gathered to leave out of it, asked once (novox/hq ADR 0299).
|
// leaving are, by walk, the merges a stop gathered to leave out of it, asked once (novox/hq ADR 0299).
|
||||||
leaving map[string][]string
|
leaving map[string][]string
|
||||||
// leaveAsked is when a walk was last asked to leave out the stopped merges it carries (leaveStopped).
|
// leaveAsked is when a walk was last asked to leave out the stopped merges it carries (leaveStopped).
|
||||||
@@ -67,6 +69,13 @@ type Holder struct {
|
|||||||
// leaveRefused is, by walk, what the controller answered when it refused both to leave a stopped merge out and
|
// leaveRefused is, by walk, what the controller answered when it refused both to leave a stopped merge out and
|
||||||
// to stop the walk whole.
|
// to stop the walk whole.
|
||||||
leaveRefused map[string]string
|
leaveRefused map[string]string
|
||||||
|
// coverAsked are the deliveries that failed or were stopped whose cover by a later finished walk waits for the
|
||||||
|
// forge's word on containment, worked out each pass (novox/hq issue 747): a walk carrying a merge stopped for one of
|
||||||
|
// them is held back meanwhile, never asked to leave it out before the answer.
|
||||||
|
coverAsked map[string]bool
|
||||||
|
// coverAskedSince is, by delivery, when the forge was first asked about its cover and has not answered: past
|
||||||
|
// coverAskBound the walk is no longer held back for it, and is asked to leave the merge out (review of issue 747).
|
||||||
|
coverAskedSince map[string]time.Time
|
||||||
// tried counts the owed effects Flush tried since this owner started, by how each ended (novox/hq issue 407).
|
// tried counts the owed effects Flush tried since this owner started, by how each ended (novox/hq issue 407).
|
||||||
tried owedTries
|
tried owedTries
|
||||||
// marks are the open pull requests read with a ready mark, by owner/repo#number (novox/hq issue 722, ready.go).
|
// marks are the open pull requests read with a ready mark, by owner/repo#number (novox/hq issue 722, ready.go).
|
||||||
@@ -303,19 +312,25 @@ func (h *Holder) PullUpdated(p PullEvent) {
|
|||||||
defer h.mu.Unlock()
|
defer h.mu.Unlock()
|
||||||
id := IDOf(p.repository(), p.HeadSHA)
|
id := IDOf(p.repository(), p.HeadSHA)
|
||||||
now := h.Now()
|
now := h.Now()
|
||||||
|
startedOver := ""
|
||||||
if d := h.deliveries[id]; d != nil {
|
if d := h.deliveries[id]; d != nil {
|
||||||
// Announced again: nothing new — but for a delivery whose verdict is out of date, the announcement is its
|
if startedOver = startedOverFor(d, p.Number); startedOver == "" {
|
||||||
// check asked again against the trunk's head (the merge tool's ask, novox/hq ADR 0323 §5), and its bound
|
// Announced again: nothing new — but for a delivery whose verdict is out of date, the announcement is its
|
||||||
// runs from now (§7). The base head it was announced with is kept, for a check this owner asks after it.
|
// check asked again against the trunk's head (the merge tool's ask, novox/hq ADR 0323 §5), and its bound
|
||||||
if p.BaseSHA != "" && d.BaseSHA != p.BaseSHA {
|
// runs from now (§7). The base head it was announced with is kept, for a check this owner asks after it.
|
||||||
d.BaseSHA = p.BaseSHA
|
if p.BaseSHA != "" && d.BaseSHA != p.BaseSHA {
|
||||||
h.keep(d)
|
d.BaseSHA = p.BaseSHA
|
||||||
|
h.keep(d)
|
||||||
|
}
|
||||||
|
if d.State == Proposed && d.outOfDate() != "" && d.AskedAgain == nil {
|
||||||
|
d.AskedAgain = &now
|
||||||
|
h.keep(d)
|
||||||
|
}
|
||||||
|
return
|
||||||
}
|
}
|
||||||
if d.State == Proposed && d.outOfDate() != "" && d.AskedAgain == nil {
|
// A delivery stopped only because its pull request closed unmerged does not end its commit's delivery for
|
||||||
d.AskedAgain = &now
|
// another pull request with the same head (novox/hq issue 783): started over below, exactly as for a first
|
||||||
h.keep(d)
|
// announcement, under the same id — never a transition out of Stopped.
|
||||||
}
|
|
||||||
return
|
|
||||||
}
|
}
|
||||||
replaced := ""
|
replaced := ""
|
||||||
for _, old := range h.deliveries {
|
for _, old := range h.deliveries {
|
||||||
@@ -329,11 +344,7 @@ func (h *Holder) PullUpdated(p PullEvent) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
d := &Delivery{ID: id, Repository: p.repository(), Commit: p.HeadSHA, Number: p.Number, Title: p.Title,
|
d, t, err := h.freshAnnouncement(p, now, startedOver)
|
||||||
Base: p.Base, Branch: p.Head, HTMLURL: p.HTMLURL, CloneURL: p.CloneURL, Paths: p.Paths,
|
|
||||||
PathsTruncated: p.PathsTruncated, Removed: p.Removed, ModuleDirs: p.ModuleDirs, ModuleDirsSaid: p.ModuleDirsSaid,
|
|
||||||
After: AfterIn(p.Body), Created: now, BaseSHA: p.BaseSHA}
|
|
||||||
t, err := Apply(d, EvAnnounced, false, Facts{Now: now, Why: fmt.Sprintf("%s#%d's head announced", d.Repository, d.Number)})
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
h.refuse(err)
|
h.refuse(err)
|
||||||
return
|
return
|
||||||
@@ -351,6 +362,34 @@ func (h *Holder) PullUpdated(p PullEvent) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// startedOverFor is why a delivery already kept at a head's id is started over for a pull request's number rather
|
||||||
|
// than taken as announced again: only when it was stopped solely by an earlier, closed duplicate pull request with
|
||||||
|
// the same head — still Stopped, ClosedUnmerged, never merged, and under another number (novox/hq issue 783).
|
||||||
|
// Empty for the ordinary 'announced again' path, or for any other stop (a person's, a group's, its own close).
|
||||||
|
func startedOverFor(d *Delivery, number int) string {
|
||||||
|
if d != nil && d.State == Stopped && d.ClosedUnmerged && d.MergedAs == "" && d.Number != number {
|
||||||
|
return fmt.Sprintf("#%d with the same head closed unmerged", d.Number)
|
||||||
|
}
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
|
// freshAnnouncement makes a delivery for p's head exactly as a first announcement would: its own id, proposed.
|
||||||
|
// startedOver, when not empty, is folded into the transition's Why, naming the pull request this replaces
|
||||||
|
// (novox/hq issue 783); the delivery it replaces is swapped out by the caller keeping the new one under the same
|
||||||
|
// id (keep), never by a transition out of its final state.
|
||||||
|
func (h *Holder) freshAnnouncement(p PullEvent, now time.Time, startedOver string) (*Delivery, Transition, error) {
|
||||||
|
d := &Delivery{ID: IDOf(p.repository(), p.HeadSHA), Repository: p.repository(), Commit: p.HeadSHA, Number: p.Number,
|
||||||
|
Title: p.Title, Base: p.Base, Branch: p.Head, HTMLURL: p.HTMLURL, CloneURL: p.CloneURL, Paths: p.Paths,
|
||||||
|
PathsTruncated: p.PathsTruncated, Removed: p.Removed, ModuleDirs: p.ModuleDirs, ModuleDirsSaid: p.ModuleDirsSaid,
|
||||||
|
After: AfterIn(p.Body), Created: now, BaseSHA: p.BaseSHA}
|
||||||
|
why := fmt.Sprintf("%s#%d's head announced", d.Repository, d.Number)
|
||||||
|
if startedOver != "" {
|
||||||
|
why = fmt.Sprintf("%s; %s", why, startedOver)
|
||||||
|
}
|
||||||
|
t, err := Apply(d, EvAnnounced, false, Facts{Now: now, Why: why})
|
||||||
|
return d, t, err
|
||||||
|
}
|
||||||
|
|
||||||
// takePlaceOf puts a member's new head — a rebase, a push — in its group in the place of the head it superseded,
|
// takePlaceOf puts a member's new head — a rebase, a push — in its group in the place of the head it superseded,
|
||||||
// when the group is closed: a member of it already reached the trunk, and a closed group takes no new member. The
|
// when the group is closed: a member of it already reached the trunk, and a closed group takes no new member. The
|
||||||
// new head has its own check, and the group's composed check runs again for the new heads rather than holding the
|
// new head has its own check, and the group's composed check runs again for the new heads rather than holding the
|
||||||
@@ -482,7 +521,29 @@ func (h *Holder) PullMerged(p PullEvent) {
|
|||||||
}
|
}
|
||||||
now := h.Now()
|
now := h.Now()
|
||||||
var ts []Transition
|
var ts []Transition
|
||||||
if d == nil {
|
if over := startedOverFor(d, p.Number); over != "" {
|
||||||
|
// A delivery stopped only by an earlier, closed duplicate pull request with the same head is not lost when
|
||||||
|
// its merge is heard without the open announcement either: it is started over for the merging pull request
|
||||||
|
// the same way an announcement would, before the merge is taken as usual (novox/hq issue 783). Heard this
|
||||||
|
// way — its own merge, with no open announcement of this pull request — it is made exactly as a first-heard
|
||||||
|
// merge in every field but its id: FromForge, and its check the forge's word on the head, so a long-past
|
||||||
|
// merge replayed from the bus's history still falls under the history rule (issue 313) and is not held for
|
||||||
|
// a person (review round 1).
|
||||||
|
fresh, t, err := h.freshAnnouncement(p, now, over)
|
||||||
|
if err != nil {
|
||||||
|
h.refuse(err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
fresh.FromForge = true
|
||||||
|
if s := p.HeadChecks["mesh/merge-gate"]; s != "" {
|
||||||
|
fresh.Check = &Verdict{At: now, Gate: verdictOfState(s), Summary: "the head's mesh/merge-gate, as the forge holds it",
|
||||||
|
Repo: verdictOfState(p.HeadChecks["mesh/repo-check"])}
|
||||||
|
}
|
||||||
|
d, ts = fresh, []Transition{t}
|
||||||
|
if d.Check != nil {
|
||||||
|
ts = append(ts, settle(d, Facts{Now: now})...)
|
||||||
|
}
|
||||||
|
} else if d == nil {
|
||||||
head := p.HeadSHA
|
head := p.HeadSHA
|
||||||
if head == "" {
|
if head == "" {
|
||||||
head = p.MergeCommit
|
head = p.MergeCommit
|
||||||
@@ -1112,6 +1173,48 @@ func (h *Holder) plan() []func() {
|
|||||||
// A published delivery whose walk ended done before its word is read the same way (novox/hq issue 474): what
|
// A published delivery whose walk ended done before its word is read the same way (novox/hq issue 474): what
|
||||||
// that walk did not build, a later delivered merge may have; and when the cover test says none did, past the
|
// that walk did not build, a later delivered merge may have; and when the cover test says none did, past the
|
||||||
// grace, it is stopped saying so — never left waiting for a start that cannot come.
|
// grace, it is stopped saying so — never left waiting for a start that cannot come.
|
||||||
|
// A delivery that failed or was stopped is read for a later finished walk of its repository that delivered its
|
||||||
|
// change after all: a walk that failed in an outage and was resumed, or the walk of a later merge that contains it
|
||||||
|
// (novox/hq issue 747). Read each tick, from what the walks and the kept deliveries say, so one stuck before this
|
||||||
|
// was built clears by itself after a restart.
|
||||||
|
h.coverAsked = map[string]bool{}
|
||||||
|
if h.coverAskedSince == nil {
|
||||||
|
h.coverAskedSince = map[string]time.Time{}
|
||||||
|
}
|
||||||
|
pending := map[string]bool{}
|
||||||
|
for _, d := range h.deliveries {
|
||||||
|
if d.State != Failed && d.State != Stopped || d.coveredStop() {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
by, ask := h.walkCover(d)
|
||||||
|
if len(ask) > 0 && by == "" {
|
||||||
|
pending[d.ID] = true
|
||||||
|
since, asked := h.coverAskedSince[d.ID]
|
||||||
|
if !asked {
|
||||||
|
since = now
|
||||||
|
h.coverAskedSince[d.ID] = now
|
||||||
|
}
|
||||||
|
// Held back only within the bound: a forge that keeps erroring never holds a walk for good.
|
||||||
|
h.coverAsked[d.ID] = now.Sub(since) < coverAskBound
|
||||||
|
}
|
||||||
|
if by != "" {
|
||||||
|
d.DeliveredBy = by
|
||||||
|
h.tellCovered(d, now)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
for _, in := range ask {
|
||||||
|
if a := h.askContains(d, in, now); a != nil {
|
||||||
|
asks = append(asks, a)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
for id := range h.coverAskedSince {
|
||||||
|
if !pending[id] {
|
||||||
|
delete(h.coverAskedSince, id)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
for _, d := range h.deliveries {
|
for _, d := range h.deliveries {
|
||||||
ended := d.endedUnstarted()
|
ended := d.endedUnstarted()
|
||||||
if d.coveredStop() && !d.CoverTold {
|
if d.coveredStop() && !d.CoverTold {
|
||||||
@@ -1195,6 +1298,8 @@ func (h *Holder) plan() []func() {
|
|||||||
}
|
}
|
||||||
h.turn(d, "its turn: a delivery in no group")
|
h.turn(d, "its turn: a delivery in no group")
|
||||||
}
|
}
|
||||||
|
// A walk that answers the merge of a delivery whose stop lapsed: that delivery's turn (novox/hq issue 753).
|
||||||
|
h.lapsedTurns()
|
||||||
// The word, once per walk, when every live delivery it carries is at its turn.
|
// The word, once per walk, when every live delivery it carries is at its turn.
|
||||||
asks = append(asks, h.words()...)
|
asks = append(asks, h.words()...)
|
||||||
|
|
||||||
@@ -1273,7 +1378,7 @@ func (h *Holder) laterOnItsWay(d *Delivery) string {
|
|||||||
// stays final, and the bus's transition event is not said again.
|
// stays final, and the bus's transition event is not said again.
|
||||||
func (h *Holder) tellCovered(d *Delivery, now time.Time) {
|
func (h *Holder) tellCovered(d *Delivery, now time.Time) {
|
||||||
d.CoverTold = true
|
d.CoverTold = true
|
||||||
line := oneLine(fmt.Sprintf("%s delivery %s: stopped, %s", now.UTC().Format(time.RFC3339), d.ID, d.coveredStopSaid()))
|
line := oneLine(fmt.Sprintf("%s delivery %s: %s, %s", now.UTC().Format(time.RFC3339), d.ID, d.State, d.coveredStopSaid()))
|
||||||
d.Owed = append(d.Owed, Effect{Kind: EffectNote, Commit: d.Commit, Line: line, Since: now})
|
d.Owed = append(d.Owed, Effect{Kind: EffectNote, Commit: d.Commit, Line: line, Since: now})
|
||||||
if d.MergedAs != "" && !sameCommit(d.MergedAs, d.Commit) {
|
if d.MergedAs != "" && !sameCommit(d.MergedAs, d.Commit) {
|
||||||
d.Owed = append(d.Owed, Effect{Kind: EffectNote, Commit: d.MergedAs, Line: line, Since: now})
|
d.Owed = append(d.Owed, Effect{Kind: EffectNote, Commit: d.MergedAs, Line: line, Since: now})
|
||||||
@@ -1283,7 +1388,7 @@ func (h *Holder) tellCovered(d *Delivery, now time.Time) {
|
|||||||
state, desc := statusOf(d)
|
state, desc := statusOf(d)
|
||||||
d.owe(Effect{Kind: EffectStatus, Commit: d.Commit, Context: "mesh/delivery", State: state, Line: desc, Since: now})
|
d.owe(Effect{Kind: EffectStatus, Commit: d.Commit, Context: "mesh/delivery", State: state, Line: desc, Since: now})
|
||||||
}
|
}
|
||||||
h.Logf("[mesh-delivery] %s: stopped, %s", d.ID, d.coveredStopSaid())
|
h.Logf("[mesh-delivery] %s: %s, %s", d.ID, d.State, d.coveredStopSaid())
|
||||||
h.keep(d)
|
h.keep(d)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1294,6 +1399,104 @@ func strandedUncovered(d *Delivery) bool {
|
|||||||
return d.State == Stopped && d.Stranded != "" && d.CoveredBy == ""
|
return d.State == Stopped && d.Stranded != "" && d.CoveredBy == ""
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// walkCover says which later finished walk delivered the change of a delivery that failed or was stopped, or
|
||||||
|
// nothing, and which walk commits the forge must still be asked about (novox/hq issue 747). A walk covers it when it
|
||||||
|
// is of its repository and trunk, done, made after its merge, its commit containing its merge — the merge itself, or
|
||||||
|
// a later one the forge says contains it — and it built every module the delivery's plan moves, each it did not
|
||||||
|
// build being one the delivery's own walk said unchanged (issue 611). This is the containment "one walk answers every
|
||||||
|
// merge it contains" uses (issue 362, ADR 0276), read after the fact: a failed walk resumed to done answers its own
|
||||||
|
// merge, and a later merge's walk answers every merge it contains. A walk that only said a module unchanged for its
|
||||||
|
// own merges says nothing of this one's, so it covers no module that way. Walks are read from the controller's records
|
||||||
|
// this owner holds and from every delivery's kept walk, so a walk long ended still counts after a restart.
|
||||||
|
func (h *Holder) walkCover(d *Delivery) (string, []string) {
|
||||||
|
if d.State != Failed && d.State != Stopped || d.MergedAs == "" || d.Plan == nil || len(d.Plan.Moved) == 0 {
|
||||||
|
return "", nil
|
||||||
|
}
|
||||||
|
var ask []string
|
||||||
|
for _, c := range h.doneWalksOf(d.Repository) {
|
||||||
|
w := c.walk
|
||||||
|
if !sameBranch(w.Branch, d.Base) || !w.Created.IsZero() && w.Created.Before(d.MergedAt.Add(-time.Minute)) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
var missing []string
|
||||||
|
for _, m := range d.Plan.Moved {
|
||||||
|
unchanged := false
|
||||||
|
if d.Walk != nil {
|
||||||
|
_, unchanged = d.Walk.Unchanged[m]
|
||||||
|
}
|
||||||
|
if !containsString(w.Moved, m) && !unchanged {
|
||||||
|
missing = append(missing, m)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(missing) > 0 {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if !sameCommit(c.commit, d.MergedAs) {
|
||||||
|
yes, known := d.ContainedIn[c.commit]
|
||||||
|
if !known && !containsString(ask, c.commit) {
|
||||||
|
ask = append(ask, c.commit)
|
||||||
|
}
|
||||||
|
if !yes {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return fmt.Sprintf("%s (%s at %s into %s, done, contains its merge %s)", w.ID, d.Repository, shortOf(c.commit, 8),
|
||||||
|
orNone(d.Base), shortOf(d.MergedAs, 8)), nil
|
||||||
|
}
|
||||||
|
return "", ask
|
||||||
|
}
|
||||||
|
|
||||||
|
// doneWalk is a finished walk and the commit of its repository it carried.
|
||||||
|
type doneWalk struct {
|
||||||
|
walk *WalkSeen
|
||||||
|
commit string
|
||||||
|
}
|
||||||
|
|
||||||
|
// doneWalksOf are the finished walks of a repository this owner knows, oldest first: the controller's records it
|
||||||
|
// holds, and each delivery's kept walk with the commit of its repository that walk carried for it.
|
||||||
|
func (h *Holder) doneWalksOf(repository string) []doneWalk {
|
||||||
|
byID := map[string]doneWalk{}
|
||||||
|
for id, s := range h.walks {
|
||||||
|
w := s.walk
|
||||||
|
if w.Release != nil || w.IsBatch() || w.State != walkDone || !strings.EqualFold(w.Repository, repository) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
byID[id] = doneWalk{walk: w.Seen(), commit: w.Commit}
|
||||||
|
}
|
||||||
|
for _, o := range h.deliveries {
|
||||||
|
if o.Walk == nil || o.Walk.State != walkDone || !strings.EqualFold(o.Repository, repository) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if _, known := byID[o.Walk.ID]; known {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
commit := o.AnsweredBy
|
||||||
|
if commit == "" && o.MergedAs != "" && sameCommit(o.Walk.Commit, o.MergedAs) {
|
||||||
|
commit = o.MergedAs
|
||||||
|
}
|
||||||
|
if commit == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
byID[o.Walk.ID] = doneWalk{walk: o.Walk, commit: commit}
|
||||||
|
}
|
||||||
|
out := make([]doneWalk, 0, len(byID))
|
||||||
|
for _, c := range byID {
|
||||||
|
out = append(out, c)
|
||||||
|
}
|
||||||
|
sort.Slice(out, func(i, j int) bool {
|
||||||
|
if !out[i].walk.Created.Equal(out[j].walk.Created) {
|
||||||
|
return out[i].walk.Created.Before(out[j].walk.Created)
|
||||||
|
}
|
||||||
|
return out[i].walk.ID < out[j].walk.ID
|
||||||
|
})
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// stopLapsed is whether a member its group stopped for another is stopped for nothing any more (see stopLapsed).
|
||||||
|
func (h *Holder) stopLapsed(d *Delivery) bool {
|
||||||
|
return stopLapsed(d, func(id string) *Delivery { return h.deliveries[id] })
|
||||||
|
}
|
||||||
|
|
||||||
// unwalkedPast is whether a merged delivery has gone without a walk past the bound one opens within, read
|
// unwalkedPast is whether a merged delivery has gone without a walk past the bound one opens within, read
|
||||||
// again after it: what holds it for a person, unless something already ends it.
|
// again after it: what holds it for a person, unless something already ends it.
|
||||||
func (h *Holder) unwalkedPast(d *Delivery, now time.Time) bool {
|
func (h *Holder) unwalkedPast(d *Delivery, now time.Time) bool {
|
||||||
@@ -1736,7 +1939,8 @@ func (h *Holder) turnsOf(g *Group, members []*Delivery, waits map[string]carryWa
|
|||||||
// A superseded member a later delivered walk carried counts as reached, as in GroupState: a group counts a
|
// A superseded member a later delivered walk carried counts as reached, as in GroupState: a group counts a
|
||||||
// covered member as delivered, so the member after it gets its turn (ADR 0270 decision 2, novox/hq
|
// covered member as delivered, so the member after it gets its turn (ADR 0270 decision 2, novox/hq
|
||||||
// issues 442 and 445).
|
// issues 442 and 445).
|
||||||
if d.State.Reached() || d.State == Superseded && d.CoveredBy != "" {
|
// A stop or failure a later walk delivered since counts as reached too (novox/hq issues 480 and 747).
|
||||||
|
if memberReached(d) {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
if sharing != "" && (d.Walk == nil || d.Walk.ID != sharing) {
|
if sharing != "" && (d.Walk == nil || d.Walk.ID != sharing) {
|
||||||
@@ -1761,7 +1965,7 @@ func (h *Holder) turnsOf(g *Group, members []*Delivery, waits map[string]carryWa
|
|||||||
}
|
}
|
||||||
if w.why != "" {
|
if w.why != "" {
|
||||||
for _, later := range ordered[i+1:] {
|
for _, later := range ordered[i+1:] {
|
||||||
if a := h.stopInGroup(byID[later.ID], fmt.Sprintf("its group %s stopped: %s superseded, and %s", g.ID,
|
if a := h.stopInGroup(byID[later.ID], d.ID, fmt.Sprintf("its group %s stopped: %s superseded, and %s", g.ID,
|
||||||
d.ID, w.why)); a != nil {
|
d.ID, w.why)); a != nil {
|
||||||
asks = append(asks, a)
|
asks = append(asks, a)
|
||||||
}
|
}
|
||||||
@@ -1788,9 +1992,13 @@ func (h *Holder) turnsOf(g *Group, members []*Delivery, waits map[string]carryWa
|
|||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
return asks
|
return asks
|
||||||
|
case h.stopLapsed(d):
|
||||||
|
// Stopped for a member whose change reached the machines since: it stops nothing more. Its own change is
|
||||||
|
// delivered by a later walk of its repository that contains it, which then covers it (novox/hq issue 747).
|
||||||
|
continue
|
||||||
case d.State == Failed || d.State == Stopped:
|
case d.State == Failed || d.State == Stopped:
|
||||||
for _, later := range ordered[i+1:] {
|
for _, later := range ordered[i+1:] {
|
||||||
if a := h.stopInGroup(byID[later.ID], fmt.Sprintf("its group %s stopped: %s %s", g.ID, d.ID, d.State)); a != nil {
|
if a := h.stopInGroup(byID[later.ID], d.ID, fmt.Sprintf("its group %s stopped: %s %s", g.ID, d.ID, d.State)); a != nil {
|
||||||
asks = append(asks, a)
|
asks = append(asks, a)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -2028,7 +2236,7 @@ func mayCarry(c, d *Delivery, module string) bool {
|
|||||||
// controller ends the walk and walks the other merges again without it, and without each later merge of its
|
// controller ends the walk and walks the other merges again without it, and without each later merge of its
|
||||||
// repository, which contains it. Every member a pass stops on one walk is gathered first, and the walk is asked
|
// repository, which contains it. Every member a pass stops on one walk is gathered first, and the walk is asked
|
||||||
// once, by the first member's ask, after the pass: a member is never walked again because its stop came second.
|
// once, by the first member's ask, after the pass: a member is never walked again because its stop came second.
|
||||||
func (h *Holder) stopInGroup(d *Delivery, why string) func() {
|
func (h *Holder) stopInGroup(d *Delivery, cause, why string) func() {
|
||||||
if d == nil || d.State.Final() {
|
if d == nil || d.State.Final() {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
@@ -2037,6 +2245,7 @@ func (h *Holder) stopInGroup(d *Delivery, why string) func() {
|
|||||||
h.refuse(err)
|
h.refuse(err)
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
d.StoppedFor = cause // the member it was stopped for: once that one is delivered, the stop lapses (novox/hq issue 747)
|
||||||
h.moved(d, []Transition{t})
|
h.moved(d, []Transition{t})
|
||||||
// Its open walk; or, its walk stopped for another merge, the batch or walk that walks its merge again (review
|
// Its open walk; or, its walk stopped for another merge, the batch or walk that walks its merge again (review
|
||||||
// of #221): a merge walked again and then stopped by its group is left out there too.
|
// of #221): a merge walked again and then stopped by its group is left out there too.
|
||||||
@@ -2118,6 +2327,12 @@ func (h *Holder) carriesStopped(walk string) ([]string, string) {
|
|||||||
if last := d.Transitions[len(d.Transitions)-1]; last.Event != EvStop || last.By == "" {
|
if last := d.Transitions[len(d.Transitions)-1]; last.Event != EvStop || last.By == "" {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
// A stop whose change a later walk delivered since, or one its group made for a member delivered since,
|
||||||
|
// holds nothing back: the group stops a walk only for a member whose change is still undelivered (novox/hq
|
||||||
|
// issue 747).
|
||||||
|
if d.coveredStop() || h.stopLapsed(d) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
// By the merges it names alone: its own commit may be another's merge it no longer answers.
|
// By the merges it names alone: its own commit may be another's merge it no longer answers.
|
||||||
if !slices.ContainsFunc(s.walk.Delivery.Merges, func(m WalkMerge) bool {
|
if !slices.ContainsFunc(s.walk.Delivery.Merges, func(m WalkMerge) bool {
|
||||||
return strings.EqualFold(m.Repository, d.Repository) && sameCommit(m.Commit, d.MergedAs)
|
return strings.EqualFold(m.Repository, d.Repository) && sameCommit(m.Commit, d.MergedAs)
|
||||||
@@ -2135,6 +2350,51 @@ func (h *Holder) carriesStopped(walk string) ([]string, string) {
|
|||||||
return out, why
|
return out, why
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// coverAwaited is whether a stopped merge a walk carries, given as `<repository>@<commit>`, waits for the forge's
|
||||||
|
// word on whether a later finished walk delivered it or the member its group stopped it for (novox/hq issue 747).
|
||||||
|
// It says since when the forge was first asked, the earliest of them. Held only within coverAskBound of the first
|
||||||
|
// ask: past it the forge's silence or error lets the walk be asked to leave the merge out. Under the lock.
|
||||||
|
func (h *Holder) coverAwaited(merges []string) (time.Time, bool) {
|
||||||
|
var since time.Time
|
||||||
|
awaited := false
|
||||||
|
for _, d := range h.deliveries {
|
||||||
|
if d.State != Stopped || d.MergedAs == "" || !containsString(merges, d.Repository+"@"+d.MergedAs) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
for _, id := range []string{d.ID, d.stoppedFor()} {
|
||||||
|
if id == "" || !h.coverAsked[id] {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if at := h.coverAskedSince[id]; !awaited || at.Before(since) {
|
||||||
|
since = at
|
||||||
|
}
|
||||||
|
awaited = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return since, awaited
|
||||||
|
}
|
||||||
|
|
||||||
|
// coverHeld says why a walk is held back while the forge is asked whether a later finished walk delivered a stopped
|
||||||
|
// merge it carries (novox/hq issue 747), or nothing when it is not. Said by `waits`, `stalled` and `stalled walks`, so
|
||||||
|
// a forge that cannot answer is loud. Under the lock.
|
||||||
|
func (h *Holder) coverHeld(walk string) string {
|
||||||
|
without, _ := h.carriesStopped(walk)
|
||||||
|
if len(without) == 0 {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
since, awaited := h.coverAwaited(without)
|
||||||
|
if !awaited {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
return fmt.Sprintf("it carries %s, whose delivery was stopped, and the forge is asked since %s whether a later "+
|
||||||
|
"finished walk delivered it; it is asked to leave the merge out if no answer comes by %s (novox/hq issue 747)",
|
||||||
|
strings.Join(without, ", "), since.Local().Format("15:04:05"), since.Add(coverAskBound).Local().Format("15:04:05"))
|
||||||
|
}
|
||||||
|
|
||||||
|
// coverAskBound is how long a walk carrying a stopped merge is held back while the forge is asked whether a later
|
||||||
|
// finished walk delivered it (novox/hq issue 747); past it, the walk is asked to leave the merge out.
|
||||||
|
const coverAskBound = 10 * time.Minute
|
||||||
|
|
||||||
// lastWhyOf is the reason of a delivery's last transition.
|
// lastWhyOf is the reason of a delivery's last transition.
|
||||||
func lastWhyOf(d *Delivery) string {
|
func lastWhyOf(d *Delivery) string {
|
||||||
if len(d.Transitions) == 0 {
|
if len(d.Transitions) == 0 {
|
||||||
@@ -2170,6 +2430,9 @@ func (h *Holder) leaveStopped() []func() {
|
|||||||
if without, _ := h.carriesStopped(id); len(without) > 0 {
|
if without, _ := h.carriesStopped(id); len(without) > 0 {
|
||||||
said := fmt.Sprintf("its walk %s is not let go: it carries %s, whose delivery was stopped, and is asked to "+
|
said := fmt.Sprintf("its walk %s is not let go: it carries %s, whose delivery was stopped, and is asked to "+
|
||||||
"leave it out (novox/hq ADR 0299)", id, strings.Join(without, ", "))
|
"leave it out (novox/hq ADR 0299)", id, strings.Join(without, ", "))
|
||||||
|
if hold := h.coverHeld(id); hold != "" {
|
||||||
|
said = "its walk " + id + " is held back: " + hold
|
||||||
|
}
|
||||||
if r := h.leaveRefused[id]; r != "" {
|
if r := h.leaveRefused[id]; r != "" {
|
||||||
said += "; the controller refused: " + r
|
said += "; the controller refused: " + r
|
||||||
}
|
}
|
||||||
@@ -2193,6 +2456,9 @@ func (h *Holder) leaveStopped() []func() {
|
|||||||
delete(h.leaveRefused, id)
|
delete(h.leaveRefused, id)
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
if _, awaited := h.coverAwaited(without); awaited {
|
||||||
|
continue // held back, not stopped, until the forge says whether a later walk delivered it (issue 747)
|
||||||
|
}
|
||||||
// Once per record of it: the record that answers an ask is newer. Asked again after five minutes, should
|
// Once per record of it: the record that answers an ask is newer. Asked again after five minutes, should
|
||||||
// the ask have gone nowhere.
|
// the ask have gone nowhere.
|
||||||
if a, asked := h.leaveAsked[id]; asked && seen.walk.Revision <= a.revision && now.Sub(a.at) < 5*time.Minute {
|
if a, asked := h.leaveAsked[id]; asked && seen.walk.Revision <= a.revision && now.Sub(a.at) < 5*time.Minute {
|
||||||
@@ -2314,18 +2580,28 @@ func againWords(a StopAnswer) string {
|
|||||||
func GroupState(g *Group, members []*Delivery) (string, string) {
|
func GroupState(g *Group, members []*Delivery) (string, string) {
|
||||||
count := map[State]int{}
|
count := map[State]int{}
|
||||||
reached, ended := 0, 0
|
reached, ended := 0, 0
|
||||||
var uncovered []string
|
var uncovered, lapsed []string
|
||||||
|
byID := map[string]*Delivery{}
|
||||||
|
for _, d := range members {
|
||||||
|
byID[d.ID] = d
|
||||||
|
}
|
||||||
for _, d := range members {
|
for _, d := range members {
|
||||||
count[d.State]++
|
|
||||||
if d.State.Final() {
|
if d.State.Final() {
|
||||||
ended++
|
ended++
|
||||||
}
|
}
|
||||||
switch {
|
switch {
|
||||||
case d.State.Reached(), d.State == Superseded && d.CoveredBy != "":
|
case memberReached(d):
|
||||||
|
// Reached, whatever its final state: a failure or stop a later walk delivered since is no failure of the
|
||||||
|
// group (novox/hq issue 747).
|
||||||
reached++
|
reached++
|
||||||
|
continue
|
||||||
|
case stopLapsed(d, func(id string) *Delivery { return byID[id] }):
|
||||||
|
lapsed = append(lapsed, d.ID)
|
||||||
|
continue
|
||||||
case d.State == Superseded:
|
case d.State == Superseded:
|
||||||
uncovered = append(uncovered, d.ID)
|
uncovered = append(uncovered, d.ID)
|
||||||
}
|
}
|
||||||
|
count[d.State]++
|
||||||
}
|
}
|
||||||
n := len(members)
|
n := len(members)
|
||||||
switch {
|
switch {
|
||||||
@@ -2335,6 +2611,11 @@ func GroupState(g *Group, members []*Delivery) (string, string) {
|
|||||||
return "stopped", "a member was stopped"
|
return "stopped", "a member was stopped"
|
||||||
case n > 0 && reached == n:
|
case n > 0 && reached == n:
|
||||||
return "delivered", "every member is delivered"
|
return "delivered", "every member is delivered"
|
||||||
|
case g.Closed && len(lapsed) > 0 && reached+len(lapsed) == n:
|
||||||
|
// Stopped for a member delivered since: its own change waits for a later walk of its repository, which then
|
||||||
|
// covers it (novox/hq issue 747).
|
||||||
|
return "delivering", fmt.Sprintf("%d of %d delivered; %s, stopped for a member delivered since, waits for a "+
|
||||||
|
"later walk of its repository", reached, n, strings.Join(lapsed, ", "))
|
||||||
case g.Closed && n > 0 && ended == n:
|
case g.Closed && n > 0 && ended == n:
|
||||||
return "stopped", supersededWhy + strings.Join(uncovered, ", ")
|
return "stopped", supersededWhy + strings.Join(uncovered, ", ")
|
||||||
case g.Closed && g.Check != nil && (g.Check.OutOfDate != "" || g.Check.Verdict == "" ||
|
case g.Closed && g.Check != nil && (g.Check.OutOfDate != "" || g.Check.Verdict == "" ||
|
||||||
|
|||||||
@@ -0,0 +1,258 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// novox/hq issue 747: in the group fix/a-module-moves-only-when-its-program-changes, the controller's member #336
|
||||||
|
// (merged as 5d22d7df) failed at 22:32 during an outage of the forge's tools. The walk of the controller's next merge,
|
||||||
|
// #339 at 60cc9ec3, which contains 5d22d7df, failed with it and was resumed to done by a rebuild. mesh-delivery still
|
||||||
|
// held #336 failed, so its group stopped every later walk of the catalogue that carried the catalogue member's merge
|
||||||
|
// 2edef5a0: the catalogue could deliver nothing. A failed or stopped delivery is covered once a later finished walk of
|
||||||
|
// its repository that contains its merge built its modules, and a group stops a walk only for a member whose change is
|
||||||
|
// still undelivered.
|
||||||
|
//
|
||||||
|
// The shape is the live state of 2026-10-14 23:07, as mesh-delivery.show gave it, read back after a restart: the
|
||||||
|
// controller holds none of the walks of the night any more, only the walk of the catalogue's next merge.
|
||||||
|
|
||||||
|
const (
|
||||||
|
g747 = "fix/a-module-moves-only-when-its-program-changes"
|
||||||
|
ctl747, cat747 = "novox/mesh-controller", "novox/mesh-catalog"
|
||||||
|
head336, merge336 = "b34055f580d4e2a4", "5d22d7df86c73452"
|
||||||
|
head339, merge339 = "e4acab32d6a20ade", "60cc9ec363dfa7c6"
|
||||||
|
head301, merge301 = "d94764f00168471f", "2edef5a00ae05605"
|
||||||
|
merge307 = "0f76b6e2aaaaaaaa"
|
||||||
|
)
|
||||||
|
|
||||||
|
var (
|
||||||
|
at747 = time.Date(2026, 10, 14, 20, 21, 6, 0, time.UTC)
|
||||||
|
id336, id339, id301 = IDOf(ctl747, head336), IDOf(ctl747, head339), IDOf(cat747, head301)
|
||||||
|
unchangedRouteProxy747 = map[string]string{"route-proxy": "its build source holds none of the changed files"}
|
||||||
|
)
|
||||||
|
|
||||||
|
// night747 is the stuck state, with the forge saying, or not, that 60cc9ec3 contains 5d22d7df.
|
||||||
|
func night747(t *testing.T, contained bool) *world { return night747With(t, contained, nil) }
|
||||||
|
|
||||||
|
// night747With is night747 with #339's walk having built only the modules given, saying the others unchanged.
|
||||||
|
func night747With(t *testing.T, contained bool, built []string) *world {
|
||||||
|
t.Helper()
|
||||||
|
return night747Over(t, func(f *fakeForge) { f.contains[merge336+" in "+merge339] = contained }, built)
|
||||||
|
}
|
||||||
|
|
||||||
|
// night747Over is night747With with the forge set as given: one that is not told the pair cannot say.
|
||||||
|
func night747Over(t *testing.T, set func(*fakeForge), built []string) *world {
|
||||||
|
t.Helper()
|
||||||
|
at := at747
|
||||||
|
failedAt := at.Add(11 * time.Minute)
|
||||||
|
store := newMemStore()
|
||||||
|
moved339, unchanged339 := []string{"build-agent", "mesh-controller"}, unchangedRouteProxy747
|
||||||
|
if built != nil {
|
||||||
|
moved339, unchanged339 = built, map[string]string{}
|
||||||
|
for _, m := range []string{"build-agent", "mesh-controller", "route-proxy"} {
|
||||||
|
if !containsString(built, m) {
|
||||||
|
unchanged339[m] = "its build source holds none of the files #339 changed"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
m336 := &Delivery{ID: id336, Repository: ctl747, Commit: head336, Number: 336, Base: "main", Branch: g747, Group: g747,
|
||||||
|
State: Failed, Since: failedAt, Created: at.Add(-time.Hour), MergedAs: merge336, MergedAt: at,
|
||||||
|
AnsweredBy: merge336, Check: &Verdict{Gate: "pass"},
|
||||||
|
Plan: planMoving(ctl747, "build-agent", "mesh-controller", "route-proxy"),
|
||||||
|
Walk: &WalkSeen{ID: "plan-336", Commit: merge336, State: walkFailed, Branch: "main", MovedSaid: true,
|
||||||
|
Note: "mesh-controller failed to build in tier 0", Created: at.Add(7 * time.Minute), Tiers: 2,
|
||||||
|
Unchanged: unchangedRouteProxy747},
|
||||||
|
Transitions: []Transition{{At: failedAt, From: Delivering, To: Failed, Event: EvFailed,
|
||||||
|
Why: "its walk failed, carrying its own commit"}}}
|
||||||
|
m339 := &Delivery{ID: id339, Repository: ctl747, Commit: head339, Number: 339, Base: "main", Branch: "fix/726",
|
||||||
|
State: Failed, Since: failedAt, Created: at.Add(-10 * time.Minute), MergedAs: merge339, MergedAt: at.Add(7 * time.Minute),
|
||||||
|
AnsweredBy: merge339, Check: &Verdict{Gate: "pass"}, Plan: planMoving(ctl747, "mesh-controller"),
|
||||||
|
// Its walk failed with #336's, and a rebuild resumed it to done: the record it keeps says so.
|
||||||
|
Walk: &WalkSeen{ID: "plan-339", Commit: merge339, State: walkDone, Branch: "main", MovedSaid: true,
|
||||||
|
Moved: moved339, Unchanged: unchanged339,
|
||||||
|
Created: at.Add(11 * time.Minute), Tiers: 2, Tier: 2},
|
||||||
|
Transitions: []Transition{{At: failedAt, From: Delivering, To: Failed, Event: EvFailed,
|
||||||
|
Why: "its walk failed, carrying its own commit"}}}
|
||||||
|
m301 := &Delivery{ID: id301, Repository: cat747, Commit: head301, Number: 301, Base: "main", Branch: g747, Group: g747,
|
||||||
|
State: Stopped, Since: failedAt, Created: at.Add(-2 * time.Hour), MergedAs: merge301, MergedAt: at.Add(7 * time.Second),
|
||||||
|
Check: &Verdict{Gate: "pass"}, Plan: planMoving(cat747, "build-agent", "route-proxy"),
|
||||||
|
Transitions: []Transition{{At: failedAt, From: Ready, To: Stopped, Event: EvStop, By: byOwner,
|
||||||
|
Why: "its group " + g747 + " stopped: " + id336 + " failed"}}}
|
||||||
|
for _, d := range []*Delivery{m336, m339, m301} {
|
||||||
|
if err := store.PutDelivery(d); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
g := &Group{ID: g747, Members: []string{id301, id336}, Closed: true, Order: []string{id336, id301}, Said: "failed",
|
||||||
|
OrderOf: commitsOf([]*Delivery{m301, m336}), Check: &GroupCheck{Asked: "check-1", At: at.Add(-time.Minute),
|
||||||
|
Verdict: "pass", Summary: "both compose", Commits: commitsOf([]*Delivery{m301, m336})}}
|
||||||
|
if err := store.PutGroup(g); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
ctlr := newFakeController()
|
||||||
|
// The catalogue's next merge: its walk carries 2edef5a0, merged before it and never walked, and waits for its word.
|
||||||
|
ctlr.put(Walk{ID: "plan-cat", Repository: cat747, Branch: "main", Commit: merge307, Created: at.Add(46 * time.Minute),
|
||||||
|
State: walkBuilding, Tiers: [][]string{{"gitea"}}, Modules: map[string]*WalkModule{"gitea": {}},
|
||||||
|
Delivery: &WalkDelivery{Awaits: "mesh-delivery", Merges: []WalkMerge{
|
||||||
|
{Repository: cat747, Commit: merge301, Carried: merge307}, {Repository: cat747, Commit: merge307}}}})
|
||||||
|
ctlr.mergedAt[merge301], ctlr.mergedAt[merge307] = m301.MergedAt, at.Add(46*time.Minute)
|
||||||
|
forge := newFakeForge()
|
||||||
|
set(forge)
|
||||||
|
w := newWorldOver(t, store, ctlr, forge, at.Add(47*time.Minute))
|
||||||
|
if err := w.h.Reconcile(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
w.settleAll()
|
||||||
|
return w
|
||||||
|
}
|
||||||
|
|
||||||
|
func groupState747(w *world) (string, string) {
|
||||||
|
g := w.h.groups[g747]
|
||||||
|
return GroupState(g, w.h.membersOf(g))
|
||||||
|
}
|
||||||
|
|
||||||
|
// **A failed member is covered by a later finished walk that contains it**: #339's walk, done at 60cc9ec3, which the
|
||||||
|
// forge says contains 5d22d7df, built build-agent and mesh-controller, and #336's own walk said route-proxy unchanged.
|
||||||
|
// #336 stays failed, reads covered by that walk on its pull request, and its group no longer reads failed. #339, whose
|
||||||
|
// own walk was resumed to done, is covered by it too.
|
||||||
|
func TestAFailedMemberIsCoveredByALaterDoneWalkThatContainsIt(t *testing.T) {
|
||||||
|
w := night747(t, true)
|
||||||
|
d := w.delivery(id336)
|
||||||
|
if state, desc := statusOf(d); d.State != Failed || state != "success" || !strings.Contains(desc, "covered after its failure") ||
|
||||||
|
!strings.Contains(desc, "plan-339") {
|
||||||
|
t.Fatalf("#336's status reads %s %q", state, desc)
|
||||||
|
}
|
||||||
|
if s := statusOn(w, ctl747, head336); !strings.HasPrefix(s, "success") {
|
||||||
|
t.Fatalf("the forge was told %q of #336", s)
|
||||||
|
}
|
||||||
|
if state, desc := statusOf(w.delivery(id339)); state != "success" || !strings.Contains(desc, "plan-339") {
|
||||||
|
t.Fatalf("#339 reads %s %q; want it covered by its own walk, resumed to done", state, desc)
|
||||||
|
}
|
||||||
|
if state, why := groupState747(w); state == "failed" || state == "stopped" {
|
||||||
|
t.Fatalf("the group reads %s: %s", state, why)
|
||||||
|
}
|
||||||
|
// Told once, and kept: a restart tells nothing again.
|
||||||
|
notes := len(w.forge.notesOn(merge336))
|
||||||
|
again := newWorldOver(t, w.store, w.ctl, w.forge, w.now.Add(time.Minute))
|
||||||
|
again.settleAll()
|
||||||
|
if state, _ := statusOf(again.delivery(id336)); len(w.forge.notesOn(merge336)) != notes || state != "success" {
|
||||||
|
t.Fatalf("a restart told it again or forgot it: %d notes, was %d", len(w.forge.notesOn(merge336)), notes)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// **The group then no longer stops the other repository's walk**: the catalogue member was stopped for #336 alone,
|
||||||
|
// so the walk of the catalogue's next merge, which carries its merge, is neither stopped nor held back, and the
|
||||||
|
// group waits for that walk to deliver the catalogue member's change.
|
||||||
|
func TestACoveredMemberNoLongerStopsTheOtherRepositorysWalk(t *testing.T) {
|
||||||
|
w := night747(t, true)
|
||||||
|
if len(w.ctl.withouts) > 0 || containsString(w.ctl.stops, "plan-cat") {
|
||||||
|
t.Fatalf("the catalogue's walk was stopped: without %v, stops %v", w.ctl.withouts, w.ctl.stops)
|
||||||
|
}
|
||||||
|
if stopped, _ := w.h.carriesStopped("plan-cat"); len(stopped) > 0 {
|
||||||
|
t.Fatalf("the catalogue's walk is held back for %v", stopped)
|
||||||
|
}
|
||||||
|
if state, why := groupState747(w); state != "delivering" || !strings.Contains(why, id301) {
|
||||||
|
t.Fatalf("the group reads %s: %s; want delivering, the catalogue member waiting for a later walk", state, why)
|
||||||
|
}
|
||||||
|
// Once a later walk of the catalogue that contains its merge built its modules, it is covered, and the group is
|
||||||
|
// delivered.
|
||||||
|
w.forge.contains[merge301+" in "+merge307] = true
|
||||||
|
r := w.ctl.walk("plan-cat")
|
||||||
|
sent := w.now
|
||||||
|
r.Modules = map[string]*WalkModule{}
|
||||||
|
for _, m := range []string{"build-agent", "route-proxy", "gitea"} {
|
||||||
|
r.Modules[m] = &WalkModule{State: "built", SentAt: &sent}
|
||||||
|
}
|
||||||
|
r.State = walkDone
|
||||||
|
w.h.WalkMoved(w.ctl.put(r))
|
||||||
|
w.later(time.Minute)
|
||||||
|
w.settleAll()
|
||||||
|
if state, desc := statusOf(w.delivery(id301)); state != "success" || !strings.Contains(desc, "plan-cat") {
|
||||||
|
t.Fatalf("the catalogue member reads %s %q", state, desc)
|
||||||
|
}
|
||||||
|
if state, why := groupState747(w); state != "delivered" {
|
||||||
|
t.Fatalf("the group reads %s: %s", state, why)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// **A failed member not contained still stops it**: the forge says 60cc9ec3 does not contain 5d22d7df, so #336's change
|
||||||
|
// never reached the machines; it stays failed and uncovered, its group failed, and the catalogue's walk that carries
|
||||||
|
// the stopped member's merge is asked to leave it out.
|
||||||
|
func TestAFailedMemberNotContainedStillStopsTheOtherRepositorysWalk(t *testing.T) {
|
||||||
|
w := night747(t, false)
|
||||||
|
d := w.delivery(id336)
|
||||||
|
if state, desc := statusOf(d); d.State != Failed || state != "failure" {
|
||||||
|
t.Fatalf("#336 is %s and reads %s %q", d.State, state, desc)
|
||||||
|
}
|
||||||
|
if state, _ := groupState747(w); state != "failed" {
|
||||||
|
t.Fatalf("the group reads %s", state)
|
||||||
|
}
|
||||||
|
if len(w.ctl.withouts) == 0 || !strings.HasPrefix(w.ctl.withouts[0], "plan-cat without "+cat747+"@"+merge301) {
|
||||||
|
t.Fatalf("the catalogue's walk was not asked to leave the stopped merge out: %v", w.ctl.withouts)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// **A later walk that only said a module unchanged covers nothing of an earlier merge that moves it**: #339's walk
|
||||||
|
// built mesh-controller alone and said build-agent unchanged for #339's own files, which says nothing of #336's; only
|
||||||
|
// #336's own walk says that of its change (route-proxy). #336 stays failed and its group still stops the catalogue.
|
||||||
|
func TestALaterWalksUnchangedCoversNoModuleOfAnEarlierMerge(t *testing.T) {
|
||||||
|
w := night747With(t, true, []string{"mesh-controller"})
|
||||||
|
if state, desc := statusOf(w.delivery(id336)); state != "failure" {
|
||||||
|
t.Fatalf("#336 reads %s %q, though no walk built build-agent for it", state, desc)
|
||||||
|
}
|
||||||
|
if state, _ := groupState747(w); state != "failed" {
|
||||||
|
t.Fatalf("the group reads %s", state)
|
||||||
|
}
|
||||||
|
if len(w.ctl.withouts) == 0 {
|
||||||
|
t.Fatal("the catalogue's walk was not asked to leave the stopped merge out")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// **A forge that cannot say holds the walk only within the bound** (review of issue 747): while the forge errors on
|
||||||
|
// whether 60cc9ec3 contains 5d22d7df, the catalogue's walk is held back, loudly, neither stopped nor asked to leave the
|
||||||
|
// merge out, and #336 still reads failed. Past the bound the forge's silence holds nothing back any more: the walk
|
||||||
|
// is asked to leave the stopped merge out, as with no later walk at all.
|
||||||
|
func TestAForgeThatCannotSayHoldsTheWalkOnlyWithinTheBound(t *testing.T) {
|
||||||
|
for _, c := range []struct {
|
||||||
|
name string
|
||||||
|
set func(*fakeForge)
|
||||||
|
}{
|
||||||
|
{"erroring", func(f *fakeForge) { f.containsDown = true }},
|
||||||
|
{"not answering for the pair", func(*fakeForge) {}},
|
||||||
|
} {
|
||||||
|
t.Run(c.name, func(t *testing.T) {
|
||||||
|
w := night747Over(t, c.set, nil)
|
||||||
|
if len(w.ctl.withouts) > 0 || len(w.ctl.stops) > 0 {
|
||||||
|
t.Fatalf("within the bound the walk was not held: without %v, stops %v", w.ctl.withouts, w.ctl.stops)
|
||||||
|
}
|
||||||
|
if state, desc := statusOf(w.delivery(id336)); state != "failure" {
|
||||||
|
t.Fatalf("#336 reads %s %q while the forge cannot say", state, desc)
|
||||||
|
}
|
||||||
|
said := ""
|
||||||
|
for _, l := range w.h.WalksWaiting() {
|
||||||
|
if l.Walk == "plan-cat" {
|
||||||
|
said += l.Says
|
||||||
|
}
|
||||||
|
}
|
||||||
|
w.later(3 * time.Minute) // past the grace an unheld walk is given, so `stalled` lists it
|
||||||
|
w.settleAll()
|
||||||
|
for _, l := range w.h.Stalled() {
|
||||||
|
if l.Walk == "plan-cat" {
|
||||||
|
said += " | " + l.Says
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if strings.Count(said, "the forge is asked since") != 2 {
|
||||||
|
t.Fatalf("the hold is not said by `stalled walks` and `stalled`: %q", said)
|
||||||
|
}
|
||||||
|
if len(w.ctl.withouts) > 0 || len(w.ctl.stops) > 0 {
|
||||||
|
t.Fatalf("within the bound the walk was not held: without %v, stops %v", w.ctl.withouts, w.ctl.stops)
|
||||||
|
}
|
||||||
|
w.later(coverAskBound + time.Minute)
|
||||||
|
w.settleAll()
|
||||||
|
if len(w.ctl.withouts) == 0 || !strings.HasPrefix(w.ctl.withouts[0], "plan-cat without "+cat747+"@"+merge301) {
|
||||||
|
t.Fatalf("past the bound the walk was not asked to leave the stopped merge out: %v", w.ctl.withouts)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,292 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// novox/hq issue 753: once issue 747's fix was live (2026-10-14 23:35), the catalogue's walk plan-1792012015539062661 at
|
||||||
|
// 0f76b6e2 (#307's merge; build-agent, gitea, route-proxy to all four machines) was resumed and said "published; its
|
||||||
|
// walk waits for mesh-delivery's word: no delivery that has not ended follows it or is answered by it". Every delivery
|
||||||
|
// it carried had ended: #301 stopped by its group for the controller's #336, delivered since, and waiting for "a later
|
||||||
|
// walk of its repository that contains its merge"; #304 stopped; #307 delivered. The only walk that contained #301's
|
||||||
|
// merge waited for #301's word: a deadlock, released by a hand `plans go`. A delivery whose stop lapsed gives its word
|
||||||
|
// to the walk that answers its merge, and a walk with no living delivery behind it says what it carries.
|
||||||
|
//
|
||||||
|
// The shape is the live state as `mesh-delivery.show` and `mesh-controller.plans` gave it, read back after a restart.
|
||||||
|
|
||||||
|
const (
|
||||||
|
merge304, head304, head307 = "4c0ffee0bbbbbbbb", "a304a304a304a304", "b307b307b307b307"
|
||||||
|
plan753 = "plan-1792012015539062661"
|
||||||
|
)
|
||||||
|
|
||||||
|
var id304, id307 = IDOf(cat747, head304), IDOf(cat747, head307)
|
||||||
|
|
||||||
|
// night753 is the stuck walk: #336 delivered (its change reached the machines), #301 stopped by the given act, #304
|
||||||
|
// stopped by its group, #307 delivered, and the catalogue's walk at #307's merge naming all three merges, waiting.
|
||||||
|
func night753(t *testing.T, stop Transition) *world { return night753With(t, stop) }
|
||||||
|
|
||||||
|
// night753With is night753 with more deliveries kept: one in #301's group is ordered after #336 and before #301.
|
||||||
|
func night753With(t *testing.T, stop Transition, more ...*Delivery) *world {
|
||||||
|
t.Helper()
|
||||||
|
return night753Over(t, stop, nil, nil, more...)
|
||||||
|
}
|
||||||
|
|
||||||
|
// night753Over is night753With with more merges named by the walk and the controller set as given before the first
|
||||||
|
// tick.
|
||||||
|
func night753Over(t *testing.T, stop Transition, merges []WalkMerge, set func(*fakeController), more ...*Delivery) *world {
|
||||||
|
t.Helper()
|
||||||
|
at := at747
|
||||||
|
store := newMemStore()
|
||||||
|
m336 := &Delivery{ID: id336, Repository: ctl747, Commit: head336, Number: 336, Base: "main", Branch: g747, Group: g747,
|
||||||
|
State: Delivered, Since: at.Add(70 * time.Minute), Created: at.Add(-time.Hour), MergedAs: merge336, MergedAt: at,
|
||||||
|
AnsweredBy: merge336, Check: &Verdict{Gate: "pass"}, Plan: planMoving(ctl747, "build-agent", "mesh-controller"),
|
||||||
|
Walk: &WalkSeen{ID: "plan-336", Commit: merge336, State: walkDone, Branch: "main", MovedSaid: true,
|
||||||
|
Moved: []string{"build-agent", "mesh-controller"}, Created: at.Add(7 * time.Minute), Tiers: 2, Tier: 2}}
|
||||||
|
m301 := &Delivery{ID: id301, Repository: cat747, Commit: head301, Number: 301, Base: "main", Branch: g747, Group: g747,
|
||||||
|
State: Stopped, Since: stop.At, Created: at.Add(-2 * time.Hour), MergedAs: merge301, MergedAt: at.Add(7 * time.Second),
|
||||||
|
Check: &Verdict{Gate: "pass"}, Plan: planMoving(cat747, "build-agent", "route-proxy"), Transitions: []Transition{stop}}
|
||||||
|
if stop.By == byOwner {
|
||||||
|
m301.StoppedFor = id336
|
||||||
|
}
|
||||||
|
m304 := &Delivery{ID: id304, Repository: cat747, Commit: head304, Number: 304, Base: "main", Branch: "fix/forge-events",
|
||||||
|
State: Stopped, Since: at.Add(80 * time.Minute), Created: at.Add(-time.Hour), MergedAs: merge304,
|
||||||
|
MergedAt: at.Add(30 * time.Minute), Check: &Verdict{Gate: "pass"},
|
||||||
|
Plan: planMoving(cat747, "build-agent", "gitea", "route-proxy"),
|
||||||
|
// Stopped with its walk, which the walk of #307 took over, and covered since by the walk of #306 (issue 747).
|
||||||
|
DeliveredBy: "plan-1792011830935225963 (novox/mesh-catalog at 0361c44a into main, done, contains its merge 4c0ffee0)",
|
||||||
|
CoverTold: true,
|
||||||
|
Walk: &WalkSeen{ID: "plan-1792009301770286799", Commit: merge304, State: walkSuperseded, Branch: "main",
|
||||||
|
TakenOverBy: plan753, Waits: true, WaitedFor: "mesh-delivery", Created: at.Add(31 * time.Minute)},
|
||||||
|
Transitions: []Transition{{At: at.Add(80 * time.Minute), From: Published, To: Stopped, Event: EvStop,
|
||||||
|
Why: "its walk plan-1792009301770286799 was stopped by mesh-delivery"}}}
|
||||||
|
m307 := &Delivery{ID: id307, Repository: cat747, Commit: head307, Number: 307, Base: "main", Branch: "fix/ts-check",
|
||||||
|
State: Delivered, Since: at.Add(90 * time.Minute), Created: at.Add(-time.Hour), MergedAs: merge307,
|
||||||
|
MergedAt: at.Add(46 * time.Minute), Check: &Verdict{Gate: "pass"}, Plan: planMoving(cat747, "build-agent")}
|
||||||
|
for _, d := range append([]*Delivery{m336, m301, m304, m307}, more...) {
|
||||||
|
if err := store.PutDelivery(d); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
inGroup, order := []*Delivery{m301, m336}, []string{id336}
|
||||||
|
for _, d := range more {
|
||||||
|
if d.Group == g747 {
|
||||||
|
inGroup, order = append(inGroup, d), append(order, d.ID)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
order = append(order, id301)
|
||||||
|
var ids []string
|
||||||
|
for _, d := range inGroup {
|
||||||
|
ids = append(ids, d.ID)
|
||||||
|
}
|
||||||
|
g := &Group{ID: g747, Members: ids, Closed: true, Order: order, Said: "delivering",
|
||||||
|
OrderOf: commitsOf(inGroup), Check: &GroupCheck{Asked: "check-1", At: at.Add(-time.Minute),
|
||||||
|
Verdict: "pass", Summary: "all compose", Commits: commitsOf(inGroup)}}
|
||||||
|
if err := store.PutGroup(g); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
ctlr := newFakeController()
|
||||||
|
modules := map[string]*WalkModule{"build-agent": {}, "gitea": {}, "route-proxy": {}}
|
||||||
|
ctlr.put(Walk{ID: plan753, Repository: cat747, Branch: "main", Commit: merge307, Created: at.Add(46 * time.Minute),
|
||||||
|
State: walkBuilding, Tiers: [][]string{{"build-agent", "gitea", "route-proxy"}}, Modules: modules,
|
||||||
|
Delivery: &WalkDelivery{Awaits: "mesh-delivery", Merges: []WalkMerge{
|
||||||
|
{Repository: cat747, Commit: merge301, Carried: merge307}, {Repository: cat747, Commit: merge304, Carried: merge307},
|
||||||
|
{Repository: cat747, Commit: merge307}}}})
|
||||||
|
if len(merges) > 0 {
|
||||||
|
r := ctlr.walk(plan753)
|
||||||
|
r.Delivery.Merges = append(merges, r.Delivery.Merges...)
|
||||||
|
ctlr.put(r)
|
||||||
|
}
|
||||||
|
if set != nil {
|
||||||
|
set(ctlr)
|
||||||
|
}
|
||||||
|
ctlr.mergedAt[merge301], ctlr.mergedAt[merge304], ctlr.mergedAt[merge307] = m301.MergedAt, m304.MergedAt, m307.MergedAt
|
||||||
|
w := newWorldOver(t, store, ctlr, newFakeForge(), at.Add(100*time.Minute))
|
||||||
|
if err := w.h.Reconcile(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
w.settleAll()
|
||||||
|
return w
|
||||||
|
}
|
||||||
|
|
||||||
|
// groupStop753 is #301's stop as its group made it: for #336, whose change was delivered since.
|
||||||
|
func groupStop753() Transition {
|
||||||
|
return Transition{At: at747.Add(11 * time.Minute), From: Ready, To: Stopped, Event: EvStop, By: byOwner,
|
||||||
|
Why: "its group " + g747 + " stopped: " + id336 + " failed"}
|
||||||
|
}
|
||||||
|
|
||||||
|
func saidOf(lines []StalledLine, walk string) (string, State) {
|
||||||
|
said, state := "", State("")
|
||||||
|
for _, l := range lines {
|
||||||
|
if l.Walk == walk {
|
||||||
|
said, state = said+l.Says, l.State
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return said, state
|
||||||
|
}
|
||||||
|
|
||||||
|
// **A stopped delivery gives its word to the later walk it waits for**: #301, stopped for #336, delivered since, waits for
|
||||||
|
// a later walk of the catalogue that contains its merge; plan-1792012015539062661 names that merge and waits for this
|
||||||
|
// owner's word. #301 gives it, once, saying why, and `stalled walks` never calls the walk unheld.
|
||||||
|
func TestAStoppedDeliveryGivesItsWordToTheWalkItWaitsFor(t *testing.T) {
|
||||||
|
w := night753(t, groupStop753())
|
||||||
|
if n := strings.Count(strings.Join(w.ctl.delivers, " "), plan753); n != 1 {
|
||||||
|
t.Fatalf("the walk was let go %d times (%v); want once", n, w.ctl.delivers)
|
||||||
|
}
|
||||||
|
d := w.delivery(id301)
|
||||||
|
if d.State != Stopped || !strings.Contains(d.LetGoWhy, lapsedWord(plan753)) || !strings.Contains(d.LetGoWhy, "issue 753") {
|
||||||
|
t.Fatalf("#301 is %s, its word given for %q", d.State, d.LetGoWhy)
|
||||||
|
}
|
||||||
|
if said, state := saidOf(w.h.WalksWaiting(), plan753); state == Unheld {
|
||||||
|
t.Fatalf("the walk reads unheld: %q", said)
|
||||||
|
}
|
||||||
|
if len(w.ctl.withouts) > 0 || len(w.ctl.stops) > 0 {
|
||||||
|
t.Fatalf("the walk was stopped: without %v, stops %v", w.ctl.withouts, w.ctl.stops)
|
||||||
|
}
|
||||||
|
// Once that walk built build-agent and route-proxy and ended done, #301 is covered by it (issue 747).
|
||||||
|
w.forge.contains[merge301+" in "+merge307] = true
|
||||||
|
r := w.ctl.walk(plan753)
|
||||||
|
sent := w.now
|
||||||
|
for _, m := range []string{"build-agent", "gitea", "route-proxy"} {
|
||||||
|
r.Modules[m] = &WalkModule{State: "built", SentAt: &sent}
|
||||||
|
}
|
||||||
|
r.State = walkDone
|
||||||
|
w.h.WalkMoved(w.ctl.put(r))
|
||||||
|
w.later(time.Minute)
|
||||||
|
w.settleAll()
|
||||||
|
if state, desc := statusOf(w.delivery(id301)); state != "success" || !strings.Contains(desc, plan753) {
|
||||||
|
t.Fatalf("#301 reads %s %q", state, desc)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// **A walk still holding a live delivery not at its turn waits for it**: the stopped delivery's word is one turn among
|
||||||
|
// the walk's, never a way past a live delivery it carries.
|
||||||
|
func TestALapsedStopDoesNotLetGoPastALiveDelivery(t *testing.T) {
|
||||||
|
at := at747
|
||||||
|
held := &Delivery{ID: IDOf(cat747, "c308c308c308c308"), Repository: cat747, Commit: "c308c308c308c308", Number: 308,
|
||||||
|
Base: "main", Branch: "fix/held", State: Held, HeldWhy: "a person's", Since: at, Created: at.Add(-time.Hour),
|
||||||
|
MergedAs: "c308aaaaaaaaaaaa", MergedAt: at.Add(40 * time.Minute), Check: &Verdict{Gate: "pass"},
|
||||||
|
Plan: planMoving(cat747, "gitea"), Walk: &WalkSeen{ID: plan753, Commit: merge307, State: walkBuilding, Branch: "main",
|
||||||
|
Waits: true, Created: at.Add(46 * time.Minute)}}
|
||||||
|
w := night753With(t, groupStop753(), held)
|
||||||
|
if containsString(w.ctl.delivers, plan753) {
|
||||||
|
t.Fatalf("the walk was let go past a held delivery it carries: %v", w.ctl.delivers)
|
||||||
|
}
|
||||||
|
if said, state := saidOf(w.h.WalksWaiting(), plan753); state != Held {
|
||||||
|
t.Fatalf("`stalled walks` reads %s %q; want the held delivery", state, said)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// **A stop by a person gives no word**: #301 stopped by hand holds its change back; the walk is asked to leave it out,
|
||||||
|
// never let go for it.
|
||||||
|
func TestAPersonsStopGivesNoWord(t *testing.T) {
|
||||||
|
w := night753(t, Transition{At: at747.Add(11 * time.Minute), From: Ready, To: Stopped, Event: EvStop, By: "jochen",
|
||||||
|
Why: "stopped by hand"})
|
||||||
|
if containsString(w.ctl.delivers, plan753) {
|
||||||
|
t.Fatalf("the walk was let go for a person's stop: %v", w.ctl.delivers)
|
||||||
|
}
|
||||||
|
if len(w.ctl.withouts) == 0 && len(w.ctl.stops) == 0 {
|
||||||
|
t.Fatal("the walk carrying a person's stop was neither left without it nor stopped")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// **A walk no living delivery holds says what it carries** (the issue's third expectation): with nothing to give it
|
||||||
|
// its word, `stalled walks` and, past the grace, `stalled` name every ended delivery it carries and its state, never
|
||||||
|
// only that nothing holds it.
|
||||||
|
func TestAWalkWithNoLivingDeliverySaysWhatItCarries(t *testing.T) {
|
||||||
|
w := night753(t, groupStop753())
|
||||||
|
// The walk read again as waiting, naming only #307's merge, which ended delivered: no delivery whose stop lapsed
|
||||||
|
// gives it its word, and no live one holds it.
|
||||||
|
r := w.ctl.walk(plan753)
|
||||||
|
r.Delivery.Go = nil
|
||||||
|
r.Delivery.Merges = []WalkMerge{{Repository: cat747, Commit: merge307}}
|
||||||
|
w.h.WalkMoved(w.ctl.put(r))
|
||||||
|
said, state := saidOf(w.h.WalksWaiting(), plan753)
|
||||||
|
if state != Unheld || !strings.Contains(said, id307+" (delivered)") {
|
||||||
|
t.Fatalf("`stalled walks` reads %s %q; want unheld, naming %s (delivered)", state, said, id307)
|
||||||
|
}
|
||||||
|
w.settleAll()
|
||||||
|
w.later(matchGrace + time.Minute)
|
||||||
|
w.settleAll()
|
||||||
|
said, _ = saidOf(w.h.Stalled(), plan753)
|
||||||
|
if !strings.Contains(said, "every delivery it carries ended: ") || !strings.Contains(said, id307+" (delivered)") {
|
||||||
|
t.Fatalf("`stalled` reads %q", said)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// m299 is a member of #301's group ordered before it, stopped for #336 as #301 was, its merge on another walk.
|
||||||
|
func m299() *Delivery {
|
||||||
|
at := at747
|
||||||
|
return &Delivery{ID: IDOf(cat747, "e299e299e299e299"), Repository: cat747, Commit: "e299e299e299e299", Number: 299,
|
||||||
|
Base: "main", Branch: g747, Group: g747, State: Stopped, StoppedFor: id336, Since: at.Add(11 * time.Minute),
|
||||||
|
Created: at.Add(-3 * time.Hour), MergedAs: "e299aaaaaaaaaaaa", MergedAt: at.Add(5 * time.Second),
|
||||||
|
Check: &Verdict{Gate: "pass"}, Plan: planMoving(cat747, "gitea"),
|
||||||
|
Transitions: []Transition{{At: at.Add(11 * time.Minute), From: Ready, To: Stopped, Event: EvStop, By: byOwner,
|
||||||
|
Why: "its group " + g747 + " stopped: " + id336 + " failed"}}}
|
||||||
|
}
|
||||||
|
|
||||||
|
// **A stopped member gives its word only in its group's order** (review of issue 753): #299, ordered before #301 and
|
||||||
|
// stopped for #336 too, is not delivered and this walk does not carry its merge, so #301 gives no word — it would land
|
||||||
|
// before #299 — and `stalled walks` names #299. When the walk names #299's merge too, it delivers both, in order, and
|
||||||
|
// is let go.
|
||||||
|
func TestAStoppedMemberGivesItsWordOnlyInItsGroupsOrder(t *testing.T) {
|
||||||
|
w := night753With(t, groupStop753(), m299())
|
||||||
|
if containsString(w.ctl.delivers, plan753) {
|
||||||
|
t.Fatalf("the walk was let go for #301 before #299, ordered before it: %v", w.ctl.delivers)
|
||||||
|
}
|
||||||
|
said, state := saidOf(w.h.WalksWaiting(), plan753)
|
||||||
|
if state == Unheld || !strings.Contains(said, m299().ID+" (stopped)") {
|
||||||
|
t.Fatalf("`stalled walks` reads %s %q; want #299 named", state, said)
|
||||||
|
}
|
||||||
|
w = night753Over(t, groupStop753(), []WalkMerge{{Repository: cat747, Commit: "e299aaaaaaaaaaaa", Carried: merge307}}, nil,
|
||||||
|
m299())
|
||||||
|
if !containsString(w.ctl.delivers, plan753) {
|
||||||
|
t.Fatalf("the walk carrying both members was not let go: %v", w.ctl.delivers)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// **A word the controller refuses is never silent** (review of issue 753): the controller keeps refusing it, and past
|
||||||
|
// the grace `stalled` lists the walk, with the refusal.
|
||||||
|
func TestARefusedWordIsListedByStalled(t *testing.T) {
|
||||||
|
w := night753Over(t, groupStop753(), nil, func(c *fakeController) { c.refuseDeliver = true })
|
||||||
|
if containsString(w.ctl.delivers, plan753) {
|
||||||
|
t.Fatalf("the walk was let go: %v", w.ctl.delivers)
|
||||||
|
}
|
||||||
|
w.later(matchGrace + time.Minute)
|
||||||
|
w.settleAll()
|
||||||
|
said, state := saidOf(w.h.Stalled(), plan753)
|
||||||
|
if state != Published || !strings.Contains(said, "the controller refused its word: the controller refuses the word") {
|
||||||
|
t.Fatalf("`stalled` reads %s %q", state, said)
|
||||||
|
}
|
||||||
|
// Once the controller takes it, the walk is let go.
|
||||||
|
w.ctl.refuseDeliver = false
|
||||||
|
w.settleAll()
|
||||||
|
if !containsString(w.ctl.delivers, plan753) {
|
||||||
|
t.Fatalf("the walk was not let go once the controller took the word: %v", w.ctl.delivers)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// **A walk held back for another stopped change is never silent** (review of issue 753): #304 was stopped by a person,
|
||||||
|
// and the controller refuses to leave its merge out or to stop the walk, so the word #301 owes is never sent; past the
|
||||||
|
// grace `stalled` lists the walk, saying why.
|
||||||
|
func TestAWalkHeldForAPersonsStopIsListedByStalled(t *testing.T) {
|
||||||
|
at := at747
|
||||||
|
m305 := &Delivery{ID: IDOf(cat747, "f305f305f305f305"), Repository: cat747, Commit: "f305f305f305f305", Number: 305,
|
||||||
|
Base: "main", Branch: "fix/other", State: Stopped, Since: at.Add(80 * time.Minute), Created: at.Add(-time.Hour),
|
||||||
|
MergedAs: "f305aaaaaaaaaaaa", MergedAt: at.Add(35 * time.Minute), Check: &Verdict{Gate: "pass"},
|
||||||
|
Plan: planMoving(cat747, "gitea"), Transitions: []Transition{{At: at.Add(80 * time.Minute), From: Ready,
|
||||||
|
To: Stopped, Event: EvStop, By: "jochen", Why: "stopped by hand"}}}
|
||||||
|
w := night753Over(t, groupStop753(), []WalkMerge{{Repository: cat747, Commit: m305.MergedAs, Carried: merge307}},
|
||||||
|
func(c *fakeController) { c.refuseWithout, c.refuseStop = true, true }, m305)
|
||||||
|
if containsString(w.ctl.delivers, plan753) {
|
||||||
|
t.Fatalf("the walk carrying a person's stop was let go: %v", w.ctl.delivers)
|
||||||
|
}
|
||||||
|
w.later(matchGrace + time.Minute)
|
||||||
|
w.settleAll()
|
||||||
|
said, _ := saidOf(w.h.Stalled(), plan753)
|
||||||
|
if !strings.Contains(said, "owe it (novox/hq issue 753)") || !strings.Contains(said, "f305aaaaaaaaaaaa") ||
|
||||||
|
!strings.Contains(said, "whose delivery was stopped") {
|
||||||
|
t.Fatalf("`stalled` reads %q", said)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,151 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// novox/hq issue 783: mesh-catalog commit b5417c28 was pushed while #305 (from branch adr-359-mesh-mail-733) was
|
||||||
|
// open, so this owner recorded its delivery as #305's. #305 then closed unmerged (its base branch had been
|
||||||
|
// deleted), and #315, opened from the same branch and head, merged it as 0914a33f. The delivery stayed stopped
|
||||||
|
// ("the pull request closed unmerged"), though it recorded merged_as 0914a33f: a rule did not hold — a delivery
|
||||||
|
// keyed by commit follows the pull request that merges that commit, and a closed duplicate does not end it — and
|
||||||
|
// the walk that carried the merge waited 64 minutes for this owner's word until a person's `plans go` released it.
|
||||||
|
//
|
||||||
|
// A delivery stopped only because its pull request closed unmerged, with no merge recorded, does not end its
|
||||||
|
// commit's delivery for another pull request with the same head: it is started over, under the same id, for the
|
||||||
|
// pull request that announces or merges it next.
|
||||||
|
|
||||||
|
const (
|
||||||
|
head783, merge783 = "d305d305d305d305", "d315d315d315d315"
|
||||||
|
head783b, merge783b = "d306d306d306d306", "d316d316d316d316"
|
||||||
|
head783c = "d307d307d307d307"
|
||||||
|
head783d, merge783d = "d308d308d308d308", "d318d318d318d318"
|
||||||
|
catalog783 = "novox/mesh-catalog"
|
||||||
|
branch783 = "adr-359-mesh-mail-733"
|
||||||
|
)
|
||||||
|
|
||||||
|
// TestADeliveryStoppedByAClosedDuplicateFollowsThePullRequestThatMergesItsCommit is the issue's own shape: #305
|
||||||
|
// announced, closed unmerged (stopped); #315 announced from the same head, checked and readied; #315 merges it,
|
||||||
|
// and the walk that answers that merge is let go — the delivery follows #315, never stuck on #305's stop.
|
||||||
|
func TestADeliveryStoppedByAClosedDuplicateFollowsThePullRequestThatMergesItsCommit(t *testing.T) {
|
||||||
|
w := newWorld(t)
|
||||||
|
id := IDOf(catalog783, head783)
|
||||||
|
|
||||||
|
w.h.PullUpdated(pr(catalog783, 305, head783, branch783, "src/a.go"))
|
||||||
|
w.h.PullClosed(pr(catalog783, 305, head783, branch783))
|
||||||
|
if w.state(id) != Stopped {
|
||||||
|
t.Fatalf("#305 closed unmerged left it %s; want stopped", w.state(id))
|
||||||
|
}
|
||||||
|
|
||||||
|
w.h.PullUpdated(pr(catalog783, 315, head783, branch783, "src/a.go"))
|
||||||
|
if d := w.delivery(id); d.State == Stopped || d.Number != 315 {
|
||||||
|
t.Fatalf("#315, the same head, left it %s, Number %d; want started over for #315, not stopped", d.State, d.Number)
|
||||||
|
}
|
||||||
|
if d := w.delivery(id); !strings.Contains(d.Transitions[len(d.Transitions)-1].Why, "#305 with the same head closed unmerged") {
|
||||||
|
t.Fatalf("the start-over's transition names what it replaces: %q", d.Transitions[len(d.Transitions)-1].Why)
|
||||||
|
}
|
||||||
|
|
||||||
|
w.h.Checked(verdict(catalog783, 315, head783, "pass"))
|
||||||
|
if d := w.delivery(id); d.State != Ready {
|
||||||
|
t.Fatalf("a passing verdict for #315 left it %s", d.State)
|
||||||
|
}
|
||||||
|
|
||||||
|
walk := w.ctl.put(aWalk("plan-783", catalog783, merge783, true))
|
||||||
|
w.h.WalkMoved(walk)
|
||||||
|
w.h.PullMerged(merged(catalog783, 315, head783, merge783))
|
||||||
|
if d := w.delivery(id); d.State != Published || d.Number != 315 {
|
||||||
|
t.Fatalf("#315's merge left it %s, Number %d; want published under #315", d.State, d.Number)
|
||||||
|
}
|
||||||
|
|
||||||
|
w.settleAll()
|
||||||
|
if len(w.ctl.delivers) != 1 || w.ctl.delivers[0] != "plan-783" {
|
||||||
|
t.Fatalf("the walk that answers #315's merge was not let go: %v", w.ctl.delivers)
|
||||||
|
}
|
||||||
|
if d := w.delivery(id); d.State != Delivering {
|
||||||
|
t.Fatalf("let go, it is %s; want delivering", d.State)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestAMergeWithoutAnAnnouncementStartsOverAStoppedDuplicateToo is decision 4: the merging pull request is never
|
||||||
|
// announced on its own (heard only as `pull.merged`) — the start-over happens in PullMerged itself, not only in
|
||||||
|
// PullUpdated, so a merge heard without the open announcement is not lost either.
|
||||||
|
func TestAMergeWithoutAnAnnouncementStartsOverAStoppedDuplicateToo(t *testing.T) {
|
||||||
|
w := newWorld(t)
|
||||||
|
id := IDOf(catalog783, head783b)
|
||||||
|
|
||||||
|
w.h.PullUpdated(pr(catalog783, 306, head783b, branch783+"-b", "src/b.go"))
|
||||||
|
w.h.PullClosed(pr(catalog783, 306, head783b, branch783+"-b"))
|
||||||
|
if w.state(id) != Stopped {
|
||||||
|
t.Fatalf("#306 closed unmerged left it %s; want stopped", w.state(id))
|
||||||
|
}
|
||||||
|
|
||||||
|
walk := w.ctl.put(aWalk("plan-783b", catalog783, merge783b, true))
|
||||||
|
w.h.WalkMoved(walk)
|
||||||
|
m := merged(catalog783, 316, head783b, merge783b)
|
||||||
|
m.HeadChecks = map[string]string{"mesh/merge-gate": "success", "mesh/repo-check": "success"}
|
||||||
|
w.h.PullMerged(m)
|
||||||
|
if d := w.delivery(id); d.State == Stopped || d.Number != 316 {
|
||||||
|
t.Fatalf("#316's merge, heard with no announcement of its own, left it %s, Number %d; want started over for #316",
|
||||||
|
d.State, d.Number)
|
||||||
|
}
|
||||||
|
|
||||||
|
w.settleAll()
|
||||||
|
if len(w.ctl.delivers) != 1 || w.ctl.delivers[0] != "plan-783b" {
|
||||||
|
t.Fatalf("the walk that answers #316's merge was not let go: %v", w.ctl.delivers)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestAStoppedDeliveryAnnouncedAgainByItsOwnPullRequestStaysStopped guards decision 3's "any other stopped
|
||||||
|
// delivery ... is untouched": the same pull request announced again after its own close is the ordinary
|
||||||
|
// 'announced again' path, never a start-over, and stays stopped.
|
||||||
|
func TestAStoppedDeliveryAnnouncedAgainByItsOwnPullRequestStaysStopped(t *testing.T) {
|
||||||
|
w := newWorld(t)
|
||||||
|
id := IDOf(catalog783, head783c)
|
||||||
|
|
||||||
|
w.h.PullUpdated(pr(catalog783, 307, head783c, branch783+"-c", "src/c.go"))
|
||||||
|
w.h.PullClosed(pr(catalog783, 307, head783c, branch783+"-c"))
|
||||||
|
if w.state(id) != Stopped {
|
||||||
|
t.Fatalf("#307 closed unmerged left it %s; want stopped", w.state(id))
|
||||||
|
}
|
||||||
|
|
||||||
|
w.h.PullUpdated(pr(catalog783, 307, head783c, branch783+"-c", "src/c.go"))
|
||||||
|
if d := w.delivery(id); d.State != Stopped || d.Number != 307 {
|
||||||
|
t.Fatalf("#307 announced again by itself left it %s, Number %d; want it to stay stopped", d.State, d.Number)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestAMergeHeardLongAfterStartingOverAStoppedDuplicateIsHistoryLikeAnyOtherReplayedMerge is review round 1's
|
||||||
|
// decision: a delivery started over from a merge heard without its open announcement (as in decision 4, above)
|
||||||
|
// takes FromForge exactly as a first-heard merge does, so one replayed from the bus's history long after it
|
||||||
|
// happened still falls under the history rule (novox/hq issue 313) and is never held for a person.
|
||||||
|
func TestAMergeHeardLongAfterStartingOverAStoppedDuplicateIsHistoryLikeAnyOtherReplayedMerge(t *testing.T) {
|
||||||
|
w := newWorld(t)
|
||||||
|
id := IDOf(catalog783, head783d)
|
||||||
|
|
||||||
|
w.h.PullUpdated(pr(catalog783, 308, head783d, branch783+"-d", "src/d.go"))
|
||||||
|
w.h.PullClosed(pr(catalog783, 308, head783d, branch783+"-d"))
|
||||||
|
if w.state(id) != Stopped {
|
||||||
|
t.Fatalf("#308 closed unmerged left it %s; want stopped", w.state(id))
|
||||||
|
}
|
||||||
|
|
||||||
|
m := merged(catalog783, 318, head783d, merge783d)
|
||||||
|
m.MergedAt = w.now.Add(-historyLate - time.Minute).UTC().Format(time.RFC3339)
|
||||||
|
w.h.PullMerged(m)
|
||||||
|
|
||||||
|
d := w.delivery(id)
|
||||||
|
if d.State == Stopped || d.Number != 318 {
|
||||||
|
t.Fatalf("#318's merge, heard long after it happened with no announcement of its own, left it %s, Number %d; "+
|
||||||
|
"want started over for #318", d.State, d.Number)
|
||||||
|
}
|
||||||
|
if !d.FromForge {
|
||||||
|
t.Fatal("a merge heard without its own announcement is made from the forge's word: FromForge is false")
|
||||||
|
}
|
||||||
|
if d.History == "" {
|
||||||
|
t.Fatalf("a merge heard %s after it happened is history, never held for a person; History is empty", historyLate+time.Minute)
|
||||||
|
}
|
||||||
|
if d.State == Held {
|
||||||
|
t.Fatalf("history is never held for a person; state is %s, HeldWhy %q", d.State, d.HeldWhy)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -83,6 +83,14 @@ type Delivery struct {
|
|||||||
// CoverTold is whether the forge was told that such a stop was covered since: its status, its view and its note
|
// CoverTold is whether the forge was told that such a stop was covered since: its status, its view and its note
|
||||||
// (novox/hq issue 480). Kept, so a restart tells it once, and a stop covered before this was built is told too.
|
// (novox/hq issue 480). Kept, so a restart tells it once, and a stop covered before this was built is told too.
|
||||||
CoverTold bool `json:"cover_told,omitempty"`
|
CoverTold bool `json:"cover_told,omitempty"`
|
||||||
|
// DeliveredBy says, for a delivery that failed or was stopped, the later finished walk of its repository whose
|
||||||
|
// commit contains its merge and which built every module it moves — each it did not build, its own walk said
|
||||||
|
// unchanged: its change reached the machines after all (novox/hq issue 747). Its state stays final, as a covered
|
||||||
|
// stop's does (issue 480), and it reads covered: its group counts it as delivered.
|
||||||
|
DeliveredBy string `json:"delivered_by,omitempty"`
|
||||||
|
// StoppedFor is, for a member its group stopped, the member whose end stopped it (novox/hq issue 747): once that
|
||||||
|
// member's change reached the machines, the stop has lapsed, and holds no walk of its repository back.
|
||||||
|
StoppedFor string `json:"stopped_for,omitempty"`
|
||||||
// TurnAwaits says, for a group member whose turn waits for a later merge to carry a superseded member before it,
|
// TurnAwaits says, for a group member whose turn waits for a later merge to carry a superseded member before it,
|
||||||
// which merges and until when; TurnAwaitsUntil is that moment, which bounds the wait (ADR 0276 decision 9,
|
// which merges and until when; TurnAwaitsUntil is that moment, which bounds the wait (ADR 0276 decision 9,
|
||||||
// novox/hq issue 445). `show` shows both, and `stalled` says them past that moment.
|
// novox/hq issue 445). `show` shows both, and `stalled` says them past that moment.
|
||||||
@@ -303,18 +311,77 @@ func (d *Delivery) FromForgesWord() bool {
|
|||||||
|
|
||||||
// shownOnItsPullRequest is whether its view and status are kept on its pull request: not for history, whose
|
// shownOnItsPullRequest is whether its view and status are kept on its pull request: not for history, whose
|
||||||
// pull request the forge may no longer hold — or holds under its number another one.
|
// pull request the forge may no longer hold — or holds under its number another one.
|
||||||
// coveredStop is whether a delivery stopped because its walk ended done before its word without building its modules
|
// coveredStop is whether a delivery that ended stopped or failed was covered since, and reads covered. A stop
|
||||||
// (novox/hq issue 474) was covered since: a later merge into its trunk that contains it was delivered, carrying every
|
// because its walk ended done before its word without building its modules (novox/hq issue 474) is covered when a
|
||||||
// module it moves to every machine running it (ADR 0270's cover test, kept as CoveredBy). Its state stays stopped —
|
// later merge into its trunk that contains it was delivered, carrying every module it moves to every machine running
|
||||||
// final, as a covered superseded member stays superseded (issue 442) — and it reads covered, by that merge (novox/hq
|
// it (ADR 0270's cover test, kept as CoveredBy; issue 480). Any stop or failure is covered when a later finished walk
|
||||||
// issue 480). A stop for any other reason never reads covered.
|
// of its repository whose commit contains its merge built its modules (DeliveredBy, novox/hq issue 747). Its state
|
||||||
|
// stays final, as a covered superseded member stays superseded (issue 442), and it reads covered, by that merge or
|
||||||
|
// walk. A person's stop with a cover found only while it was held keeps its error.
|
||||||
func (d *Delivery) coveredStop() bool {
|
func (d *Delivery) coveredStop() bool {
|
||||||
return d.State == Stopped && d.Stranded != "" && d.CoveredBy != ""
|
switch d.State {
|
||||||
|
case Stopped:
|
||||||
|
return d.Stranded != "" && d.CoveredBy != "" || d.DeliveredBy != ""
|
||||||
|
case Failed:
|
||||||
|
return d.DeliveredBy != ""
|
||||||
|
}
|
||||||
|
return false
|
||||||
}
|
}
|
||||||
|
|
||||||
// coveredStopSaid is how a covered stop reads: on the forge's status, in its view and note, and in `show`.
|
// coveredStopSaid is how a covered stop or failure reads: on the forge's status, in its view and note, and in `show`.
|
||||||
func (d *Delivery) coveredStopSaid() string {
|
func (d *Delivery) coveredStopSaid() string {
|
||||||
return "covered after its stop: a later merge that contains it was delivered: " + d.CoveredBy
|
ended := "its stop"
|
||||||
|
if d.State == Failed {
|
||||||
|
ended = "its failure"
|
||||||
|
}
|
||||||
|
if d.Stranded != "" && d.CoveredBy != "" {
|
||||||
|
return "covered after " + ended + ": a later merge that contains it was delivered: " + d.CoveredBy
|
||||||
|
}
|
||||||
|
return "covered after " + ended + ": a later walk that contains it was delivered: " + d.DeliveredBy
|
||||||
|
}
|
||||||
|
|
||||||
|
// memberReached is whether a group counts a member as delivered: its change reached the machines, or had nothing to
|
||||||
|
// reach them with — delivered, history or covered, a superseded member a later delivered walk carried (issue 442),
|
||||||
|
// or a stop or failure covered since (issues 480 and 747).
|
||||||
|
func memberReached(d *Delivery) bool {
|
||||||
|
return d.State.Reached() || d.State == Superseded && d.CoveredBy != "" || d.coveredStop()
|
||||||
|
}
|
||||||
|
|
||||||
|
// stoppedFor is the member whose end stopped this one in its group: kept since novox/hq issue 747, and read from the
|
||||||
|
// stop's reason for a stop made before ("its group <id> stopped: <member> …").
|
||||||
|
func (d *Delivery) stoppedFor() string {
|
||||||
|
if d.StoppedFor != "" || d.State != Stopped || d.Group == "" {
|
||||||
|
return d.StoppedFor
|
||||||
|
}
|
||||||
|
prefix := "its group " + d.Group + " stopped: "
|
||||||
|
for i := len(d.Transitions) - 1; i >= 0; i-- {
|
||||||
|
t := d.Transitions[i]
|
||||||
|
if t.Event != EvStop {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if t.By != byOwner || !strings.HasPrefix(t.Why, prefix) {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
member, _, _ := strings.Cut(strings.TrimPrefix(t.Why, prefix), " ")
|
||||||
|
return strings.TrimRight(member, ",;:")
|
||||||
|
}
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
|
// stopLapsed is whether a member its group stopped for another member is stopped for nothing any more: that member's
|
||||||
|
// change reached the machines since — delivered, or its failure covered by a later walk (novox/hq issue 747). Its own
|
||||||
|
// change is still to deliver, by a later walk of its repository that contains it; its stop holds none back. find
|
||||||
|
// gives a delivery by its id.
|
||||||
|
func stopLapsed(d *Delivery, find func(string) *Delivery) bool {
|
||||||
|
if d.State != Stopped || d.coveredStop() {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
by := d.stoppedFor()
|
||||||
|
if by == "" || by == d.ID {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
o := find(by)
|
||||||
|
return o != nil && memberReached(o)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (d *Delivery) shownOnItsPullRequest() bool {
|
func (d *Delivery) shownOnItsPullRequest() bool {
|
||||||
|
|||||||
@@ -72,6 +72,10 @@ func (h *Holder) waitsFor(d *Delivery) string {
|
|||||||
return "its walk: " + clip(d.Walk.Note, 120)
|
return "its walk: " + clip(d.Walk.Note, 120)
|
||||||
}
|
}
|
||||||
case Stopped:
|
case Stopped:
|
||||||
|
if h.stopLapsed(d) {
|
||||||
|
return "a later walk of its repository that contains its merge: its group stopped it for " + d.stoppedFor() +
|
||||||
|
", whose change was delivered since (novox/hq issue 747)"
|
||||||
|
}
|
||||||
if strandedUncovered(d) {
|
if strandedUncovered(d) {
|
||||||
return "a later delivered merge to carry " + strings.Join(d.notMovedBy(), ", ") + ": it is stopped, and " +
|
return "a later delivered merge to carry " + strings.Join(d.notMovedBy(), ", ") + ": it is stopped, and " +
|
||||||
"listed by stalled until one does (novox/hq issue 480)"
|
"listed by stalled until one does (novox/hq issue 480)"
|
||||||
|
|||||||
@@ -21,7 +21,7 @@ operator, it is accepted only on the operator's Approve, given on a channel that
|
|||||||
| `links` | `number`, `kind` | the record's typed links, out and in |
|
| `links` | `number`, `kind` | the record's typed links, out and in |
|
||||||
| `history` | `number`, `kind`, `from`, `to`, `bound` | every version; with `from` and `to`, the difference between the two, line by line |
|
| `history` | `number`, `kind`, `from`, `to`, `bound` | every version; with `from` and `to`, the difference between the two, line by line |
|
||||||
| `reserve` | `kind`, `for` | the next decision or research number, asked of the issue tracker (`issue-tracker.reserve`); spent whether or not its record is accepted |
|
| `reserve` | `kind`, `for` | the next decision or research number, asked of the issue tracker (`issue-tracker.reserve`); spent whether or not its record is accepted |
|
||||||
| `propose` | the record whole | proposed, or refused naming every rule it breaks; an accepted record changes only by a progressive insight (`insight`). Each version proposed is then reviewed by the knowledge base itself (below) |
|
| `propose` | the record whole | proposed, or refused naming every rule it breaks; an accepted record changes only by a progressive insight (`insight`), but an active research effort takes new versions freely, each reviewed as a proposal. Each version proposed is then reviewed by the knowledge base itself (below) |
|
||||||
| `import` | `dry-run`, `commit` | hq's decision records and research efforts at the commit the records module stands at, each whole at its own number |
|
| `import` | `dry-run`, `commit` | hq's decision records and research efforts at the commit the records module stands at, each whole at its own number |
|
||||||
| `return` | `number`, `reason` | drafted clauses sent back to be drafted again; refused for a record whose clauses another record changed or links by number |
|
| `return` | `number`, `reason` | drafted clauses sent back to be drafted again; refused for a record whose clauses another record changed or links by number |
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,150 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// effortAsk is a research effort proposal with the given documents.
|
||||||
|
func effortAsk(n int, status string, docs ...Section) ProposeAsk {
|
||||||
|
return ProposeAsk{Kind: KindResearch, Number: n, Title: "A vendor-neutral inference layer", Date: "2026-10-11",
|
||||||
|
Deciders: []string{"jochen"}, Status: status, Sections: docs,
|
||||||
|
Links: []LinkAsk{{Kind: LinkTouches, To: "0290"}}}
|
||||||
|
}
|
||||||
|
|
||||||
|
// novox/hq issue 844: an active research effort takes new versions freely, each reviewed as a proposal.
|
||||||
|
func TestAnActiveEffortTakesANewVersionWithANewDocument(t *testing.T) {
|
||||||
|
w := imported(t, newMemStore())
|
||||||
|
r, err := w.k.ReserveNumber("author@g14", KindResearch, "an effort")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
overview := Section{Name: "00-overview.md", Text: "What an inference layer would be."}
|
||||||
|
w.accepted(t, "author@g14", effortAsk(r.Number, StatusActive, overview))
|
||||||
|
key := Key{KindResearch, r.Number}
|
||||||
|
reviews := len(w.judge.reviews)
|
||||||
|
next := effortAsk(r.Number, StatusActive, overview, Section{Name: "01-findings.md", Text: "A new finding."})
|
||||||
|
next.Why = "findings added"
|
||||||
|
if _, err := w.k.Propose("author@g14", next); err != nil {
|
||||||
|
t.Fatalf("an active effort refused a new version: %v", err)
|
||||||
|
}
|
||||||
|
e := mustGet(t, w.k.Store, key)
|
||||||
|
if e.Pending == nil || e.Current.Section("01-findings.md") != "" {
|
||||||
|
t.Fatalf("the new version was served before its review")
|
||||||
|
}
|
||||||
|
w.reviewAll(t)
|
||||||
|
q := w.judge.reviews[reviews]
|
||||||
|
if strings.Contains(q, "a correction of fact (a progressive insight)") {
|
||||||
|
t.Fatalf("the review judged a new version as an insight:\n%.400s", q)
|
||||||
|
}
|
||||||
|
e = mustGet(t, w.k.Store, key)
|
||||||
|
if e.Pending != nil || e.Current.Status != StatusActive || e.Current.Section("01-findings.md") != "A new finding." ||
|
||||||
|
len(e.Current.Insights) != 0 {
|
||||||
|
t.Fatalf("after the review: %s, pending %v, insights %v", e.Current.Status, e.Pending != nil, e.Current.Insights)
|
||||||
|
}
|
||||||
|
h, err := w.k.History(key, 0, 0, 0)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for _, v := range h.Versions {
|
||||||
|
if v.Insight {
|
||||||
|
t.Errorf("version %d is marked an insight", v.N)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAGraduatedEffortAndADecisionStillTakeOnlyAnInsight(t *testing.T) {
|
||||||
|
w := imported(t, newMemStore())
|
||||||
|
r, err := w.k.ReserveNumber("author@g14", KindResearch, "an effort")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
overview := Section{Name: "00-overview.md", Text: "What an inference layer would be."}
|
||||||
|
grad := effortAsk(r.Number, StatusGraduated, overview)
|
||||||
|
grad.Links = append(grad.Links, LinkAsk{Kind: LinkBecame, To: "0290"})
|
||||||
|
w.accepted(t, "author@g14", grad)
|
||||||
|
next := effortAsk(r.Number, StatusGraduated, overview, Section{Name: "01-findings.md", Text: "More."})
|
||||||
|
next.Links = grad.Links
|
||||||
|
if _, err := w.k.Propose("author@g14", next); err == nil || !strings.Contains(err.Error(), "progressive insight") {
|
||||||
|
t.Fatalf("a graduated effort took a plain version: %v", err)
|
||||||
|
}
|
||||||
|
n := w.reserved(t, "author@g14")
|
||||||
|
w.accepted(t, "author@g14", valid(n))
|
||||||
|
p := valid(n)
|
||||||
|
p.Sections[2].Text = "Changed."
|
||||||
|
if _, err := w.k.Propose("author@g14", p); err == nil || !strings.Contains(err.Error(), "progressive insight") {
|
||||||
|
t.Fatalf("a decision took a plain version: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// An old active effort's new text is held to the rules as of the day it is proposed, not the day the effort began.
|
||||||
|
func TestANewVersionOfAnOldActiveEffortIsHeldToTodaysRules(t *testing.T) {
|
||||||
|
w := imported(t, newMemStore())
|
||||||
|
w.clock = time.Date(2026, 10, 1, 9, 0, 0, 0, time.UTC)
|
||||||
|
r, err := w.k.ReserveNumber("author@g14", KindResearch, "an old effort")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
overview := Section{Name: "00-overview.md", Text: "What an inference layer would be."}
|
||||||
|
first := effortAsk(r.Number, StatusActive, overview)
|
||||||
|
first.Date = "2026-10-01"
|
||||||
|
w.accepted(t, "author@g14", first)
|
||||||
|
w.clock = time.Date(2026, 10, 15, 9, 0, 0, 0, time.UTC)
|
||||||
|
next := effortAsk(r.Number, StatusActive, overview, Section{Name: "01-findings.md", Text: "The control plane wrote them."})
|
||||||
|
next.Date = "2026-10-01"
|
||||||
|
if _, err := w.k.Propose("author@g14", next); err == nil || !strings.Contains(err.Error(), "retired-word") {
|
||||||
|
t.Fatalf("a retired word in new text of an old effort: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAChangesVerdictLeavesTheServedVersionServedAndGraduatingNeedsABecameLink(t *testing.T) {
|
||||||
|
w := imported(t, newMemStore())
|
||||||
|
r, err := w.k.ReserveNumber("author@g14", KindResearch, "an effort")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
overview := Section{Name: "00-overview.md", Text: "What an inference layer would be."}
|
||||||
|
w.accepted(t, "author@g14", effortAsk(r.Number, StatusActive, overview))
|
||||||
|
key := Key{KindResearch, r.Number}
|
||||||
|
w.judge.review = `{"verdict":"changes","findings":["01-findings.md: say where the finding comes from"]}`
|
||||||
|
next := effortAsk(r.Number, StatusActive, overview, Section{Name: "01-findings.md", Text: "A new finding."})
|
||||||
|
if _, err := w.k.Propose("author@g14", next); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
w.reviewAll(t)
|
||||||
|
e := mustGet(t, w.k.Store, key)
|
||||||
|
if e.Pending == nil || e.Pending.Reviews[len(e.Pending.Reviews)-1].Verdict != VerdictChanges ||
|
||||||
|
e.Current.Status != StatusActive || e.Current.Section("01-findings.md") != "" {
|
||||||
|
t.Fatalf("after changes: pending %v, current %+v", e.Pending != nil, e.Current.Sections)
|
||||||
|
}
|
||||||
|
grad := effortAsk(r.Number, StatusGraduated, overview)
|
||||||
|
if _, err := w.k.Propose("author@g14", grad); err == nil || !strings.Contains(err.Error(), "became") {
|
||||||
|
t.Fatalf("graduated without a became link: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// An earlier marked insight in an old effort's text does not predate the effort, and a new version is accepted.
|
||||||
|
func TestAnOldActiveEffortWithAMarkedInsightTakesANewVersion(t *testing.T) {
|
||||||
|
w := imported(t, newMemStore())
|
||||||
|
w.clock = time.Date(2026, 10, 1, 9, 0, 0, 0, time.UTC)
|
||||||
|
r, err := w.k.ReserveNumber("author@g14", KindResearch, "an old effort")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
overview := Section{Name: "00-overview.md", Text: "What an inference layer would be.\n\n**Progressive insight — 2026-10-05.** A fact was corrected."}
|
||||||
|
first := effortAsk(r.Number, StatusActive, overview)
|
||||||
|
first.Date = "2026-10-01"
|
||||||
|
w.accepted(t, "author@g14", first)
|
||||||
|
w.clock = time.Date(2026, 10, 15, 9, 0, 0, 0, time.UTC)
|
||||||
|
next := effortAsk(r.Number, StatusActive, overview, Section{Name: "01-findings.md", Text: "A new finding."})
|
||||||
|
next.Date = "2026-10-01"
|
||||||
|
if _, err := w.k.Propose("author@g14", next); err != nil {
|
||||||
|
t.Fatalf("a new version with an earlier marked insight: %v", err)
|
||||||
|
}
|
||||||
|
w.reviewAll(t)
|
||||||
|
e := mustGet(t, w.k.Store, Key{KindResearch, r.Number})
|
||||||
|
if e.Pending != nil || e.Current.Section("01-findings.md") != "A new finding." || e.Current.Date != "2026-10-01" {
|
||||||
|
t.Fatalf("after the review: pending %v, date %s", e.Pending != nil, e.Current.Date)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -357,7 +357,7 @@ func (k *KB) Get(a GetAsk) (*Got, error) {
|
|||||||
g.Reserved = fmt.Sprintf("reserved by %s at %s for %q", r.ReservedBy, r.ReservedAt.Format("2006-01-02 15:04 UTC"), r.ReservedFor)
|
g.Reserved = fmt.Sprintf("reserved by %s at %s for %q", r.ReservedBy, r.ReservedAt.Format("2006-01-02 15:04 UTC"), r.ReservedFor)
|
||||||
}
|
}
|
||||||
if e.Pending != nil && a.Version == 0 {
|
if e.Pending != nil && a.Version == 0 {
|
||||||
g.Pending = fmt.Sprintf("version %d, a progressive insight, waits for its review: history shows what it changes", e.PendingVersion)
|
g.Pending = fmt.Sprintf("version %d, a progressive insight or a new version of an active effort, waits for its review: history shows what it changes", e.PendingVersion)
|
||||||
}
|
}
|
||||||
if r.Ask != nil {
|
if r.Ask != nil {
|
||||||
g.Waiting = "the operator's ask: " + r.Ask.Said
|
g.Waiting = "the operator's ask: " + r.Ask.Said
|
||||||
|
|||||||
@@ -335,7 +335,10 @@ func (k *KB) Propose(caller string, a ProposeAsk) (*Proposed, error) {
|
|||||||
return nil, fmt.Errorf("%s has a progressive insight by %s waiting for its review: only its author, proven by the "+
|
return nil, fmt.Errorf("%s has a progressive insight by %s waiting for its review: only its author, proven by the "+
|
||||||
"runtime, revises it (this call is by %s). Nothing was proposed", key, held.Pending.Author, caller)
|
"runtime, revises it (this call is by %s). Nothing was proposed", key, held.Pending.Author, caller)
|
||||||
}
|
}
|
||||||
if strings.TrimSpace(a.Insight) == "" {
|
// An active research effort is still being researched: it takes new versions freely, each reviewed as a
|
||||||
|
// proposal while the served version stays (novox/hq issue 844). Every other accepted record needs an insight.
|
||||||
|
revision := a.Kind == KindResearch && held.Current.Status == StatusActive && strings.TrimSpace(a.Insight) == ""
|
||||||
|
if strings.TrimSpace(a.Insight) == "" && !revision {
|
||||||
return nil, fmt.Errorf("%s is %s: it changes only by a new record, or by a progressive insight, a correction of fact "+
|
return nil, fmt.Errorf("%s is %s: it changes only by a new record, or by a progressive insight, a correction of fact "+
|
||||||
"that leaves the decision standing (give insight: what stood before). Nothing was proposed", key, held.Current.Status)
|
"that leaves the decision standing (give insight: what stood before). Nothing was proposed", key, held.Current.Status)
|
||||||
}
|
}
|
||||||
@@ -349,8 +352,14 @@ func (k *KB) Propose(caller string, a ProposeAsk) (*Proposed, error) {
|
|||||||
r.Pending = &PendingState{Insight: Insight{Date: k.now().Format("2006-01-02"), Before: strings.TrimSpace(a.Insight), By: caller}}
|
r.Pending = &PendingState{Insight: Insight{Date: k.now().Format("2006-01-02"), Before: strings.TrimSpace(a.Insight), By: caller}}
|
||||||
r.Date = held.Current.Date
|
r.Date = held.Current.Date
|
||||||
r.Effort = a.Status
|
r.Effort = a.Status
|
||||||
|
if revision {
|
||||||
|
r.Pending = &PendingState{}
|
||||||
|
}
|
||||||
if why == "" {
|
if why == "" {
|
||||||
why = "a progressive insight proposed"
|
why = "a progressive insight proposed"
|
||||||
|
if revision {
|
||||||
|
why = "a new version proposed"
|
||||||
|
}
|
||||||
}
|
}
|
||||||
default:
|
default:
|
||||||
return nil, fmt.Errorf("%s is %s and takes no proposal; nothing was proposed", key, held.Working().Status)
|
return nil, fmt.Errorf("%s is %s and takes no proposal; nothing was proposed", key, held.Working().Status)
|
||||||
@@ -370,7 +379,13 @@ func (k *KB) Propose(caller string, a ProposeAsk) (*Proposed, error) {
|
|||||||
if !current && a.Kind == KindResearch && r.Effort == "" {
|
if !current && a.Kind == KindResearch && r.Effort == "" {
|
||||||
r.Effort = held.Current.Status
|
r.Effort = held.Current.Status
|
||||||
}
|
}
|
||||||
refusals = append(refusals, JudgeRecord(r, held, k.lookup(ctx), ref)...)
|
// A new version of an active effort is new text: the dated rules hold it from the day it is proposed, while the
|
||||||
|
// record keeps its own date (novox/hq issue 844).
|
||||||
|
day := r.Date
|
||||||
|
if r.Pending != nil && !r.IsInsight() {
|
||||||
|
day = k.now().Format("2006-01-02")
|
||||||
|
}
|
||||||
|
refusals = append(refusals, JudgeRecordOn(r, held, k.lookup(ctx), ref, day)...)
|
||||||
if raw, _ := json.Marshal(r); len(raw) > MostRecordBytes {
|
if raw, _ := json.Marshal(r); len(raw) > MostRecordBytes {
|
||||||
refusals = append(refusals, Refusal{Rule: "size", Says: fmt.Sprintf("the record is %d bytes; a record holds at most %d", len(raw), MostRecordBytes)})
|
refusals = append(refusals, Refusal{Rule: "size", Says: fmt.Sprintf("the record is %d bytes; a record holds at most %d", len(raw), MostRecordBytes)})
|
||||||
}
|
}
|
||||||
@@ -381,7 +396,7 @@ func (k *KB) Propose(caller string, a ProposeAsk) (*Proposed, error) {
|
|||||||
}
|
}
|
||||||
return nil, fmt.Errorf("%s is refused by %d rule(s), and nothing was written:\n- %s", key, len(refusals), strings.Join(lines, "\n- "))
|
return nil, fmt.Errorf("%s is refused by %d rule(s), and nothing was written:\n- %s", key, len(refusals), strings.Join(lines, "\n- "))
|
||||||
}
|
}
|
||||||
v := Version{By: caller, At: k.now(), Why: why, Record: r, Insight: !current}
|
v := Version{By: caller, At: k.now(), Why: why, Record: r, Insight: r.IsInsight()}
|
||||||
if err := k.Store.Save(ctx, v, current, false, held.Revision); err != nil {
|
if err := k.Store.Save(ctx, v, current, false, held.Revision); err != nil {
|
||||||
return nil, fmt.Errorf("%s could not be written (%v); nothing was proposed", key, err)
|
return nil, fmt.Errorf("%s could not be written (%v); nothing was proposed", key, err)
|
||||||
}
|
}
|
||||||
@@ -397,7 +412,10 @@ func (k *KB) Propose(caller string, a ProposeAsk) (*Proposed, error) {
|
|||||||
if r.NeedsOperator {
|
if r.NeedsOperator {
|
||||||
says += ", and then the operator's Approve"
|
says += ", and then the operator's Approve"
|
||||||
}
|
}
|
||||||
if !current {
|
if !current && !r.IsInsight() {
|
||||||
|
says = fmt.Sprintf("A new version of %s is proposed as version %d. The knowledge base reviews it now; the served version stays "+
|
||||||
|
"until that review approves it", key, n)
|
||||||
|
} else if !current {
|
||||||
says = fmt.Sprintf("A progressive insight to %s is proposed as version %d. The knowledge base reviews it now; the accepted "+
|
says = fmt.Sprintf("A progressive insight to %s is proposed as version %d. The knowledge base reviews it now; the accepted "+
|
||||||
"version is served until that review approves it", key, n)
|
"version is served until that review approves it", key, n)
|
||||||
}
|
}
|
||||||
@@ -416,8 +434,11 @@ func (k *KB) accept(ctx context.Context, held *Entry, w *Record, by, why, warran
|
|||||||
at := k.now()
|
at := k.now()
|
||||||
if insight {
|
if insight {
|
||||||
in := r.Pending.Insight
|
in := r.Pending.Insight
|
||||||
|
isInsight := r.IsInsight()
|
||||||
r.Pending = nil
|
r.Pending = nil
|
||||||
r.Insights = append(r.Insights, in)
|
if isInsight {
|
||||||
|
r.Insights = append(r.Insights, in)
|
||||||
|
}
|
||||||
r.Status = held.Current.Status
|
r.Status = held.Current.Status
|
||||||
if r.Kind == KindResearch && contains(ResearchStatuses, r.Effort) && held.Current.Status != StatusSuperseded {
|
if r.Kind == KindResearch && contains(ResearchStatuses, r.Effort) && held.Current.Status != StatusSuperseded {
|
||||||
r.Status = r.Effort
|
r.Status = r.Effort
|
||||||
@@ -442,7 +463,7 @@ func (k *KB) accept(ctx context.Context, held *Entry, w *Record, by, why, warran
|
|||||||
if warrant != "" {
|
if warrant != "" {
|
||||||
why += "; " + warrant
|
why += "; " + warrant
|
||||||
}
|
}
|
||||||
if err := k.Store.Save(ctx, Version{By: by, At: at, Why: why, Record: r, Insight: insight}, true, true, held.Revision); err != nil {
|
if err := k.Store.Save(ctx, Version{By: by, At: at, Why: why, Record: r, Insight: insight && w.IsInsight()}, true, true, held.Revision); err != nil {
|
||||||
return nil, fmt.Errorf("%s could not be accepted (%v); it stays proposed", r.Key(), err)
|
return nil, fmt.Errorf("%s could not be accepted (%v); it stays proposed", r.Key(), err)
|
||||||
}
|
}
|
||||||
k.emit(EventAccepted, r, by, why, held.Versions+1)
|
k.emit(EventAccepted, r, by, why, held.Versions+1)
|
||||||
|
|||||||
@@ -917,7 +917,7 @@ func Tools(k *KB, caller func(map[string]any) string) []stdio.Tool {
|
|||||||
"links (supersedes, changes \"0276 §4\", extends, became, cites), and whether it needs the operator, with the brief its ask shows. Refused, naming " +
|
"links (supersedes, changes \"0276 §4\", extends, became, cites), and whether it needs the operator, with the brief its ask shows. Refused, naming " +
|
||||||
"every reason at once and writing nothing, when it breaks one of the 30 rules that judge one record or its prose — a " +
|
"every reason at once and writing nothing, when it breaks one of the 30 rules that judge one record or its prose — a " +
|
||||||
"link to nothing, a clause without a bold lead, a superseded record cited alone, an unknown topic, a retired word. A " +
|
"link to nothing, a clause without a bold lead, a superseded record cited alone, an unknown topic, a retired word. A " +
|
||||||
"proposal is revised by proposing it again; an accepted record changes only by a progressive insight (insight: what " +
|
"proposal is revised by proposing it again; an accepted record (an active research effort excepted, which takes new versions) changes only by a progressive insight (insight: what " +
|
||||||
"stood before). The knowledge base then reviews each version itself, in a judgement that sees only the record, the " +
|
"stood before). The knowledge base then reviews each version itself, in a judgement that sees only the record, the " +
|
||||||
"records it links and the rules. A decision that names no home and that no accepted record links is not judged: " +
|
"records it links and the rules. A decision that names no home and that no accepted record links is not judged: " +
|
||||||
"its review asks changes for the whole-set rule cited, which its acceptance would break. On approve it is accepted (a record that needs the operator waits for the operator's " +
|
"its review asks changes for the whole-set rule cited, which its acceptance would break. On approve it is accepted (a record that needs the operator waits for the operator's " +
|
||||||
|
|||||||
@@ -192,7 +192,7 @@ func (k *KB) askOperator(ctx context.Context, held *Entry, w *Record, by string,
|
|||||||
if err != nil || e == nil {
|
if err != nil || e == nil {
|
||||||
return "", fmt.Errorf("%s: %s, and that could not be kept (%v)", w.Key(), said, err)
|
return "", fmt.Errorf("%s: %s, and that could not be kept (%v)", w.Key(), said, err)
|
||||||
}
|
}
|
||||||
if err := k.Store.Save(ctx, Version{By: ReviewBy, At: k.now(), Why: said, Record: w, Insight: e.Pending != nil},
|
if err := k.Store.Save(ctx, Version{By: ReviewBy, At: k.now(), Why: said, Record: w, Insight: e.Pending != nil && e.Pending.IsInsight()},
|
||||||
e.Pending == nil, false, e.Revision); err != nil {
|
e.Pending == nil, false, e.Revision); err != nil {
|
||||||
return "", fmt.Errorf("%s: %s, and that could not be kept (%v)", w.Key(), said, err)
|
return "", fmt.Errorf("%s: %s, and that could not be kept (%v)", w.Key(), said, err)
|
||||||
}
|
}
|
||||||
@@ -208,7 +208,7 @@ func (k *KB) askOperator(ctx context.Context, held *Entry, w *Record, by string,
|
|||||||
return unasked(fmt.Sprintf("the ask could not be made (%v)", err))
|
return unasked(fmt.Sprintf("the ask could not be made (%v)", err))
|
||||||
}
|
}
|
||||||
w.Ask = oa
|
w.Ask = oa
|
||||||
if err := k.Store.Save(ctx, Version{By: by, At: k.now(), Why: "reviewed: approved; the operator is asked", Record: w, Insight: held.Pending != nil},
|
if err := k.Store.Save(ctx, Version{By: by, At: k.now(), Why: "reviewed: approved; the operator is asked", Record: w, Insight: held.Pending != nil && held.Pending.IsInsight()},
|
||||||
held.Pending == nil, false, held.Revision); err != nil {
|
held.Pending == nil, false, held.Revision); err != nil {
|
||||||
return "", fmt.Errorf("%s: the review could not be kept (%v); the operator was not asked", w.Key(), err)
|
return "", fmt.Errorf("%s: the review could not be kept (%v); the operator was not asked", w.Key(), err)
|
||||||
}
|
}
|
||||||
@@ -267,7 +267,7 @@ func (k *KB) Reconcile() ([]string, error) {
|
|||||||
func (k *KB) endAsk(ctx context.Context, e *Entry, w *Record, why string) string {
|
func (k *KB) endAsk(ctx context.Context, e *Entry, w *Record, why string) string {
|
||||||
w = w.Clone()
|
w = w.Clone()
|
||||||
w.Ask.Said = why
|
w.Ask.Said = why
|
||||||
if err := k.Store.Save(ctx, Version{By: "the knowledge base", At: k.now(), Why: "the operator's ask " + why, Record: w, Insight: e.Pending != nil},
|
if err := k.Store.Save(ctx, Version{By: "the knowledge base", At: k.now(), Why: "the operator's ask " + why, Record: w, Insight: e.Pending != nil && e.Pending.IsInsight()},
|
||||||
e.Pending == nil, false, e.Revision); err != nil {
|
e.Pending == nil, false, e.Revision); err != nil {
|
||||||
return fmt.Sprintf("%s: the end of its ask could not be kept (%v)", w.Key(), err)
|
return fmt.Sprintf("%s: the end of its ask could not be kept (%v)", w.Key(), err)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -360,6 +360,10 @@ type PendingState struct {
|
|||||||
Insight Insight `json:"insight"`
|
Insight Insight `json:"insight"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// IsInsight says the record is waiting as a progressive insight, a correction of fact that says what stood before.
|
||||||
|
// A pending version without one is a new version of an active research effort (novox/hq issue 844).
|
||||||
|
func (r *Record) IsInsight() bool { return r.Pending != nil && r.Pending.Insight.Before != "" }
|
||||||
|
|
||||||
// Key is the record's key.
|
// Key is the record's key.
|
||||||
func (r *Record) Key() Key { return Key{r.Kind, r.Number} }
|
func (r *Record) Key() Key { return Key{r.Kind, r.Number} }
|
||||||
|
|
||||||
|
|||||||
@@ -111,7 +111,7 @@ var reviewSchema = map[string]any{
|
|||||||
func ReviewQuestion(r *Record, linked []*Record) string {
|
func ReviewQuestion(r *Record, linked []*Record) string {
|
||||||
var b strings.Builder
|
var b strings.Builder
|
||||||
what := "a proposed " + r.Kind + " record"
|
what := "a proposed " + r.Kind + " record"
|
||||||
if r.Pending != nil {
|
if r.IsInsight() {
|
||||||
what = "a correction of fact (a progressive insight) proposed to an accepted " + r.Kind + " record; what stood before: " + r.Pending.Insight.Before
|
what = "a correction of fact (a progressive insight) proposed to an accepted " + r.Kind + " record; what stood before: " + r.Pending.Insight.Before
|
||||||
}
|
}
|
||||||
fmt.Fprintf(&b, "You review %s of the mesh's knowledge base, %s. You see only the record, the records it links, and the "+
|
fmt.Fprintf(&b, "You review %s of the mesh's knowledge base, %s. You see only the record, the records it links, and the "+
|
||||||
@@ -475,7 +475,7 @@ func (k *KB) reviewOne(key Key) (string, error) {
|
|||||||
version = now.PendingVersion
|
version = now.PendingVersion
|
||||||
}
|
}
|
||||||
save := func(r *Record, why string) error {
|
save := func(r *Record, why string) error {
|
||||||
return k.Store.Save(ctx, Version{By: by, At: k.now(), Why: why, Record: r, Insight: now.Pending != nil},
|
return k.Store.Save(ctx, Version{By: by, At: k.now(), Why: why, Record: r, Insight: now.Pending != nil && now.Pending.IsInsight()},
|
||||||
now.Pending == nil, false, now.Revision)
|
now.Pending == nil, false, now.Revision)
|
||||||
}
|
}
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|||||||
@@ -123,9 +123,16 @@ func allText(r *Record) string {
|
|||||||
// JudgeRecord holds a proposed record to the 30 rules. held is the entry its number holds (a reservation, the
|
// JudgeRecord holds a proposed record to the 30 rules. held is the entry its number holds (a reservation, the
|
||||||
// proposal being revised, or the accepted record an insight corrects); lookup reads every other record.
|
// proposal being revised, or the accepted record an insight corrects); lookup reads every other record.
|
||||||
func JudgeRecord(r *Record, held *Entry, lookup Lookup, ref *Reference) []Refusal {
|
func JudgeRecord(r *Record, held *Entry, lookup Lookup, ref *Reference) []Refusal {
|
||||||
|
return JudgeRecordOn(r, held, lookup, ref, r.Date)
|
||||||
|
}
|
||||||
|
|
||||||
|
// JudgeRecordOn is JudgeRecord with the day the dated rules are held from: the record's own day for a new
|
||||||
|
// record, the day of the proposal for a new version of an active research effort (novox/hq issue 844). The rules
|
||||||
|
// that read the record's date (the date and insight-form rules) keep reading r.Date.
|
||||||
|
func JudgeRecordOn(r *Record, held *Entry, lookup Lookup, ref *Reference, day string) []Refusal {
|
||||||
var out []Refusal
|
var out []Refusal
|
||||||
refuse := func(rule, format string, a ...any) {
|
refuse := func(rule, format string, a ...any) {
|
||||||
if Holds(rule, r.Kind, r.Date) {
|
if Holds(rule, r.Kind, day) {
|
||||||
out = append(out, Refusal{Rule: rule, Says: fmt.Sprintf(format, a...)})
|
out = append(out, Refusal{Rule: rule, Says: fmt.Sprintf(format, a...)})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,40 @@
|
|||||||
|
# mesh-mail-notifier — the default holder of mail-notifier
|
||||||
|
|
||||||
|
Built from mesh-mail's source (novox/hq ADR 0378 §9, as amended by ADR 0379, issue 800 part B): it carries a
|
||||||
|
byte-identical copy of mesh-mail's event shape (`mailevent.go`, pinned by `copies_test.go`, the way
|
||||||
|
`modules/networkmanager`'s `uplink-tools` carries `ifwatchd`'s own file, ADR 0241). Nothing here names
|
||||||
|
Betterbird (ADR 0379 §3), and nothing here reaches a mailbox: it only hears what mesh-mail already said.
|
||||||
|
|
||||||
|
## What it does
|
||||||
|
|
||||||
|
It claims `mail-notifier` (held once per machine, needing `node-notifier` there) and shows each new inbox mail
|
||||||
|
on this machine's notifier, through `node-notifier.send`, at most once:
|
||||||
|
|
||||||
|
- quiet during a first sync or a resync (mesh-mail already says nothing then, `imap.go` ~387)
|
||||||
|
- quiet for a mail from the account's own address (its address, from `mail-store.accounts`; not known, the
|
||||||
|
notice is shown)
|
||||||
|
- quiet for a mail already marked read (`\Seen` in its flags)
|
||||||
|
- quiet for anything outside the inbox
|
||||||
|
- of one account's qualifying mail within a sliding minute, the first three are shown one by one; the rest of
|
||||||
|
that minute's mail is folded into one notice naming the count and the latest sender, rather than shown one
|
||||||
|
by one
|
||||||
|
|
||||||
|
A notice `node-notifier.send` refuses stays pending and is tried again, at most once a minute, without holding
|
||||||
|
up any other account's notice; nothing is deleted before it is shown. Each notice shown emits this module's
|
||||||
|
own event `shown` (account and message id, never sender or subject), and is kept in its own state (`shown`) so
|
||||||
|
a redelivered event is never counted or shown twice — by the mail's message id wherever it has one, so a
|
||||||
|
UIDVALIDITY reset cannot make an already-shown mail look new.
|
||||||
|
|
||||||
|
## Where it runs
|
||||||
|
|
||||||
|
**Its own account** (`runs-as`), because it hears mesh-mail's restricted stream `MAIL` (novox/hq ADR 0376 §2):
|
||||||
|
the machine's runtime runs as the operator's account, which every agent shares and which never hears a
|
||||||
|
restricted stream. Held once per machine (wherever `mail-notifier` is assigned); the operator grants it the
|
||||||
|
accounts it announces, in mesh-mail's trusted setting `grants` (`mail-store`'s stream).
|
||||||
|
|
||||||
|
## Tools
|
||||||
|
|
||||||
|
| tool | does |
|
||||||
|
|---|---|
|
||||||
|
| `mail-notifier.status` | every account it is granted, when it last showed each one a notice, and any notice still failing to send (its count, last error, since when). Read-only |
|
||||||
|
| `mail_notifier_health` | how it is: unhealthy once a notice has failed to show for more than ten minutes, healthy otherwise, even mid-retry. Not (yet) the node-engine's own check — this is one process, judged only by its own tool (ADR 0227 rule 8); a watching process of its own is issue 841, not yet decided. Read-only |
|
||||||
@@ -0,0 +1,31 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
// mesh-mail-notifier carries a byte-identical copy of mesh-mail's event shape (novox/hq ADR 0378 §9, issue 800
|
||||||
|
// part B), the way modules/networkmanager's uplink-tools carries ifwatchd's own file (ADR 0241, copies_test.go):
|
||||||
|
// each builds alone, so each holds its own copy, and this test holds them to one text.
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
// source is where mesh-mail's own copy of the file lives.
|
||||||
|
var source = filepath.Join("..", "..", "..", "mesh-mail", "cmd", "mesh-mail")
|
||||||
|
|
||||||
|
func TestCarriesTheSameMailEventAsMeshMail(t *testing.T) {
|
||||||
|
for _, f := range []string{"mailevent.go"} {
|
||||||
|
mine, err := os.ReadFile(f)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
theirs, err := os.ReadFile(filepath.Join(source, f))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("mesh-mail's copy of %s could not be read: %v", f, err)
|
||||||
|
}
|
||||||
|
if !bytes.Equal(mine, theirs) {
|
||||||
|
t.Errorf("this copy of %s differs from mesh-mail's: change both together", f)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,62 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
// mesh-mail-notifier's own health (review round 1 of issue 800, which added this tool beside mail-notifier's
|
||||||
|
// seat verb status, as mesh-mail's own health tool is mail_health beside its seat tools). A notice that fails
|
||||||
|
// to send stays pending and is retried, at most once a minute (Flush, notifier.go); the account's
|
||||||
|
// failing-since stays in the Store, so this tool — its own process, asked separately by the node-engine or by
|
||||||
|
// mesh_call — sees the same account the live notifier does, with no state of its own to lose between asks.
|
||||||
|
//
|
||||||
|
// Unhealthy once an account's notice has failed to show for more than UnhealthyAfter; healthy otherwise, even
|
||||||
|
// while a notice is mid-retry.
|
||||||
|
//
|
||||||
|
// STOP (review round 1 of issue 800, not decided by its note): this tool is served and tested, but module.json
|
||||||
|
// does not name it as the notify process's own health check — mesh-mail-notifier has one process (notify), and
|
||||||
|
// the catalogue's gate refuses a module judged only by its own tool, with no check of another kind beside it
|
||||||
|
// (ADR 0227 rule 8, ADR 0240 rule 2; internal/catalogue/health.go ~297 in mesh-controller). mesh-mail avoids
|
||||||
|
// this because it has two processes — keep (kind unit) beside watch (kind tool, mail_health); mesh-mail-notifier
|
||||||
|
// has only notify, so wiring this tool the same way trips the gate. notify's health stays kind unit (liveness,
|
||||||
|
// as it was before this round) until the lead decides how a one-process module gets an automatic, node-engine
|
||||||
|
// enforced health of this kind: a second process judged some other way, or a change to the gate's rule.
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/health"
|
||||||
|
)
|
||||||
|
|
||||||
|
// HealthTool is the module's own health tool the node-engine asks (beside mail-notifier.status, the seat's own verb).
|
||||||
|
const HealthTool = "mail_notifier_health"
|
||||||
|
|
||||||
|
// UnhealthyAfter is how long a notice may fail to send before that is unhealthy, not merely retrying.
|
||||||
|
const UnhealthyAfter = 10 * time.Minute
|
||||||
|
|
||||||
|
// Judge is mesh-mail-notifier's health, from the failures the Store keeps.
|
||||||
|
func Judge(failures map[string]FailureStatus, now time.Time) health.Answer {
|
||||||
|
if len(failures) == 0 {
|
||||||
|
return health.Healthy("every notice shown; none pending a retry")
|
||||||
|
}
|
||||||
|
accounts := make([]string, 0, len(failures))
|
||||||
|
for account := range failures {
|
||||||
|
accounts = append(accounts, account)
|
||||||
|
}
|
||||||
|
sort.Strings(accounts)
|
||||||
|
|
||||||
|
var problems, retrying []string
|
||||||
|
for _, account := range accounts {
|
||||||
|
f := failures[account]
|
||||||
|
age := now.Sub(f.Since).Round(time.Second)
|
||||||
|
if now.Sub(f.Since) > UnhealthyAfter {
|
||||||
|
problems = append(problems, fmt.Sprintf("%s: its notice has failed to show for %s (%d attempt(s), last: %s)",
|
||||||
|
account, age, f.Count, f.LastError))
|
||||||
|
} else {
|
||||||
|
retrying = append(retrying, fmt.Sprintf("%s has been retrying for %s", account, age))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(problems) > 0 {
|
||||||
|
return health.Unhealthy(strings.Join(problems, "; "))
|
||||||
|
}
|
||||||
|
return health.Healthy(strings.Join(retrying, "; "))
|
||||||
|
}
|
||||||
@@ -0,0 +1,39 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/health"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestJudgeIsUnhealthyOnlyPastUnhealthyAfter(t *testing.T) {
|
||||||
|
now := time.Date(2026, 10, 15, 12, 0, 0, 0, time.UTC)
|
||||||
|
|
||||||
|
if a := Judge(nil, now); a.State != health.StateHealthy {
|
||||||
|
t.Fatalf("no failures: %+v, want healthy", a)
|
||||||
|
}
|
||||||
|
|
||||||
|
recent := map[string]FailureStatus{"work": {Count: 1, LastError: "no answer", Since: now.Add(-2 * time.Minute)}}
|
||||||
|
if a := Judge(recent, now); a.State != health.StateHealthy {
|
||||||
|
t.Fatalf("a failure 2m old: %+v, want healthy (still retrying)", a)
|
||||||
|
}
|
||||||
|
|
||||||
|
stale := map[string]FailureStatus{"work": {Count: 11, LastError: "no answer", Since: now.Add(-11 * time.Minute)}}
|
||||||
|
a := Judge(stale, now)
|
||||||
|
if a.State != health.StateUnhealthy {
|
||||||
|
t.Fatalf("a failure 11m old: %+v, want unhealthy (past UnhealthyAfter)", a)
|
||||||
|
}
|
||||||
|
if !strings.Contains(a.Why, "work") || !strings.Contains(a.Why, "no answer") {
|
||||||
|
t.Errorf("Why = %q, want it to name the account and the last error", a.Why)
|
||||||
|
}
|
||||||
|
|
||||||
|
mixed := map[string]FailureStatus{
|
||||||
|
"work": {Count: 11, LastError: "no answer", Since: now.Add(-11 * time.Minute)},
|
||||||
|
"home": {Count: 1, LastError: "no answer", Since: now.Add(-time.Minute)},
|
||||||
|
}
|
||||||
|
if a := Judge(mixed, now); a.State != health.StateUnhealthy || !strings.Contains(a.Why, "work") {
|
||||||
|
t.Errorf("one stale failure beside one fresh one: %+v, want unhealthy naming work", a)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,13 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
// The shape of a mail-store message event (novox/hq ADR 0378 §2, issue 800 part A), moved here on its own so
|
||||||
|
// mesh-mail-notifier (issue 800 part B) carries a byte-identical copy, the way modules/networkmanager's
|
||||||
|
// uplink-tools carries ifwatchd's own file (novox/hq ADR 0241, copies_test.go). Nothing here reaches a mailbox or
|
||||||
|
// a server; it only says where an account's change is told on the bus and what the three changes are called.
|
||||||
|
|
||||||
|
// EventKey is where an account's change is said: mail.<account>.message.<added|changed|removed> (research 049
|
||||||
|
// document 02 §5), the account by its name in the setting, never its address.
|
||||||
|
func EventKey(account, change string) string { return "mail." + account + ".message." + change }
|
||||||
|
|
||||||
|
// Changes are the three things said of a message.
|
||||||
|
var Changes = []string{"added", "changed", "removed"}
|
||||||
@@ -0,0 +1,232 @@
|
|||||||
|
// mesh-mail-notifier: the default claimant of mail-notifier, built from mesh-mail's own event shape (novox/hq
|
||||||
|
// ADR 0378 §9, issue 800 part B). A Go bundle the node's runtime launches, its own account (runs-as, ADR 0376
|
||||||
|
// §2): the one process serves mail-notifier.status and the module's own health tool (mail_notifier_health,
|
||||||
|
// review round 1 of issue 800), and, beside stdio.Serve, hears mesh-mail's added events and shows them
|
||||||
|
// (exactly as modules/mesh-knowledge's serve starts go hear(k) and go background(k) before stdio.Serve) — not
|
||||||
|
// a separate `notify` process, which systemd gave an empty stdin and so exited at once, every five seconds
|
||||||
|
// (novox/hq issue 859). What this module says, it says on stderr; stdout is the MCP channel.
|
||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"os"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||||
|
)
|
||||||
|
|
||||||
|
func logf(format string, a ...any) { fmt.Fprintf(os.Stderr, format+"\n", a...) }
|
||||||
|
|
||||||
|
// AddedPattern is the mesh-mail stream's event this module consumes (its own `consumes` entry, novox/hq ADR
|
||||||
|
// 0363, ADR 0378 §9): a new, changed or removed message's `added` only.
|
||||||
|
const AddedPattern = "mesh-mail.mail.*.message.added"
|
||||||
|
|
||||||
|
// StatusTool is the mail-notifier seat's one verb this module serves.
|
||||||
|
const StatusTool = "mail-notifier.status"
|
||||||
|
|
||||||
|
// ShownEvent is this module's own event (its `emits`), one per notice shown.
|
||||||
|
const ShownEvent = "shown"
|
||||||
|
|
||||||
|
func main() {
|
||||||
|
n := &Notifier{
|
||||||
|
Send: func(args map[string]any) error {
|
||||||
|
_, err := stdio.Ask(sendKey(), args)
|
||||||
|
return err
|
||||||
|
},
|
||||||
|
Accounts: readAccounts,
|
||||||
|
Store: busStore{},
|
||||||
|
EmitShown: func(account, messageID string) error {
|
||||||
|
return stdio.Emit(ShownEvent, map[string]any{"account": account, "message-id": messageID})
|
||||||
|
},
|
||||||
|
Logf: logf,
|
||||||
|
}
|
||||||
|
n.init()
|
||||||
|
|
||||||
|
tools := []stdio.Tool{
|
||||||
|
{Name: StatusTool,
|
||||||
|
Description: "The accounts mesh-mail-notifier announces on this machine, and when it last showed a " +
|
||||||
|
"notice of new mail (novox/hq ADR 0378 §7). Read-only.",
|
||||||
|
Run: func(map[string]any) (any, error) { return n.Status() }},
|
||||||
|
{Name: HealthTool,
|
||||||
|
Description: "How mesh-mail-notifier is: unhealthy once a notice has failed to show for more than " +
|
||||||
|
"ten minutes, healthy otherwise, even mid-retry (review round 1 of issue 800). Read-only.",
|
||||||
|
Run: func(map[string]any) (any, error) {
|
||||||
|
failures, err := busStore{}.Failures()
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return Judge(failures, time.Now()), nil
|
||||||
|
}},
|
||||||
|
}
|
||||||
|
if err := run(n, watch, os.Stdin, os.Stdout, tools); err != nil {
|
||||||
|
logf("%v", err)
|
||||||
|
os.Exit(1)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// sendKey is the ask key Send uses: this machine's own node-notifier, named exactly as desk-channel asks
|
||||||
|
// another machine's ("seat:node-notifier.send@"+machine) and as the SDK's own Emit reads this machine's name
|
||||||
|
// (MESH_NODE, which the runtime sets) — not the bare seat, which any machine's node-notifier may answer
|
||||||
|
// first, leaving this one's notices undelivered (novox/hq issue 865).
|
||||||
|
func sendKey() string { return "seat:node-notifier.send@" + os.Getenv("MESH_NODE") }
|
||||||
|
|
||||||
|
// run is main's startup path, factored so a test can run it: the watcher started before stdio.Serve, exactly
|
||||||
|
// as mesh-knowledge's serve starts go hear(k) and go background(k) before stdio.Serve — so the watcher keeps
|
||||||
|
// running whatever stdio.Serve does with the stream it is given, including returning at once when stdin is
|
||||||
|
// already at EOF, as systemd gave the old separate `notify` process (novox/hq issue 859). The node's runtime
|
||||||
|
// launches this one process and keeps it alive.
|
||||||
|
func run(n *Notifier, watcher func(*Notifier), in io.Reader, out io.Writer, tools []stdio.Tool) error {
|
||||||
|
go watcher(n)
|
||||||
|
return stdio.ServeOn("", tools, in, out)
|
||||||
|
}
|
||||||
|
|
||||||
|
// watch hears mesh-mail's added events and, once a second, flushes what Notifier.Flush finds ready: an
|
||||||
|
// account's first mails of its window at once, and its folded overflow once the window is over (ADR 0378 §9,
|
||||||
|
// the lead's reading, review round 1 of issue 800).
|
||||||
|
func watch(n *Notifier) { watchWith(n, stdio.Subscribe, time.Sleep) }
|
||||||
|
|
||||||
|
// watchWith is watch with its subscribe and sleep factored out, so a test can make a subscribe fail without
|
||||||
|
// the real bus and without the real wait. A subscribe the runtime refuses is retried, 2 seconds doubling to
|
||||||
|
// at most a minute, logging each refusal — exactly as modules/mesh-knowledge's hear does — never fatal: a
|
||||||
|
// refusal here must never take mail-notifier.status and mail_notifier_health down with it (novox/hq issue
|
||||||
|
// 859, review round 1; the old os.Exit(1) did exactly that).
|
||||||
|
func watchWith(n *Notifier, subscribe func(string, func(stdio.Envelope) error) error, sleep func(time.Duration)) {
|
||||||
|
for wait := 2 * time.Second; ; wait = min(wait*2, time.Minute) {
|
||||||
|
sleep(wait)
|
||||||
|
err := subscribe(AddedPattern, func(e stdio.Envelope) error {
|
||||||
|
var body EventBody
|
||||||
|
if err := json.Unmarshal(e.Body, &body); err != nil {
|
||||||
|
logf("[mesh-mail-notifier] an added event could not be read; taken and dropped: %v", err)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return n.Add(body)
|
||||||
|
})
|
||||||
|
if err == nil {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
logf("[mesh-mail-notifier] does not hear %s yet (%v); asking again in %s", AddedPattern, err, wait)
|
||||||
|
}
|
||||||
|
ticker := time.NewTicker(1 * time.Second)
|
||||||
|
defer ticker.Stop()
|
||||||
|
for range ticker.C {
|
||||||
|
if err := n.Flush(); err != nil {
|
||||||
|
logf("[mesh-mail-notifier] a buffered notice could not be shown: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// readAccounts asks mail-store.accounts for the accounts this module is granted, by name to own address.
|
||||||
|
func readAccounts() (map[string]string, error) {
|
||||||
|
raw, err := stdio.Ask("seat:mail-store.accounts", map[string]any{})
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
var answer struct {
|
||||||
|
Accounts []struct {
|
||||||
|
Account string `json:"account"`
|
||||||
|
Address string `json:"address"`
|
||||||
|
} `json:"accounts"`
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal(raw, &answer); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
out := map[string]string{}
|
||||||
|
for _, a := range answer.Accounts {
|
||||||
|
out[a.Account] = a.Address
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// busStore keeps the notifier's own state (the `shown` bucket the manifest declares): which messages were
|
||||||
|
// shown already, each account's last notice, and each account's notice still failing to send.
|
||||||
|
type busStore struct{}
|
||||||
|
|
||||||
|
func stateKey(parts ...string) string {
|
||||||
|
words := make([]string, len(parts))
|
||||||
|
for i, p := range parts {
|
||||||
|
var b strings.Builder
|
||||||
|
for _, r := range p {
|
||||||
|
switch {
|
||||||
|
case r >= 'a' && r <= 'z', r >= 'A' && r <= 'Z', r >= '0' && r <= '9', r == '-':
|
||||||
|
b.WriteRune(r)
|
||||||
|
default:
|
||||||
|
b.WriteRune('_')
|
||||||
|
}
|
||||||
|
}
|
||||||
|
words[i] = b.String()
|
||||||
|
}
|
||||||
|
return strings.Join(words, "_")
|
||||||
|
}
|
||||||
|
|
||||||
|
func (busStore) HandleSeen(handle string) (bool, error) {
|
||||||
|
e, err := stdio.State("shown").Get("handle_" + stateKey(strings.Split(handle, "\x1f")...))
|
||||||
|
return e != nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
func (busStore) MarkHandle(handle string) error {
|
||||||
|
_, err := stdio.State("shown").Put("handle_"+stateKey(strings.Split(handle, "\x1f")...), true)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
func (busStore) PutStatus(account string, s AccountStatus) error {
|
||||||
|
_, err := stdio.State("shown").Put("status_"+stateKey(account), s)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
func (busStore) Statuses() (map[string]AccountStatus, error) {
|
||||||
|
keys, err := stdio.State("shown").Keys()
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
out := map[string]AccountStatus{}
|
||||||
|
for _, k := range keys {
|
||||||
|
account, ok := strings.CutPrefix(k, "status_")
|
||||||
|
if !ok {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
e, err := stdio.State("shown").Get(k)
|
||||||
|
if err != nil || e == nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
var s AccountStatus
|
||||||
|
if json.Unmarshal(e.Value, &s) == nil {
|
||||||
|
out[account] = s
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (busStore) PutFailure(account string, f FailureStatus) error {
|
||||||
|
_, err := stdio.State("shown").Put("failure_"+stateKey(account), f)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
func (busStore) ClearFailure(account string) error {
|
||||||
|
return stdio.State("shown").Delete("failure_" + stateKey(account))
|
||||||
|
}
|
||||||
|
|
||||||
|
func (busStore) Failures() (map[string]FailureStatus, error) {
|
||||||
|
keys, err := stdio.State("shown").Keys()
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
out := map[string]FailureStatus{}
|
||||||
|
for _, k := range keys {
|
||||||
|
account, ok := strings.CutPrefix(k, "failure_")
|
||||||
|
if !ok {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
e, err := stdio.State("shown").Get(k)
|
||||||
|
if err != nil || e == nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
var f FailureStatus
|
||||||
|
if json.Unmarshal(e.Value, &f) == nil {
|
||||||
|
out[account] = f
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,38 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The manifest's own shape (novox/hq issue 859): the watch loop runs in the base invocation beside
|
||||||
|
// stdio.Serve, so no resource declares a separate `notify` process any more — the thing that exited at once,
|
||||||
|
// every five seconds, because a systemd-run process's stdin is empty and stdio.Serve returns at EOF.
|
||||||
|
|
||||||
|
type manifestResources struct {
|
||||||
|
Resources []map[string]any `json:"resources"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func readManifestResources(t *testing.T) manifestResources {
|
||||||
|
t.Helper()
|
||||||
|
raw, err := os.ReadFile(filepath.Join("..", "..", "module.json"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var m manifestResources
|
||||||
|
if err := json.Unmarshal(raw, &m); err != nil {
|
||||||
|
t.Fatalf("module.json: %v", err)
|
||||||
|
}
|
||||||
|
return m
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNoProcessResourceIsDeclared(t *testing.T) {
|
||||||
|
m := readManifestResources(t)
|
||||||
|
for _, r := range m.Resources {
|
||||||
|
if r["type"] == "process" {
|
||||||
|
t.Fatalf("module.json still declares a process resource (%v): the watcher runs in the base invocation now, not a separate process (novox/hq issue 859)", r)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,537 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
// mesh-mail-notifier: the default holder of mail-notifier (novox/hq ADR 0378 §7, as amended by ADR 0379 §4,
|
||||||
|
// issue 800 part B). It hears mesh-mail's added events on its own account (runs-as, ADR 0376 §2) and shows each
|
||||||
|
// new inbox mail at most once, through node-notifier.send: quiet during a first sync (mesh-mail already says
|
||||||
|
// nothing then, imap.go ~387), for mail from the account's own address, for mail already marked read (\Seen),
|
||||||
|
// and for anything outside the inbox. For each new qualifying mail of an account, arriving at t, N counts that
|
||||||
|
// account's qualifying mail — shown or folded alike, each counted once — that arrived in the 60 seconds up to
|
||||||
|
// and including t: N <= FoldOver is shown at once; N > FoldOver is folded. The first folded mail opens the
|
||||||
|
// account's fold; every folded mail joins it; the fold is shown, as one notice naming how many it holds and
|
||||||
|
// the latest sender, once Window has passed since its own first mail — after which a new fold may open. The
|
||||||
|
// count is a true sliding window of arrivals, not a per-session quota that only resets after a quiet Window:
|
||||||
|
// review round 2's reading kept a session's count alive, and so folding, for as long as any gap stayed under
|
||||||
|
// Window, which folds a steady trickle (one mail every 50s, say) forever even though no burst ever happened;
|
||||||
|
// review round 3 of issue 800 replaces that reading with this one. Nothing pending — a folded mail awaiting
|
||||||
|
// its fold's deadline, or a failed send awaiting its retry — is ever dropped (novox/hq ADR 0378 §9, review
|
||||||
|
// round 3 of issue 800, which replaces the round 1 and round 2 readings).
|
||||||
|
//
|
||||||
|
// Nothing here names Betterbird (ADR 0379 §3).
|
||||||
|
|
||||||
|
import (
|
||||||
|
"sort"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Inbox is the one folder a new mail is shown for.
|
||||||
|
const Inbox = "INBOX"
|
||||||
|
|
||||||
|
// Seen is the flag that means a mail was already read: shown for nothing it marks.
|
||||||
|
const SeenFlag = "\\Seen"
|
||||||
|
|
||||||
|
// FoldOver is how many new mails from one account, within one window, are still shown one by one; the rest of
|
||||||
|
// that window is folded into a single notice (novox/hq ADR 0378 §9: "more than three").
|
||||||
|
const FoldOver = 3
|
||||||
|
|
||||||
|
// Window is the trailing 60 seconds ADR 0378 §9 counts a mail's N over (every qualifying mail of the account,
|
||||||
|
// shown or folded alike, that arrived in the 60 seconds up to and including this one), and the time a fold
|
||||||
|
// waits, from its own first mail, before it is shown (review round 3 of issue 800, which replaces the round 1
|
||||||
|
// and round 2 readings: the count is a true sliding window of arrivals, not a per-session quota that stays
|
||||||
|
// alive, and so keeps folding, for as long as any gap between mails stays under Window).
|
||||||
|
const Window = time.Minute
|
||||||
|
|
||||||
|
// RetryBackoff is how long the notifier waits, after an account's notice fails to send, before it tries that
|
||||||
|
// account again: at most once a minute (review round 1 of issue 800), so a send that keeps failing is not
|
||||||
|
// retried on every Flush.
|
||||||
|
const RetryBackoff = time.Minute
|
||||||
|
|
||||||
|
// AccountsCacheFor is how long mail-store.accounts's answer is reused before it is asked again (review round
|
||||||
|
// 1 of issue 800, bullet 5): neither the quiet rule nor status asks it more than once a minute.
|
||||||
|
const AccountsCacheFor = time.Minute
|
||||||
|
|
||||||
|
// Person is a sender or a recipient, as mesh-mail's event says it.
|
||||||
|
type Person struct {
|
||||||
|
Name string `json:"name,omitempty"`
|
||||||
|
Address string `json:"address"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// EventBody is what mesh-mail's added event says of a message (metadata only; mailevent.go names where it is
|
||||||
|
// said). Unknown fields are ignored: the notifier reads only what it needs.
|
||||||
|
type EventBody struct {
|
||||||
|
Account string `json:"account"`
|
||||||
|
Folder string `json:"folder"`
|
||||||
|
UID uint32 `json:"uid"`
|
||||||
|
UIDValidity uint32 `json:"uidvalidity"`
|
||||||
|
MessageID string `json:"message-id"`
|
||||||
|
Subject string `json:"subject"`
|
||||||
|
From []Person `json:"from"`
|
||||||
|
Flags []string `json:"flags"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// Handle names one message, stably, for the dedupe state (novox/hq ADR 0378 §9: "a redelivered event is not
|
||||||
|
// shown twice"). The message id is the key wherever a message carries one, so a UIDVALIDITY reset — which
|
||||||
|
// changes every UID in a folder — can never make an already-shown mail look new, or a genuinely new one look
|
||||||
|
// already shown; only a message with no message id falls back to its folder, UIDVALIDITY and UID (review
|
||||||
|
// round 1 of issue 800).
|
||||||
|
func (e EventBody) Handle() string {
|
||||||
|
if e.MessageID != "" {
|
||||||
|
return e.Account + "\x1f" + e.MessageID
|
||||||
|
}
|
||||||
|
return e.Account + "\x1f" + e.Folder + "\x1f" +
|
||||||
|
strconv.FormatUint(uint64(e.UIDValidity), 10) + "\x1f" + strconv.FormatUint(uint64(e.UID), 10)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e EventBody) hasFlag(flag string) bool {
|
||||||
|
for _, f := range e.Flags {
|
||||||
|
if strings.EqualFold(f, flag) {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
// sender is how a message's first sender reads in a notice: its name and address, or its address alone.
|
||||||
|
func (e EventBody) sender() string {
|
||||||
|
if len(e.From) == 0 {
|
||||||
|
return "(unknown sender)"
|
||||||
|
}
|
||||||
|
p := e.From[0]
|
||||||
|
if p.Name == "" {
|
||||||
|
return p.Address
|
||||||
|
}
|
||||||
|
return p.Name + " <" + p.Address + ">"
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e EventBody) fromAddress() string {
|
||||||
|
if len(e.From) == 0 {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
return e.From[0].Address
|
||||||
|
}
|
||||||
|
|
||||||
|
// Notice is one notification the notifier shows: its args for node-notifier.send, and the message its `shown`
|
||||||
|
// event names (the latest one, for a folded notice).
|
||||||
|
type Notice struct {
|
||||||
|
Account string
|
||||||
|
Args map[string]any
|
||||||
|
Handles []string // every message this notice accounts for, marked seen once shown
|
||||||
|
Latest EventBody
|
||||||
|
}
|
||||||
|
|
||||||
|
// FailureStatus is what the notifier knows of an account's notice still failing to send (review round 1 of
|
||||||
|
// issue 800): how many attempts, the last error, and since when. Kept in the Store, not only in memory, so
|
||||||
|
// the health tool — its own process, asked separately by the node-engine — sees the same account the live
|
||||||
|
// notifier does.
|
||||||
|
type FailureStatus struct {
|
||||||
|
Count int `json:"count"`
|
||||||
|
LastError string `json:"last-error"`
|
||||||
|
Since time.Time `json:"since"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// Store is what the notifier keeps of its own (the `shown` state the manifest declares): whether a message
|
||||||
|
// was shown already, each account's last notice, and each account's notice still failing to send.
|
||||||
|
type Store interface {
|
||||||
|
HandleSeen(handle string) (bool, error)
|
||||||
|
MarkHandle(handle string) error
|
||||||
|
PutStatus(account string, s AccountStatus) error
|
||||||
|
Statuses() (map[string]AccountStatus, error)
|
||||||
|
PutFailure(account string, f FailureStatus) error
|
||||||
|
ClearFailure(account string) error
|
||||||
|
Failures() (map[string]FailureStatus, error)
|
||||||
|
}
|
||||||
|
|
||||||
|
// AccountStatus is what the notifier last showed of one account.
|
||||||
|
type AccountStatus struct {
|
||||||
|
At time.Time `json:"at"`
|
||||||
|
Summary string `json:"summary"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// accountBuffer is one account's state: history is every qualifying mail's arrival time, kept only for the
|
||||||
|
// trailing Window, that N is counted against (review round 3 of issue 800); immediate holds mail waiting to be
|
||||||
|
// shown one by one at the next Flush; folded holds the account's current fold (its first FoldOver mails within
|
||||||
|
// Window are never folded, so a fold only ever holds the overflow from the fourth qualifying mail of its own
|
||||||
|
// window onward), due as one notice once Window has passed since foldStart, its first mail's arrival.
|
||||||
|
type accountBuffer struct {
|
||||||
|
history []time.Time
|
||||||
|
foldStart time.Time // when the current fold's first mail arrived; zero while nothing is folded
|
||||||
|
immediate []EventBody
|
||||||
|
folded []EventBody
|
||||||
|
}
|
||||||
|
|
||||||
|
// Notifier buffers qualifying mail per account and shows it: the first FoldOver mails of a window one by one,
|
||||||
|
// the rest folded once the window ends. A notice that fails to send is tried again, at most once a minute,
|
||||||
|
// without holding up any other account's notice.
|
||||||
|
type Notifier struct {
|
||||||
|
// Send calls node-notifier.send on this machine.
|
||||||
|
Send func(args map[string]any) error
|
||||||
|
// Accounts reads the accounts the notifier is granted, by mail-store.accounts: each one's own address, for
|
||||||
|
// the quiet rule (novox/hq ADR 0378 §9's LEAD note). An account not known there is shown anyway.
|
||||||
|
Accounts func() (map[string]string, error)
|
||||||
|
Store Store
|
||||||
|
// EmitShown says mesh-mail-notifier's own event `shown`: account and message id, never sender or subject.
|
||||||
|
EmitShown func(account, messageID string) error
|
||||||
|
Now func() time.Time
|
||||||
|
Logf func(string, ...any)
|
||||||
|
|
||||||
|
mu sync.Mutex
|
||||||
|
pending map[string]*accountBuffer
|
||||||
|
buffered map[string]bool // a message's Handle while it is waiting to be shown: never counted twice
|
||||||
|
cooldown map[string]time.Time // account -> not retried before this time, after a failed send
|
||||||
|
|
||||||
|
accountsAt time.Time
|
||||||
|
accountsVal map[string]string
|
||||||
|
accountsErr error
|
||||||
|
}
|
||||||
|
|
||||||
|
func (n *Notifier) init() {
|
||||||
|
n.mu.Lock()
|
||||||
|
defer n.mu.Unlock()
|
||||||
|
if n.pending == nil {
|
||||||
|
n.pending = map[string]*accountBuffer{}
|
||||||
|
n.buffered = map[string]bool{}
|
||||||
|
n.cooldown = map[string]time.Time{}
|
||||||
|
}
|
||||||
|
if n.Now == nil {
|
||||||
|
n.Now = time.Now
|
||||||
|
}
|
||||||
|
if n.Logf == nil {
|
||||||
|
n.Logf = func(string, ...any) {}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Add takes one added event. It returns an error only to have the event offered again (the handler
|
||||||
|
// Subscribe wraps this in follows that rule).
|
||||||
|
func (n *Notifier) Add(e EventBody) error {
|
||||||
|
n.init()
|
||||||
|
handle := e.Handle()
|
||||||
|
seen, err := n.Store.HandleSeen(handle)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if seen {
|
||||||
|
return nil // a redelivery of a message already shown (or already judged not to be)
|
||||||
|
}
|
||||||
|
own, known, err := n.ownAddress(e.Account)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if !n.qualifies(e, own, known) {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
n.mu.Lock()
|
||||||
|
defer n.mu.Unlock()
|
||||||
|
if n.buffered[handle] {
|
||||||
|
return nil // a redelivery of a message still waiting to be shown: never counted twice
|
||||||
|
}
|
||||||
|
now := n.Now()
|
||||||
|
b := n.pending[e.Account]
|
||||||
|
if b == nil {
|
||||||
|
b = &accountBuffer{}
|
||||||
|
n.pending[e.Account] = b
|
||||||
|
}
|
||||||
|
// N is every qualifying mail of this account — shown or folded alike, each counted once — that arrived in
|
||||||
|
// the 60 seconds up to and including now: a true sliding window over arrivals, not a per-session quota that
|
||||||
|
// stays alive, and so keeps folding, for as long as any gap stays under Window (review round 3 of issue
|
||||||
|
// 800, which replaces the round 1 and round 2 readings; nothing in this count depends on whether an earlier
|
||||||
|
// mail has been shown yet).
|
||||||
|
cutoff := now.Add(-Window)
|
||||||
|
history := make([]time.Time, 0, len(b.history)+1)
|
||||||
|
for _, at := range b.history {
|
||||||
|
if !at.Before(cutoff) {
|
||||||
|
history = append(history, at)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
b.history = append(history, now)
|
||||||
|
n.buffered[handle] = true
|
||||||
|
if len(b.history) <= FoldOver {
|
||||||
|
b.immediate = append(b.immediate, e)
|
||||||
|
} else {
|
||||||
|
if len(b.folded) == 0 {
|
||||||
|
b.foldStart = now
|
||||||
|
}
|
||||||
|
b.folded = append(b.folded, e)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// qualifies says whether a mail is shown at all: a new inbox mail, not the account's own, not already read.
|
||||||
|
func (n *Notifier) qualifies(e EventBody, ownAddress string, ownKnown bool) bool {
|
||||||
|
if !strings.EqualFold(e.Folder, Inbox) {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
if e.hasFlag(SeenFlag) {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
if ownKnown && ownAddress != "" && strings.EqualFold(e.fromAddress(), ownAddress) {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
func (n *Notifier) ownAddress(account string) (string, bool, error) {
|
||||||
|
accounts, err := n.readAccounts()
|
||||||
|
if err != nil {
|
||||||
|
// Not known: ADR 0378 §9's LEAD note says the notice is shown when the address cannot be read. Logged,
|
||||||
|
// not retried: mail-store being unreachable is not this event's fault.
|
||||||
|
n.Logf("[mesh-mail-notifier] the account's own address could not be read (%v); the notice is shown", err)
|
||||||
|
return "", false, nil
|
||||||
|
}
|
||||||
|
addr, ok := accounts[account]
|
||||||
|
return addr, ok, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// readAccounts asks mail-store.accounts, reusing its last answer for AccountsCacheFor so neither the quiet
|
||||||
|
// rule nor status reads it more than once a minute (review round 1 of issue 800, bullet 5).
|
||||||
|
func (n *Notifier) readAccounts() (map[string]string, error) {
|
||||||
|
n.mu.Lock()
|
||||||
|
if n.accountsVal != nil && n.Now().Sub(n.accountsAt) < AccountsCacheFor {
|
||||||
|
val, err := n.accountsVal, n.accountsErr
|
||||||
|
n.mu.Unlock()
|
||||||
|
return val, err
|
||||||
|
}
|
||||||
|
n.mu.Unlock()
|
||||||
|
|
||||||
|
val, err := n.Accounts()
|
||||||
|
n.mu.Lock()
|
||||||
|
n.accountsVal, n.accountsErr, n.accountsAt = val, err, n.Now()
|
||||||
|
n.mu.Unlock()
|
||||||
|
return val, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Flush shows what is ready: each account's first FoldOver mails of its window, one by one, as soon as Flush
|
||||||
|
// runs, and the window's folded overflow once the window has run its minute. Accounts are judged in a sorted
|
||||||
|
// order, not a map's iteration order, so the tests do not depend on it (review round 1 of issue 800, BLOCKING
|
||||||
|
// 1). A notice that fails to send leaves its mail pending, tried again once RetryBackoff has passed, and never
|
||||||
|
// holds up another account's notice (BLOCKING 2): nothing is deleted before it is shown.
|
||||||
|
func (n *Notifier) Flush() error {
|
||||||
|
n.init()
|
||||||
|
now := n.Now()
|
||||||
|
|
||||||
|
n.mu.Lock()
|
||||||
|
accounts := make([]string, 0, len(n.pending))
|
||||||
|
for account := range n.pending {
|
||||||
|
accounts = append(accounts, account)
|
||||||
|
}
|
||||||
|
n.mu.Unlock()
|
||||||
|
sort.Strings(accounts)
|
||||||
|
|
||||||
|
for _, account := range accounts {
|
||||||
|
n.flushAccount(account, now)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// flushAccount ships one account's ready notices (its immediate mail, and its fold once Window has passed
|
||||||
|
// since the fold's own first mail), and keeps whatever fails to send pending, under cooldown, for the next
|
||||||
|
// Flush. Nothing is deleted before it is shown (review round 1 BLOCKING 2), and nothing pending is ever lost
|
||||||
|
// (review round 2 BLOCKING, kept under review round 3's rule; see Add).
|
||||||
|
func (n *Notifier) flushAccount(account string, now time.Time) {
|
||||||
|
n.mu.Lock()
|
||||||
|
if cd, ok := n.cooldown[account]; ok && now.Before(cd) {
|
||||||
|
n.mu.Unlock()
|
||||||
|
return // backing off after a failed send; tried again once RetryBackoff has passed
|
||||||
|
}
|
||||||
|
b := n.pending[account]
|
||||||
|
if b == nil {
|
||||||
|
n.mu.Unlock()
|
||||||
|
return
|
||||||
|
}
|
||||||
|
immediate := append([]EventBody(nil), b.immediate...)
|
||||||
|
// The fold is due Window after its own first mail (review round 3 of issue 800).
|
||||||
|
foldDue := len(b.folded) > 0 && now.Sub(b.foldStart) >= Window
|
||||||
|
var fold []EventBody
|
||||||
|
if foldDue {
|
||||||
|
fold = append([]EventBody(nil), b.folded...)
|
||||||
|
}
|
||||||
|
n.mu.Unlock()
|
||||||
|
|
||||||
|
if len(immediate) == 0 && len(fold) == 0 {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
sentImmediate := 0
|
||||||
|
var failErr error
|
||||||
|
for _, e := range immediate {
|
||||||
|
if err := n.show(n.singleNotice(account, e)); err != nil {
|
||||||
|
failErr = err
|
||||||
|
break
|
||||||
|
}
|
||||||
|
sentImmediate++
|
||||||
|
}
|
||||||
|
foldSent := false
|
||||||
|
if failErr == nil && len(fold) > 0 {
|
||||||
|
if err := n.show(n.foldNotice(account, fold, len(fold))); err != nil {
|
||||||
|
failErr = err
|
||||||
|
} else {
|
||||||
|
foldSent = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
n.mu.Lock()
|
||||||
|
if cur := n.pending[account]; cur == b {
|
||||||
|
cur.immediate = cur.immediate[sentImmediate:]
|
||||||
|
if foldSent {
|
||||||
|
cur.folded = cur.folded[len(fold):]
|
||||||
|
if len(cur.folded) == 0 {
|
||||||
|
cur.foldStart = time.Time{} // nothing left folded: a new fold may open (review round 3 of issue 800)
|
||||||
|
} else {
|
||||||
|
// More mail folded in while this batch was being sent: its own Window starts now, counted from
|
||||||
|
// when it was first noticed. Nothing of it is lost either way.
|
||||||
|
cur.foldStart = now
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if failErr != nil {
|
||||||
|
n.cooldown[account] = now.Add(RetryBackoff)
|
||||||
|
n.recordFailureLocked(account, failErr, now)
|
||||||
|
} else {
|
||||||
|
delete(n.cooldown, account)
|
||||||
|
n.clearFailureLocked(account)
|
||||||
|
}
|
||||||
|
n.mu.Unlock()
|
||||||
|
}
|
||||||
|
|
||||||
|
// recordFailureLocked keeps an account's failure in the Store (mu held by the caller, released around the
|
||||||
|
// Store calls since they may reach the bus): its count, its last error, and since when, so a later Flush and
|
||||||
|
// the health tool agree on how long it has been failing.
|
||||||
|
func (n *Notifier) recordFailureLocked(account string, sendErr error, now time.Time) {
|
||||||
|
n.mu.Unlock()
|
||||||
|
defer n.mu.Lock()
|
||||||
|
since := now
|
||||||
|
count := 1
|
||||||
|
if prev, err := n.Store.Failures(); err == nil {
|
||||||
|
if p, ok := prev[account]; ok {
|
||||||
|
since = p.Since
|
||||||
|
count = p.Count + 1
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err := n.Store.PutFailure(account, FailureStatus{Count: count, LastError: sendErr.Error(), Since: since}); err != nil {
|
||||||
|
n.Logf("[mesh-mail-notifier] a failure could not be kept (%v): the health tool may not see it", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (n *Notifier) clearFailureLocked(account string) {
|
||||||
|
n.mu.Unlock()
|
||||||
|
defer n.mu.Lock()
|
||||||
|
if err := n.Store.ClearFailure(account); err != nil {
|
||||||
|
n.Logf("[mesh-mail-notifier] a cleared failure could not be kept cleared: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// singleNotice is one mail shown on its own: one of an account's first FoldOver mails of its window.
|
||||||
|
func (n *Notifier) singleNotice(account string, e EventBody) Notice {
|
||||||
|
return Notice{
|
||||||
|
Account: account,
|
||||||
|
Args: map[string]any{
|
||||||
|
"summary": e.Subject,
|
||||||
|
"body": e.sender() + " — " + account,
|
||||||
|
"app_name": "mesh-mail",
|
||||||
|
},
|
||||||
|
Handles: []string{e.Handle()},
|
||||||
|
Latest: e,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// foldNotice is one notice for everything the account's current fold holds — its qualifying mail from the
|
||||||
|
// fourth within some trailing 60 seconds onward — naming how many mail that fold holds (not the account's
|
||||||
|
// whole history of arrivals) and the latest sender (novox/hq ADR 0378 §9, review round 3 of issue 800, which
|
||||||
|
// replaces the round 1 and round 2 readings: "<k> more new mail(s) in <account>, latest from <sender of the
|
||||||
|
// last>", "mail" for one and "mails" for more).
|
||||||
|
func (n *Notifier) foldNotice(account string, events []EventBody, count int) Notice {
|
||||||
|
latest := events[len(events)-1]
|
||||||
|
notice := Notice{
|
||||||
|
Account: account,
|
||||||
|
Args: map[string]any{
|
||||||
|
"summary": strconv.Itoa(count) + " more new " + mailWord(count) + " in " + account,
|
||||||
|
"body": "latest from " + latest.sender(),
|
||||||
|
"app_name": "mesh-mail",
|
||||||
|
},
|
||||||
|
Latest: latest,
|
||||||
|
}
|
||||||
|
for _, e := range events {
|
||||||
|
notice.Handles = append(notice.Handles, e.Handle())
|
||||||
|
}
|
||||||
|
return notice
|
||||||
|
}
|
||||||
|
|
||||||
|
// mailWord says "mail" for a count of one, "mails" for anything else (review round 2 of issue 800, bullet 3;
|
||||||
|
// kept by review round 3).
|
||||||
|
func mailWord(count int) string {
|
||||||
|
if count == 1 {
|
||||||
|
return "mail"
|
||||||
|
}
|
||||||
|
return "mails"
|
||||||
|
}
|
||||||
|
|
||||||
|
// show sends one notice, marks every message it accounts for seen, keeps the account's status, and emits
|
||||||
|
// `shown`. Nothing is marked, kept or emitted when Send fails: the caller keeps the notice pending instead.
|
||||||
|
func (n *Notifier) show(notice Notice) error {
|
||||||
|
if err := n.Send(notice.Args); err != nil {
|
||||||
|
n.Logf("[mesh-mail-notifier] %s's notice could not be shown: %v", notice.Account, err)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
n.mu.Lock()
|
||||||
|
for _, h := range notice.Handles {
|
||||||
|
delete(n.buffered, h)
|
||||||
|
}
|
||||||
|
n.mu.Unlock()
|
||||||
|
for _, h := range notice.Handles {
|
||||||
|
if err := n.Store.MarkHandle(h); err != nil {
|
||||||
|
n.Logf("[mesh-mail-notifier] shown, but not kept as shown (%v): a redelivery may show it again", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
summary, _ := notice.Args["summary"].(string)
|
||||||
|
if err := n.Store.PutStatus(notice.Account, AccountStatus{At: n.Now(), Summary: summary}); err != nil {
|
||||||
|
n.Logf("[mesh-mail-notifier] shown, but the status could not be kept: %v", err)
|
||||||
|
}
|
||||||
|
return n.EmitShown(notice.Account, notice.Latest.MessageID)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Status is the notifier's account of itself, for mail-notifier.status (novox/hq ADR 0378 §7: "accounts it
|
||||||
|
// announces there, when it last showed a notice"; review round 1 of issue 800: the accounts
|
||||||
|
// mail-store.accounts gives it, and the failures it is retrying).
|
||||||
|
func (n *Notifier) Status() (map[string]any, error) {
|
||||||
|
statuses, err := n.Store.Statuses()
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
failures, err := n.Store.Failures()
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
known, knownErr := n.readAccounts()
|
||||||
|
accounts := map[string]any{}
|
||||||
|
for account := range known {
|
||||||
|
accounts[account] = map[string]any{}
|
||||||
|
}
|
||||||
|
for account, s := range statuses {
|
||||||
|
accounts[account] = map[string]any{"at": s.At.UTC().Format(time.RFC3339), "shown": s.Summary}
|
||||||
|
}
|
||||||
|
out := map[string]any{
|
||||||
|
"accounts": accounts,
|
||||||
|
"rules": []string{
|
||||||
|
"it shows each new inbox mail at most once, through node-notifier.send on this machine (novox/hq ADR 0378 §7)",
|
||||||
|
"quiet during a first sync, for mail from the account's own address, for mail already marked read, and for anything outside the inbox",
|
||||||
|
"of one account's qualifying mail, one arriving when fewer than three of the account's mails fell in the 60 seconds before it is shown at once; the rest is folded and shown, as one notice naming how many it holds and the latest sender, 60 seconds after the fold's own first mail",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
if knownErr != nil {
|
||||||
|
out["accounts-error"] = knownErr.Error()
|
||||||
|
}
|
||||||
|
if len(failures) > 0 {
|
||||||
|
failing := map[string]any{}
|
||||||
|
for account, f := range failures {
|
||||||
|
failing[account] = map[string]any{
|
||||||
|
"count": f.Count,
|
||||||
|
"last-error": f.LastError,
|
||||||
|
"since": f.Since.UTC().Format(time.RFC3339),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
out["failing"] = failing
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,569 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// fakeStore is Store kept in memory, for the test alone.
|
||||||
|
type fakeStore struct {
|
||||||
|
mu sync.Mutex
|
||||||
|
handles map[string]bool
|
||||||
|
statuses map[string]AccountStatus
|
||||||
|
failures map[string]FailureStatus
|
||||||
|
}
|
||||||
|
|
||||||
|
func newFakeStore() *fakeStore {
|
||||||
|
return &fakeStore{
|
||||||
|
handles: map[string]bool{},
|
||||||
|
statuses: map[string]AccountStatus{},
|
||||||
|
failures: map[string]FailureStatus{},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *fakeStore) HandleSeen(handle string) (bool, error) {
|
||||||
|
s.mu.Lock()
|
||||||
|
defer s.mu.Unlock()
|
||||||
|
return s.handles[handle], nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *fakeStore) MarkHandle(handle string) error {
|
||||||
|
s.mu.Lock()
|
||||||
|
defer s.mu.Unlock()
|
||||||
|
s.handles[handle] = true
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *fakeStore) PutStatus(account string, st AccountStatus) error {
|
||||||
|
s.mu.Lock()
|
||||||
|
defer s.mu.Unlock()
|
||||||
|
s.statuses[account] = st
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *fakeStore) Statuses() (map[string]AccountStatus, error) {
|
||||||
|
s.mu.Lock()
|
||||||
|
defer s.mu.Unlock()
|
||||||
|
out := map[string]AccountStatus{}
|
||||||
|
for k, v := range s.statuses {
|
||||||
|
out[k] = v
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *fakeStore) PutFailure(account string, f FailureStatus) error {
|
||||||
|
s.mu.Lock()
|
||||||
|
defer s.mu.Unlock()
|
||||||
|
s.failures[account] = f
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *fakeStore) ClearFailure(account string) error {
|
||||||
|
s.mu.Lock()
|
||||||
|
defer s.mu.Unlock()
|
||||||
|
delete(s.failures, account)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *fakeStore) Failures() (map[string]FailureStatus, error) {
|
||||||
|
s.mu.Lock()
|
||||||
|
defer s.mu.Unlock()
|
||||||
|
out := map[string]FailureStatus{}
|
||||||
|
for k, v := range s.failures {
|
||||||
|
out[k] = v
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// fakeClock is a clock the test moves by hand, so Flush's window needs no real sleep.
|
||||||
|
type fakeClock struct {
|
||||||
|
mu sync.Mutex
|
||||||
|
now time.Time
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *fakeClock) Now() time.Time {
|
||||||
|
c.mu.Lock()
|
||||||
|
defer c.mu.Unlock()
|
||||||
|
return c.now
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *fakeClock) Advance(d time.Duration) {
|
||||||
|
c.mu.Lock()
|
||||||
|
defer c.mu.Unlock()
|
||||||
|
c.now = c.now.Add(d)
|
||||||
|
}
|
||||||
|
|
||||||
|
func newTestNotifier(t *testing.T, clock *fakeClock, store *fakeStore, sent *[]map[string]any) *Notifier {
|
||||||
|
t.Helper()
|
||||||
|
return &Notifier{
|
||||||
|
Now: clock.Now,
|
||||||
|
Send: func(args map[string]any) error {
|
||||||
|
*sent = append(*sent, args)
|
||||||
|
return nil
|
||||||
|
},
|
||||||
|
Accounts: func() (map[string]string, error) { return map[string]string{}, nil },
|
||||||
|
Store: store,
|
||||||
|
EmitShown: func(string, string) error { return nil },
|
||||||
|
Logf: t.Logf,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func mail(account string, uid int, messageID, subject, senderName string) EventBody {
|
||||||
|
return EventBody{Account: account, Folder: "INBOX", UID: uint32(uid), MessageID: messageID, Subject: subject,
|
||||||
|
From: []Person{{Name: senderName, Address: "s@example.com"}}}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestFlushShowsAccountsInASortedOrderNotAMapsIterationOrder is the review's first test for BLOCKING 1: Flush
|
||||||
|
// used to walk a map, so which account's notice went out first was whichever order Go's map iteration gave
|
||||||
|
// that run. Run with -count=50: green every time only once Flush sorts the accounts itself.
|
||||||
|
func TestFlushShowsAccountsInASortedOrderNotAMapsIterationOrder(t *testing.T) {
|
||||||
|
clock := &fakeClock{now: time.Date(2026, 10, 15, 12, 0, 0, 0, time.UTC)}
|
||||||
|
store := newFakeStore()
|
||||||
|
var sent []map[string]any
|
||||||
|
n := newTestNotifier(t, clock, store, &sent)
|
||||||
|
|
||||||
|
for _, account := range []string{"zeta", "mango", "alpha", "delta"} {
|
||||||
|
if err := n.Add(mail(account, 1, account+"-m1", account, "Someone")); err != nil {
|
||||||
|
t.Fatalf("Add(%s): %v", account, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err := n.Flush(); err != nil {
|
||||||
|
t.Fatalf("Flush: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
want := []string{"alpha", "delta", "mango", "zeta"}
|
||||||
|
if len(sent) != len(want) {
|
||||||
|
t.Fatalf("sent %d notice(s), want %d: %+v", len(sent), len(want), sent)
|
||||||
|
}
|
||||||
|
for i, account := range want {
|
||||||
|
if sent[i]["summary"] != account {
|
||||||
|
t.Fatalf("sent[%d]'s summary is %v, want %q: accounts must be flushed in sorted order: %+v",
|
||||||
|
i, sent[i]["summary"], account, sent)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestAFailedSendIsRetriedWithoutLosingItOrHoldingUpOtherAccounts is the review's second test for BLOCKING 2:
|
||||||
|
// a notice node-notifier.send refuses stays pending (nothing is deleted before it is shown), is tried again
|
||||||
|
// at the next Flush no sooner than RetryBackoff, and another account's notice still goes out meanwhile.
|
||||||
|
func TestAFailedSendIsRetriedWithoutLosingItOrHoldingUpOtherAccounts(t *testing.T) {
|
||||||
|
clock := &fakeClock{now: time.Date(2026, 10, 15, 12, 0, 0, 0, time.UTC)}
|
||||||
|
store := newFakeStore()
|
||||||
|
var sent []map[string]any
|
||||||
|
failSend := true
|
||||||
|
n := &Notifier{
|
||||||
|
Now: clock.Now,
|
||||||
|
Send: func(args map[string]any) error {
|
||||||
|
if args["summary"] == "Broken" && failSend {
|
||||||
|
return errors.New("node-notifier.send: no answer")
|
||||||
|
}
|
||||||
|
sent = append(sent, args)
|
||||||
|
return nil
|
||||||
|
},
|
||||||
|
Accounts: func() (map[string]string, error) { return map[string]string{}, nil },
|
||||||
|
Store: store,
|
||||||
|
EmitShown: func(string, string) error { return nil },
|
||||||
|
Logf: t.Logf,
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := n.Add(mail("broke", 1, "b1", "Broken", "Someone")); err != nil {
|
||||||
|
t.Fatalf("Add(broke): %v", err)
|
||||||
|
}
|
||||||
|
if err := n.Add(mail("ok", 1, "o1", "Fine", "Someone")); err != nil {
|
||||||
|
t.Fatalf("Add(ok): %v", err)
|
||||||
|
}
|
||||||
|
if err := n.Flush(); err != nil {
|
||||||
|
t.Fatalf("Flush: %v", err)
|
||||||
|
}
|
||||||
|
if len(sent) != 1 || sent[0]["summary"] != "Fine" {
|
||||||
|
t.Fatalf("sent = %+v, want only ok's notice: broke's failure must not hold it up", sent)
|
||||||
|
}
|
||||||
|
if failures, err := store.Failures(); err != nil || failures["broke"].Count != 1 {
|
||||||
|
t.Fatalf("Store.Failures() = %+v, %v; want broke recorded once", failures, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Before RetryBackoff has passed, broke's notice is not retried yet.
|
||||||
|
clock.Advance(30 * time.Second)
|
||||||
|
if err := n.Flush(); err != nil {
|
||||||
|
t.Fatalf("Flush: %v", err)
|
||||||
|
}
|
||||||
|
if len(sent) != 1 {
|
||||||
|
t.Fatalf("sent %d notice(s) before RetryBackoff elapsed, want 1 (broke not retried yet): %+v", len(sent), sent)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Once RetryBackoff has passed, the next Flush tries it again; nothing was lost.
|
||||||
|
failSend = false
|
||||||
|
clock.Advance(RetryBackoff)
|
||||||
|
if err := n.Flush(); err != nil {
|
||||||
|
t.Fatalf("Flush: %v", err)
|
||||||
|
}
|
||||||
|
if len(sent) != 2 || sent[1]["summary"] != "Broken" {
|
||||||
|
t.Fatalf("sent = %+v, want broke's notice too, after RetryBackoff", sent)
|
||||||
|
}
|
||||||
|
if failures, _ := store.Failures(); len(failures) != 0 {
|
||||||
|
t.Errorf("Store.Failures() = %+v, want it cleared once broke's notice was shown", failures)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestABurstShowsTheFirstThreeAtOnceAndFoldsTheRestAtTheWindowsEnd is the review's third test (BLOCKING 3,
|
||||||
|
// ADR 0378 §9, the lead's reading, the fold notice's wording updated by review round 3 of issue 800): the
|
||||||
|
// first three of an account's qualifying mail within a minute are shown one by one as Flush runs; the fourth
|
||||||
|
// onward is folded and shown as one notice once the window — a minute from the first of them — is over.
|
||||||
|
func TestABurstShowsTheFirstThreeAtOnceAndFoldsTheRestAtTheWindowsEnd(t *testing.T) {
|
||||||
|
clock := &fakeClock{now: time.Date(2026, 10, 15, 12, 0, 0, 0, time.UTC)}
|
||||||
|
store := newFakeStore()
|
||||||
|
var sent []map[string]any
|
||||||
|
n := newTestNotifier(t, clock, store, &sent)
|
||||||
|
|
||||||
|
account := "home"
|
||||||
|
for i := 1; i <= 4; i++ {
|
||||||
|
if err := n.Add(mail(account, i, fmt.Sprintf("m%d", i), fmt.Sprintf("Mail %d", i), fmt.Sprintf("Sender %d", i))); err != nil {
|
||||||
|
t.Fatalf("Add mail %d: %v", i, err)
|
||||||
|
}
|
||||||
|
if err := n.Flush(); err != nil {
|
||||||
|
t.Fatalf("Flush: %v", err)
|
||||||
|
}
|
||||||
|
clock.Advance(10 * time.Second)
|
||||||
|
}
|
||||||
|
if len(sent) != 3 {
|
||||||
|
t.Fatalf("4 mails 10s apart: sent %d notice(s) before the window ends, want 3 shown one by one: %+v", len(sent), sent)
|
||||||
|
}
|
||||||
|
for i, want := range []string{"Mail 1", "Mail 2", "Mail 3"} {
|
||||||
|
if sent[i]["summary"] != want {
|
||||||
|
t.Errorf("sent[%d] = %+v, want summary %q", i, sent[i], want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The window is over a minute after it opened (the first mail): the fourth, folded, is shown now.
|
||||||
|
clock.Advance(Window)
|
||||||
|
if err := n.Flush(); err != nil {
|
||||||
|
t.Fatalf("Flush: %v", err)
|
||||||
|
}
|
||||||
|
if len(sent) != 4 {
|
||||||
|
t.Fatalf("after the window ended, sent %d notice(s), want 4 (the folded fourth mail too): %+v", len(sent), sent)
|
||||||
|
}
|
||||||
|
if sent[3]["summary"] != "1 more new mail in home" || !strings.Contains(sent[3]["body"].(string), "Sender 4") {
|
||||||
|
t.Errorf("the fold notice is %+v, want it to count only what the fold holds (1), naming Sender 4", sent[3])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestARedeliveredEventIsNeverShownOrCountedTwice is the review's fourth test: a redelivery of a mail already
|
||||||
|
// shown is dropped (the Store knows it), and a redelivery of a mail still buffered, waiting to be folded, is
|
||||||
|
// never counted a second time toward the fold.
|
||||||
|
func TestARedeliveredEventIsNeverShownOrCountedTwice(t *testing.T) {
|
||||||
|
clock := &fakeClock{now: time.Date(2026, 10, 15, 12, 0, 0, 0, time.UTC)}
|
||||||
|
store := newFakeStore()
|
||||||
|
var sent []map[string]any
|
||||||
|
n := newTestNotifier(t, clock, store, &sent)
|
||||||
|
|
||||||
|
account := "home"
|
||||||
|
one := mail(account, 1, "m1", "Hi", "Alice")
|
||||||
|
if err := n.Add(one); err != nil {
|
||||||
|
t.Fatalf("Add: %v", err)
|
||||||
|
}
|
||||||
|
if err := n.Flush(); err != nil {
|
||||||
|
t.Fatalf("Flush: %v", err)
|
||||||
|
}
|
||||||
|
if len(sent) != 1 {
|
||||||
|
t.Fatalf("sent %d notice(s), want 1", len(sent))
|
||||||
|
}
|
||||||
|
if err := n.Add(one); err != nil { // redelivered after it was shown
|
||||||
|
t.Fatalf("Add (redelivered, already shown): %v", err)
|
||||||
|
}
|
||||||
|
if err := n.Flush(); err != nil {
|
||||||
|
t.Fatalf("Flush: %v", err)
|
||||||
|
}
|
||||||
|
if len(sent) != 1 {
|
||||||
|
t.Fatalf("a redelivered, already-shown event was shown again: sent now %+v", sent)
|
||||||
|
}
|
||||||
|
|
||||||
|
for i := 2; i <= 4; i++ {
|
||||||
|
if err := n.Add(mail(account, i, fmt.Sprintf("m%d", i), "Hi", fmt.Sprintf("Sender %d", i))); err != nil {
|
||||||
|
t.Fatalf("Add mail %d: %v", i, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
fourth := mail(account, 4, "m4", "Hi", "Sender 4")
|
||||||
|
if err := n.Add(fourth); err != nil { // redelivered while still buffered, folded, not yet shown
|
||||||
|
t.Fatalf("Add (redelivered, buffered): %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
clock.Advance(Window)
|
||||||
|
if err := n.Flush(); err != nil {
|
||||||
|
t.Fatalf("Flush: %v", err)
|
||||||
|
}
|
||||||
|
if len(sent) != 4 {
|
||||||
|
t.Fatalf("sent %d notice(s), want 4 (one, mail 2, mail 3, and one fold notice): %+v", len(sent), sent)
|
||||||
|
}
|
||||||
|
if sent[3]["summary"] != "1 more new mail in home" {
|
||||||
|
t.Errorf("the fold notice is %+v, want it to count the redelivered fourth mail once, not twice, among the fold's 1", sent[3])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestDedupeKeyIsTheMessageIDAcrossAUIDVALIDITYChange is the review's fifth test: the dedupe key is the
|
||||||
|
// account and the message id wherever one is given, so a UIDVALIDITY reset — which changes every UID in a
|
||||||
|
// folder — cannot make an already-shown mail look new. Only a message with no message id falls back to its
|
||||||
|
// folder, UIDVALIDITY and UID, and that fallback still tells two different mails apart.
|
||||||
|
func TestDedupeKeyIsTheMessageIDAcrossAUIDVALIDITYChange(t *testing.T) {
|
||||||
|
clock := &fakeClock{now: time.Date(2026, 10, 15, 12, 0, 0, 0, time.UTC)}
|
||||||
|
store := newFakeStore()
|
||||||
|
var sent []map[string]any
|
||||||
|
n := newTestNotifier(t, clock, store, &sent)
|
||||||
|
|
||||||
|
account := "home"
|
||||||
|
first := EventBody{Account: account, Folder: "INBOX", UID: 5, UIDValidity: 1, MessageID: "stable-id",
|
||||||
|
Subject: "Hi", From: []Person{{Address: "a@example.com"}}}
|
||||||
|
if err := n.Add(first); err != nil {
|
||||||
|
t.Fatalf("Add: %v", err)
|
||||||
|
}
|
||||||
|
if err := n.Flush(); err != nil {
|
||||||
|
t.Fatalf("Flush: %v", err)
|
||||||
|
}
|
||||||
|
if len(sent) != 1 {
|
||||||
|
t.Fatalf("sent %d, want 1", len(sent))
|
||||||
|
}
|
||||||
|
|
||||||
|
// The same message, redelivered after a UIDVALIDITY reset changed its UID: same message id, not shown again.
|
||||||
|
resent := EventBody{Account: account, Folder: "INBOX", UID: 99, UIDValidity: 2, MessageID: "stable-id",
|
||||||
|
Subject: "Hi", From: []Person{{Address: "a@example.com"}}}
|
||||||
|
if err := n.Add(resent); err != nil {
|
||||||
|
t.Fatalf("Add (UIDVALIDITY reset, same message id): %v", err)
|
||||||
|
}
|
||||||
|
if err := n.Flush(); err != nil {
|
||||||
|
t.Fatalf("Flush: %v", err)
|
||||||
|
}
|
||||||
|
if len(sent) != 1 {
|
||||||
|
t.Fatalf("a mail with the same message id after a UIDVALIDITY reset was shown again: sent now %+v", sent)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Two different mails with no message id, at the same UID but a different UIDVALIDITY, are not confused.
|
||||||
|
noID1 := EventBody{Account: account, Folder: "INBOX", UID: 7, UIDValidity: 1, Subject: "No id",
|
||||||
|
From: []Person{{Address: "a@example.com"}}}
|
||||||
|
noID2 := EventBody{Account: account, Folder: "INBOX", UID: 7, UIDValidity: 2, Subject: "No id, new validity",
|
||||||
|
From: []Person{{Address: "a@example.com"}}}
|
||||||
|
if err := n.Add(noID1); err != nil {
|
||||||
|
t.Fatalf("Add noID1: %v", err)
|
||||||
|
}
|
||||||
|
if err := n.Flush(); err != nil {
|
||||||
|
t.Fatalf("Flush: %v", err)
|
||||||
|
}
|
||||||
|
if err := n.Add(noID2); err != nil {
|
||||||
|
t.Fatalf("Add noID2: %v", err)
|
||||||
|
}
|
||||||
|
if err := n.Flush(); err != nil {
|
||||||
|
t.Fatalf("Flush: %v", err)
|
||||||
|
}
|
||||||
|
if len(sent) != 3 {
|
||||||
|
t.Fatalf("sent %d, want 3: noID1 and noID2 are different mails, their UIDVALIDITY differs: %+v", len(sent), sent)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestQualifyingRulesStillHold keeps the original qualifying rules proven (an already-read mail, a mail from
|
||||||
|
// the account's own address, and a mail outside the inbox are never shown), now that buffering is per window
|
||||||
|
// rather than per quiet period.
|
||||||
|
func TestQualifyingRulesStillHold(t *testing.T) {
|
||||||
|
clock := &fakeClock{now: time.Date(2026, 10, 15, 12, 0, 0, 0, time.UTC)}
|
||||||
|
store := newFakeStore()
|
||||||
|
var sent []map[string]any
|
||||||
|
n := &Notifier{
|
||||||
|
Now: clock.Now,
|
||||||
|
Send: func(args map[string]any) error { sent = append(sent, args); return nil },
|
||||||
|
Accounts: func() (map[string]string, error) { return map[string]string{"work": "me@work.example"}, nil },
|
||||||
|
Store: store,
|
||||||
|
EmitShown: func(string, string) error { return nil },
|
||||||
|
Logf: t.Logf,
|
||||||
|
}
|
||||||
|
already := EventBody{Account: "work", Folder: "INBOX", UID: 2, MessageID: "m2", Subject: "Old",
|
||||||
|
From: []Person{{Address: "bob@example.com"}}, Flags: []string{"\\Seen"}}
|
||||||
|
own := EventBody{Account: "work", Folder: "INBOX", UID: 3, MessageID: "m3", Subject: "Sent",
|
||||||
|
From: []Person{{Address: "me@work.example"}}}
|
||||||
|
elsewhere := EventBody{Account: "work", Folder: "Archive", UID: 4, MessageID: "m4", Subject: "Filed",
|
||||||
|
From: []Person{{Address: "carol@example.com"}}}
|
||||||
|
for _, e := range []EventBody{already, own, elsewhere} {
|
||||||
|
if err := n.Add(e); err != nil {
|
||||||
|
t.Fatalf("Add(%+v): %v", e, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err := n.Flush(); err != nil {
|
||||||
|
t.Fatalf("Flush: %v", err)
|
||||||
|
}
|
||||||
|
if len(sent) != 0 {
|
||||||
|
t.Fatalf("sent %+v, want nothing: an already-read mail, the account's own mail and mail outside the inbox never qualify", sent)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestABufferIsNeverReplacedWhileItHoldsAnUnsentMail is review round 2 of issue 800's first test (BLOCKING):
|
||||||
|
// a send that fails keeps its mail pending; once the cooldown has passed, but before the next Flush retries
|
||||||
|
// it, a new qualifying mail arrives. Neither mail is lost: both are shown once the retry succeeds.
|
||||||
|
func TestABufferIsNeverReplacedWhileItHoldsAnUnsentMail(t *testing.T) {
|
||||||
|
clock := &fakeClock{now: time.Date(2026, 10, 15, 12, 0, 0, 0, time.UTC)}
|
||||||
|
store := newFakeStore()
|
||||||
|
var sent []map[string]any
|
||||||
|
failSend := true
|
||||||
|
n := &Notifier{
|
||||||
|
Now: clock.Now,
|
||||||
|
Send: func(args map[string]any) error {
|
||||||
|
if failSend {
|
||||||
|
return errors.New("node-notifier.send: no answer")
|
||||||
|
}
|
||||||
|
sent = append(sent, args)
|
||||||
|
return nil
|
||||||
|
},
|
||||||
|
Accounts: func() (map[string]string, error) { return map[string]string{}, nil },
|
||||||
|
Store: store,
|
||||||
|
EmitShown: func(string, string) error { return nil },
|
||||||
|
Logf: t.Logf,
|
||||||
|
}
|
||||||
|
|
||||||
|
account := "home"
|
||||||
|
if err := n.Add(mail(account, 1, "m1", "Mail 1", "Sender 1")); err != nil {
|
||||||
|
t.Fatalf("Add mail 1: %v", err)
|
||||||
|
}
|
||||||
|
if err := n.Flush(); err != nil {
|
||||||
|
t.Fatalf("Flush: %v", err)
|
||||||
|
}
|
||||||
|
if len(sent) != 0 {
|
||||||
|
t.Fatalf("sent %d notice(s), want 0: the send failed", len(sent))
|
||||||
|
}
|
||||||
|
|
||||||
|
// The cooldown passes, but no retry has happened yet (Flush is not called again before the new mail
|
||||||
|
// arrives): a buffer this old must not be thrown away for a fresh one.
|
||||||
|
clock.Advance(RetryBackoff)
|
||||||
|
if err := n.Add(mail(account, 2, "m2", "Mail 2", "Sender 2")); err != nil {
|
||||||
|
t.Fatalf("Add mail 2: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
failSend = false
|
||||||
|
if err := n.Flush(); err != nil {
|
||||||
|
t.Fatalf("Flush: %v", err)
|
||||||
|
}
|
||||||
|
if len(sent) != 2 {
|
||||||
|
t.Fatalf("sent %d notice(s), want 2 (mail 1's retry and mail 2, neither lost): %+v", len(sent), sent)
|
||||||
|
}
|
||||||
|
for i, want := range []string{"Mail 1", "Mail 2"} {
|
||||||
|
if sent[i]["summary"] != want {
|
||||||
|
t.Errorf("sent[%d] = %+v, want summary %q", i, sent[i], want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestTheFoldFollowsATrueSlidingWindowOfArrivalsNotASessionQuota is review round 3 of issue 800's first test,
|
||||||
|
// replacing round 2's (which contradicted round 3's rule): N, for a mail arriving at t, is every qualifying
|
||||||
|
// mail of the account — shown or folded alike — that arrived in the 60 seconds up to and including t. Mails at
|
||||||
|
// 0, 50, 55, 70, 75 and 80 seconds: at 70s the preceding 60 seconds (from 10s) hold 50, 55 and 70 themselves
|
||||||
|
// (0s has aged out), N=3, so 70 is still shown at once, not folded; at 75s the preceding 60 seconds (from 15s)
|
||||||
|
// still hold 50 too, so it, 55, 70 and 75 themselves make N=4, and 75 folds; 80 folds with it (its own
|
||||||
|
// preceding 60 seconds hold 50, 55, 70, 75 and 80, N=5). The fold is due 60 seconds after its own first mail
|
||||||
|
// (75s), at 135s, as one notice naming only what it holds (2) and the latest sender (80s's).
|
||||||
|
func TestTheFoldFollowsATrueSlidingWindowOfArrivalsNotASessionQuota(t *testing.T) {
|
||||||
|
clock := &fakeClock{now: time.Date(2026, 10, 15, 12, 0, 0, 0, time.UTC)}
|
||||||
|
store := newFakeStore()
|
||||||
|
var sent []map[string]any
|
||||||
|
n := newTestNotifier(t, clock, store, &sent)
|
||||||
|
|
||||||
|
account := "home"
|
||||||
|
deltas := []time.Duration{0, 50 * time.Second, 5 * time.Second, 15 * time.Second, 5 * time.Second, 5 * time.Second}
|
||||||
|
for i, d := range deltas {
|
||||||
|
clock.Advance(d)
|
||||||
|
if err := n.Add(mail(account, i+1, fmt.Sprintf("m%d", i+1), fmt.Sprintf("Mail %d", i+1), fmt.Sprintf("Sender %d", i+1))); err != nil {
|
||||||
|
t.Fatalf("Add mail %d: %v", i+1, err)
|
||||||
|
}
|
||||||
|
if err := n.Flush(); err != nil {
|
||||||
|
t.Fatalf("Flush at mail %d: %v", i+1, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(sent) != 4 {
|
||||||
|
t.Fatalf("sent %d notice(s), want 4 shown one by one (0s, 50s, 55s, 70s): %+v", len(sent), sent)
|
||||||
|
}
|
||||||
|
for i, want := range []string{"Mail 1", "Mail 2", "Mail 3", "Mail 4"} {
|
||||||
|
if sent[i]["summary"] != want {
|
||||||
|
t.Errorf("sent[%d] = %+v, want summary %q", i, sent[i], want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The fold's own window — a minute from its first mail, 75s — ends at 135s.
|
||||||
|
clock.Advance(55 * time.Second) // now at 80s + 55s = 135s
|
||||||
|
if err := n.Flush(); err != nil {
|
||||||
|
t.Fatalf("Flush: %v", err)
|
||||||
|
}
|
||||||
|
if len(sent) != 5 {
|
||||||
|
t.Fatalf("sent %d notice(s), want 5 (75s and 80s folded into one, at 135s): %+v", len(sent), sent)
|
||||||
|
}
|
||||||
|
if sent[4]["summary"] != "2 more new mails in home" || !strings.Contains(sent[4]["body"].(string), "Sender 6") {
|
||||||
|
t.Errorf("the fold notice is %+v, want it to count only what it holds (2), naming Sender 6 (80s, the latest)", sent[4])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestASteadyTrickleNeverFoldsForever is review round 3 of issue 800's second test: the bug the round found
|
||||||
|
// (a session that only resets after a quiet minute folds a steady stream forever, even without a burst). One
|
||||||
|
// mail every 50 seconds, 20 of them: each one's preceding 60 seconds holds only the one before it, so N never
|
||||||
|
// exceeds 2, and all 20 are shown at once — none ever fold.
|
||||||
|
func TestASteadyTrickleNeverFoldsForever(t *testing.T) {
|
||||||
|
clock := &fakeClock{now: time.Date(2026, 10, 15, 12, 0, 0, 0, time.UTC)}
|
||||||
|
store := newFakeStore()
|
||||||
|
var sent []map[string]any
|
||||||
|
n := newTestNotifier(t, clock, store, &sent)
|
||||||
|
|
||||||
|
account := "home"
|
||||||
|
for i := 1; i <= 20; i++ {
|
||||||
|
if i > 1 {
|
||||||
|
clock.Advance(50 * time.Second)
|
||||||
|
}
|
||||||
|
if err := n.Add(mail(account, i, fmt.Sprintf("m%d", i), fmt.Sprintf("Mail %d", i), fmt.Sprintf("Sender %d", i))); err != nil {
|
||||||
|
t.Fatalf("Add mail %d: %v", i, err)
|
||||||
|
}
|
||||||
|
if err := n.Flush(); err != nil {
|
||||||
|
t.Fatalf("Flush at mail %d: %v", i, err)
|
||||||
|
}
|
||||||
|
if len(sent) != i {
|
||||||
|
t.Fatalf("after mail %d, sent %d notice(s), want %d: a steady trickle must never fold: %+v", i, len(sent), i, sent)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for i, got := range sent {
|
||||||
|
want := fmt.Sprintf("Mail %d", i+1)
|
||||||
|
if got["summary"] != want {
|
||||||
|
t.Errorf("sent[%d] = %+v, want summary %q", i, got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestFiveMailsWithinTenSecondsFoldAfterThree is review round 3 of issue 800's third test: five mails within
|
||||||
|
// ten seconds — the first three shown at once, the fourth opens a fold, the fifth joins it, and the fold is
|
||||||
|
// shown 60 seconds after the fourth (the fold's own first mail) as one notice.
|
||||||
|
func TestFiveMailsWithinTenSecondsFoldAfterThree(t *testing.T) {
|
||||||
|
clock := &fakeClock{now: time.Date(2026, 10, 15, 12, 0, 0, 0, time.UTC)}
|
||||||
|
store := newFakeStore()
|
||||||
|
var sent []map[string]any
|
||||||
|
n := newTestNotifier(t, clock, store, &sent)
|
||||||
|
|
||||||
|
account := "home"
|
||||||
|
for i := 1; i <= 5; i++ {
|
||||||
|
if i > 1 {
|
||||||
|
clock.Advance(2 * time.Second)
|
||||||
|
}
|
||||||
|
if err := n.Add(mail(account, i, fmt.Sprintf("m%d", i), fmt.Sprintf("Mail %d", i), fmt.Sprintf("Sender %d", i))); err != nil {
|
||||||
|
t.Fatalf("Add mail %d: %v", i, err)
|
||||||
|
}
|
||||||
|
if err := n.Flush(); err != nil {
|
||||||
|
t.Fatalf("Flush at mail %d: %v", i, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(sent) != 3 {
|
||||||
|
t.Fatalf("sent %d notice(s), want 3 shown one by one, the fourth and fifth folded, not yet due: %+v", len(sent), sent)
|
||||||
|
}
|
||||||
|
|
||||||
|
// The fold is due 60 seconds after its own first mail (the fourth, at 6s).
|
||||||
|
clock.Advance(Window)
|
||||||
|
if err := n.Flush(); err != nil {
|
||||||
|
t.Fatalf("Flush: %v", err)
|
||||||
|
}
|
||||||
|
if len(sent) != 4 {
|
||||||
|
t.Fatalf("sent %d notice(s), want 4 (one fold notice carrying the fourth and fifth): %+v", len(sent), sent)
|
||||||
|
}
|
||||||
|
if sent[3]["summary"] != "2 more new mails in home" || !strings.Contains(sent[3]["body"].(string), "Sender 5") {
|
||||||
|
t.Errorf("the fold notice is %+v, want it to count only what it holds (2), naming Sender 5 (the latest)", sent[3])
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,114 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"io"
|
||||||
|
"strings"
|
||||||
|
"sync/atomic"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||||
|
)
|
||||||
|
|
||||||
|
// TestRunStartsTheWatcherEvenWhenStdinIsAtEOF proves the watcher runs beside stdio.Serve in the one process the
|
||||||
|
// node's runtime launches (novox/hq ADR 0378 §9, as mesh-knowledge's serve starts go hear(k) and go
|
||||||
|
// background(k) before stdio.Serve), not only in a separate process something keeps asking over stdin. run is
|
||||||
|
// given stdin already at EOF, exactly what systemd gave the old separate `notify` process: it exited at once,
|
||||||
|
// every five seconds (novox/hq issue 859). The watcher must still be running once run has returned — proved by
|
||||||
|
// making it actually answer after run returns, not by the absence of a signal that nothing but the test's own
|
||||||
|
// cleanup could ever raise.
|
||||||
|
func TestRunStartsTheWatcherEvenWhenStdinIsAtEOF(t *testing.T) {
|
||||||
|
started := make(chan struct{})
|
||||||
|
ping := make(chan struct{})
|
||||||
|
pong := make(chan struct{})
|
||||||
|
|
||||||
|
fakeWatcher := func(n *Notifier) {
|
||||||
|
close(started)
|
||||||
|
for range ping {
|
||||||
|
pong <- struct{}{}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
n := &Notifier{Store: newFakeStore()}
|
||||||
|
n.init()
|
||||||
|
|
||||||
|
done := make(chan error, 1)
|
||||||
|
go func() { done <- run(n, fakeWatcher, strings.NewReader(""), io.Discard, nil) }()
|
||||||
|
|
||||||
|
select {
|
||||||
|
case <-started:
|
||||||
|
case <-time.After(time.Second):
|
||||||
|
t.Fatal("the watcher was not started before stdio.Serve returned")
|
||||||
|
}
|
||||||
|
|
||||||
|
select {
|
||||||
|
case err := <-done:
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("run over stdin at EOF: %v", err)
|
||||||
|
}
|
||||||
|
case <-time.After(time.Second):
|
||||||
|
t.Fatal("run did not return once stdin reached EOF, as systemd gives a process nobody talks to")
|
||||||
|
}
|
||||||
|
|
||||||
|
// run has returned (stdio.Serve hit EOF at once); the watcher must still be live and answering, not merely
|
||||||
|
// not yet torn down. A watcher that stopped when run returned would never take this ping.
|
||||||
|
select {
|
||||||
|
case ping <- struct{}{}:
|
||||||
|
case <-time.After(time.Second):
|
||||||
|
t.Fatal("the watcher is no longer running once run returned: it must not stop just because stdio.Serve did")
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case <-pong:
|
||||||
|
case <-time.After(time.Second):
|
||||||
|
t.Fatal("the watcher did not answer once run returned")
|
||||||
|
}
|
||||||
|
close(ping)
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestWatchRetriesASubscribeRefusalWithBackoffAndNeverExits proves watch no longer exits the process on a
|
||||||
|
// subscribe the runtime refuses — the old os.Exit(1) took mail-notifier.status and mail_notifier_health down
|
||||||
|
// with it — but retries with backoff and logs each refusal, as modules/mesh-knowledge's hear does (novox/hq
|
||||||
|
// issue 859, review round 1). A subscribe that fails twice then succeeds must leave the process serving (run
|
||||||
|
// still returns normally over stdin at EOF, never os.Exit) and the watcher subscribed (its third, successful
|
||||||
|
// call).
|
||||||
|
func TestWatchRetriesASubscribeRefusalWithBackoffAndNeverExits(t *testing.T) {
|
||||||
|
var calls int32
|
||||||
|
subscribed := make(chan struct{})
|
||||||
|
fakeSubscribe := func(pattern string, handle func(stdio.Envelope) error) error {
|
||||||
|
if atomic.AddInt32(&calls, 1) <= 2 {
|
||||||
|
return errors.New("the runtime refused this subscribe")
|
||||||
|
}
|
||||||
|
close(subscribed)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
noSleep := func(time.Duration) {}
|
||||||
|
|
||||||
|
n := &Notifier{Store: newFakeStore()}
|
||||||
|
n.init()
|
||||||
|
|
||||||
|
done := make(chan error, 1)
|
||||||
|
go func() {
|
||||||
|
done <- run(n, func(n *Notifier) { watchWith(n, fakeSubscribe, noSleep) },
|
||||||
|
strings.NewReader(""), io.Discard, nil)
|
||||||
|
}()
|
||||||
|
|
||||||
|
select {
|
||||||
|
case err := <-done:
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("run over stdin at EOF: %v", err)
|
||||||
|
}
|
||||||
|
case <-time.After(time.Second):
|
||||||
|
t.Fatal("run did not return once stdin reached EOF: a subscribe refusal must not hold up or kill the process")
|
||||||
|
}
|
||||||
|
|
||||||
|
select {
|
||||||
|
case <-subscribed:
|
||||||
|
case <-time.After(time.Second):
|
||||||
|
t.Fatal("the watcher never subscribed after its first two refusals")
|
||||||
|
}
|
||||||
|
|
||||||
|
if got := atomic.LoadInt32(&calls); got != 3 {
|
||||||
|
t.Fatalf("subscribe was called %d time(s), want 3 (two refusals, then the one that succeeded)", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,17 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import "testing"
|
||||||
|
|
||||||
|
// TestSendKeyNamesThisMachine proves Send asks its own machine's node-notifier, not the bare seat: a bare
|
||||||
|
// "seat:node-notifier.send" is answered by whichever machine's node-notifier takes it first, so notices sent
|
||||||
|
// from g14 and shanks were lost to "503 no responders" or another machine's desktop (novox/hq issue 865).
|
||||||
|
// desk-channel names the machine it asks the same way ("seat:node-notifier.send@"+machine,
|
||||||
|
// modules/desk-channel/cmd/desk-channel/main.go); sendKey must read this machine's own name the same way the
|
||||||
|
// SDK's own Emit does, from MESH_NODE, which the runtime sets.
|
||||||
|
func TestSendKeyNamesThisMachine(t *testing.T) {
|
||||||
|
t.Setenv("MESH_NODE", "g14")
|
||||||
|
|
||||||
|
if got, want := sendKey(), "seat:node-notifier.send@g14"; got != want {
|
||||||
|
t.Fatalf("sendKey() = %q, want %q (it must name this machine, as desk-channel does)", got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,5 @@
|
|||||||
|
module mesh-mail-notifier
|
||||||
|
|
||||||
|
go 1.22
|
||||||
|
|
||||||
|
require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261013182924-5933eb757d44
|
||||||
@@ -0,0 +1,2 @@
|
|||||||
|
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261013182924-5933eb757d44 h1:r5xIo5qWOXQdCSMxvqJy8kV4U+P/QWXFK6971Ha6wU8=
|
||||||
|
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261013182924-5933eb757d44/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY=
|
||||||
@@ -0,0 +1,67 @@
|
|||||||
|
{
|
||||||
|
"module": "mesh-mail-notifier",
|
||||||
|
"version": "1",
|
||||||
|
"runs-as": "mesh-mail-notifier",
|
||||||
|
"claims": [
|
||||||
|
{
|
||||||
|
"name": "mail-notifier",
|
||||||
|
"scope": "node",
|
||||||
|
"serves": ["status"]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"consumes": [
|
||||||
|
"mesh-mail.mail.*.message.added"
|
||||||
|
],
|
||||||
|
"invokes": [
|
||||||
|
"seat:node-notifier.send",
|
||||||
|
"seat:mail-store.accounts"
|
||||||
|
],
|
||||||
|
"emits": [
|
||||||
|
"shown"
|
||||||
|
],
|
||||||
|
"state": [
|
||||||
|
{
|
||||||
|
"name": "shown",
|
||||||
|
"ttl-seconds": 2592000
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"tools": [
|
||||||
|
"mail-notifier.status",
|
||||||
|
"mail_notifier_health"
|
||||||
|
],
|
||||||
|
"own-secrets": {
|
||||||
|
"broker": "${dir:state}/broker"
|
||||||
|
},
|
||||||
|
"secrets-owner": "mesh-mail-notifier",
|
||||||
|
"resources": [
|
||||||
|
{
|
||||||
|
"id": "account",
|
||||||
|
"type": "user",
|
||||||
|
"name": "mesh-mail-notifier",
|
||||||
|
"shell": "/usr/bin/nologin",
|
||||||
|
"home": "/var/lib/mesh-mail-notifier"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "state",
|
||||||
|
"type": "directory",
|
||||||
|
"mode": "0700",
|
||||||
|
"owner": "mesh-mail-notifier",
|
||||||
|
"place": "."
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"build": {
|
||||||
|
"artifacts": [
|
||||||
|
{
|
||||||
|
"name": "tools",
|
||||||
|
"kind": "bundle",
|
||||||
|
"language": "go",
|
||||||
|
"system": "arch",
|
||||||
|
"from": "cmd/mesh-mail-notifier",
|
||||||
|
"binary": "mesh-mail-notifier",
|
||||||
|
"loads": [
|
||||||
|
"mesh-mail-notifier"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,81 @@
|
|||||||
|
# mesh-mail — the operator's mail accounts, watched and said on the bus
|
||||||
|
|
||||||
|
One module holds the operator's mail accounts, watches them, and says each change of a message on the bus for the
|
||||||
|
modules the operator grants it to (novox/hq ADR 0359, research 049 document 02). One instance, on the control-node.
|
||||||
|
Reading is the first stage: accounts of the kind `imap`, read-only — nothing here flags, moves, deletes or drafts.
|
||||||
|
Sending is `mail_send` alone (novox/hq ADR 0361 §2): it sends nothing itself, and reaches an account's outgoing
|
||||||
|
server only once the operator approves it on a channel that proves who answered.
|
||||||
|
|
||||||
|
## Its seats (novox/hq ADR 0378 as amended by ADR 0379, issue 800 part A)
|
||||||
|
|
||||||
|
It declares three mail seats and claims the one it holds:
|
||||||
|
|
||||||
|
- **mail-store** (held once for the mesh) — serves `accounts`, `search`, `read` and `send`; mesh-mail claims it,
|
||||||
|
through the same handlers as its own `mail_*` tools. The catalogue's own check (`checks/mailstore`) refuses any
|
||||||
|
other manifest that claims it.
|
||||||
|
- **mail-webclient** (held once for the mesh) — serves `open` and `status`; unheld until issue 580.
|
||||||
|
- **mail-notifier** (held once per machine, needing `node-notifier` there) — serves `status`; held by
|
||||||
|
mesh-mail-notifier (issue 800 part B), a module of its own built from this one's source, which tells the
|
||||||
|
operator of each new inbox mail through `node-notifier.send`.
|
||||||
|
|
||||||
|
## What it does
|
||||||
|
|
||||||
|
- **Watches.** For each account it keeps one connection: IDLE on the inbox, ended and begun again every two minutes
|
||||||
|
(which proves the server still answers), and a look at every other folder every five minutes. Folders are opened
|
||||||
|
read-only and bodies are peeked, so watching never marks a message seen.
|
||||||
|
- **Indexes.** One row per message and folder in its own database (`postgres-database`): metadata, flags, the text
|
||||||
|
for search, and the attachments' references. Each attachment is kept once, by its SHA-256, in its bucket
|
||||||
|
(`s3-bucket`, from `minio`).
|
||||||
|
- **Says each change.** A message added, its flags changed, or removed is said as the event
|
||||||
|
`mail.<account>.message.<added|changed|removed>`: account, folder, uid, message id, date, sender, recipients,
|
||||||
|
subject, size, flags and attachment references; never the body, never an attachment's bytes. The events go to the
|
||||||
|
module's own stream `MAIL`, kept seven days, never to the shared EVENTS stream (novox/hq ADR 0363); a module hears an
|
||||||
|
account's part only when its `consumes` names mesh-mail and the setting `grants` names it for that account. The first look at a
|
||||||
|
folder indexes what is there and says nothing: the bus hears what changed since the watch began. The keeper writes
|
||||||
|
each event in the same step as the change it says; the bundle says it through the runtime and removes it once the
|
||||||
|
bus took it.
|
||||||
|
|
||||||
|
## Its settings, both trusted
|
||||||
|
|
||||||
|
`accounts` — per account name (a lower-case token, which names the account on the bus and its secret):
|
||||||
|
|
||||||
|
```json
|
||||||
|
{"mesh": {"kind": "imap", "address": "<the address>",
|
||||||
|
"incoming": {"host": "<the IMAP server>", "port": 993, "security": "tls"},
|
||||||
|
"outgoing": {"host": "<the SMTP server>", "port": 465, "security": "tls"},
|
||||||
|
"folders": ["optional: the folders watched besides the inbox; every folder when left out"]}}
|
||||||
|
```
|
||||||
|
|
||||||
|
`outgoing` is the server `mail_send` sends through: host, port, and `security` `tls` (465, implicit) or `starttls`
|
||||||
|
(587). A second `imap` account on another server is one more entry and one more password; no code changes. `user`
|
||||||
|
signs in under another name than the address.
|
||||||
|
|
||||||
|
`grants` — per account, who may read its mail, and who may send from it: a module's name (on every machine),
|
||||||
|
`<machine>.<module>`, or `person.<name>`. Nobody by default. It restricts the stream `MAIL` on the bus (a person's
|
||||||
|
entry grants nothing there) and the tools `mail_search`, `mail_read` and `mail_send` — the same check for all three.
|
||||||
|
|
||||||
|
The password of an account is the own secret `mail-credential-<account>`, given at the desk; the mesh never makes one.
|
||||||
|
|
||||||
|
## Its tools
|
||||||
|
|
||||||
|
| Tool | Answers |
|
||||||
|
|---|---|
|
||||||
|
| `mail_accounts` | each account's kind, state and why, when its server last answered, its folders; the address only to a caller granted it |
|
||||||
|
| `mail_search` | messages of the accounts the caller is granted, by every word of a query, newest first, no body |
|
||||||
|
| `mail_read` | one message's metadata and text, or one attachment's bytes, to a caller granted its account |
|
||||||
|
| `mail_health` | healthy; unhealthy naming the account and why (connection down, or no answer for five minutes); waits naming `mail-credential-<account>` while its password is not given |
|
||||||
|
| `mail_send` | raises an ask on the operator channel (the account, the recipients, the subject, the first lines) for a caller granted the account; answers at once with the ask's id. Sends nothing itself |
|
||||||
|
| `mail_send_status` | where a send stands, by its ask id: waiting, sent, declined, ended without a choice, or not proven |
|
||||||
|
|
||||||
|
The caller is the runtime's word, never an argument. `mail_send` sends through the account's outgoing server only
|
||||||
|
once the operator's Approve comes back on a channel that proves who answered, and once: a warrant delivered twice
|
||||||
|
never sends twice (novox/hq ADR 0361 §2).
|
||||||
|
|
||||||
|
The mail-store seat's four verbs, `mail-store.accounts`, `mail-store.search`, `mail-store.read` and
|
||||||
|
`mail-store.send`, answer through the same handlers as `mail_accounts`, `mail_search`, `mail_read` and `mail_send`
|
||||||
|
above — the same grants, the same refusals, the same ask-then-Approve for a send.
|
||||||
|
|
||||||
|
## Its processes
|
||||||
|
|
||||||
|
`prepare` brings the index's schema to this version's (numbered migrations, and `schema.sql` for a fresh database).
|
||||||
|
`keep` watches the accounts and is judged by its unit. `watch` judges every minute and is judged by `mail_health`.
|
||||||
@@ -0,0 +1,260 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
// The operator's mail accounts and who may read them (novox/hq ADR 0359 §1, §4, §5): two trusted settings, each a
|
||||||
|
// file the mesh writes from the module's settings and changes only on the operator's approval. `accounts` says which
|
||||||
|
// server each account is on and how to sign in (never a credential); `grants` says, per account, which callers may
|
||||||
|
// read its mail. The password of an account is the module's own secret mail-credential-<account>, given at the desk
|
||||||
|
// (ADR 0283 §6), at the path the secrets directory and that name make.
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"regexp"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The kinds of account an adapter exists for. Only `imap` is built (ADR 0359 §5); `imap-oauth` and `graph` are later
|
||||||
|
// decisions (§7), and an account of a kind not listed here is refused by name.
|
||||||
|
const KindIMAP = "imap"
|
||||||
|
|
||||||
|
// Kinds is every kind this version has an adapter for.
|
||||||
|
var Kinds = []string{KindIMAP}
|
||||||
|
|
||||||
|
// SecretFamily is the prefix of every account's own secret: mail-credential-<account> (ADR 0283 §6).
|
||||||
|
const SecretFamily = "mail-credential-"
|
||||||
|
|
||||||
|
// accountName is what an account may be called: a member name of the secret family (ADR 0283 §6) and one token of a
|
||||||
|
// bus subject, so never a dot, a slash or an upper-case letter.
|
||||||
|
var accountName = regexp.MustCompile(`^[a-z0-9][a-z0-9-]{0,39}$`)
|
||||||
|
|
||||||
|
// Server is where an account's mail is reached.
|
||||||
|
type Server struct {
|
||||||
|
Host string `json:"host"`
|
||||||
|
Port int `json:"port"`
|
||||||
|
// Security is tls (TLS from the first byte) or starttls (plain, then upgraded before signing in). Nothing
|
||||||
|
// signs in over a connection that is not encrypted.
|
||||||
|
Security string `json:"security"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// Address is host:port.
|
||||||
|
func (s Server) Address() string { return fmt.Sprintf("%s:%d", s.Host, s.Port) }
|
||||||
|
|
||||||
|
// Account is one of the operator's mail accounts, as the setting `accounts` declares it.
|
||||||
|
type Account struct {
|
||||||
|
// Name is the key the account has in the setting; set when the setting is read.
|
||||||
|
Name string `json:"-"`
|
||||||
|
Kind string `json:"kind"`
|
||||||
|
// Address is the account's mail address, said only to a caller granted the account.
|
||||||
|
Address string `json:"address"`
|
||||||
|
// User is the name it signs in with; the address when empty.
|
||||||
|
User string `json:"user,omitempty"`
|
||||||
|
Incoming Server `json:"incoming"`
|
||||||
|
Outgoing *Server `json:"outgoing,omitempty"`
|
||||||
|
// Folders are the folders watched besides the inbox; every folder the server lists when empty.
|
||||||
|
Folders []string `json:"folders,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// SignInAs is the name the account signs in with.
|
||||||
|
func (a Account) SignInAs() string {
|
||||||
|
if a.User != "" {
|
||||||
|
return a.User
|
||||||
|
}
|
||||||
|
return a.Address
|
||||||
|
}
|
||||||
|
|
||||||
|
// Secret is the name of the account's own secret.
|
||||||
|
func (a Account) Secret() string { return SecretFamily + a.Name }
|
||||||
|
|
||||||
|
// AccountsFile is the shape of the trusted setting `accounts`, as the mesh writes it.
|
||||||
|
type AccountsFile struct {
|
||||||
|
Accounts map[string]Account `json:"accounts"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// ReadAccounts reads the accounts setting, refusing it whole when any account in it cannot be watched: an account
|
||||||
|
// half-understood would be watched wrongly, and the operator approved the setting as one.
|
||||||
|
func ReadAccounts(path string) ([]Account, error) {
|
||||||
|
if path == "" {
|
||||||
|
return nil, errors.New("the module was started without its accounts setting (MESH_MAIL_ACCOUNTS)")
|
||||||
|
}
|
||||||
|
raw, err := os.ReadFile(path)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("the accounts setting cannot be read: %w", err)
|
||||||
|
}
|
||||||
|
return ParseAccounts(raw)
|
||||||
|
}
|
||||||
|
|
||||||
|
// ParseAccounts reads the accounts setting's content.
|
||||||
|
func ParseAccounts(raw []byte) ([]Account, error) {
|
||||||
|
var f AccountsFile
|
||||||
|
dec := json.NewDecoder(strings.NewReader(string(raw)))
|
||||||
|
dec.DisallowUnknownFields()
|
||||||
|
if err := dec.Decode(&f); err != nil {
|
||||||
|
return nil, fmt.Errorf("the accounts setting is not the module's: %v", err)
|
||||||
|
}
|
||||||
|
var out []Account
|
||||||
|
var problems []string
|
||||||
|
for name, a := range f.Accounts {
|
||||||
|
a.Name = name
|
||||||
|
if err := a.Check(); err != nil {
|
||||||
|
problems = append(problems, err.Error())
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
out = append(out, a)
|
||||||
|
}
|
||||||
|
if len(problems) > 0 {
|
||||||
|
sort.Strings(problems)
|
||||||
|
return nil, errors.New("the accounts setting is refused: " + strings.Join(problems, "; "))
|
||||||
|
}
|
||||||
|
sort.Slice(out, func(i, j int) bool { return out[i].Name < out[j].Name })
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check says why an account cannot be watched, or nil.
|
||||||
|
func (a Account) Check() error {
|
||||||
|
say := func(format string, args ...any) error {
|
||||||
|
return fmt.Errorf("the account %q: %s", a.Name, fmt.Sprintf(format, args...))
|
||||||
|
}
|
||||||
|
if !accountName.MatchString(a.Name) {
|
||||||
|
return say("is not an account's name: a lower-case letter or digit, then lower-case letters, digits and hyphens, at most 40")
|
||||||
|
}
|
||||||
|
known := false
|
||||||
|
for _, k := range Kinds {
|
||||||
|
known = known || a.Kind == k
|
||||||
|
}
|
||||||
|
if !known {
|
||||||
|
return say("is of the kind %q, and this version watches only %s", a.Kind, strings.Join(Kinds, ", "))
|
||||||
|
}
|
||||||
|
if !strings.Contains(a.Address, "@") {
|
||||||
|
return say("has no mail address")
|
||||||
|
}
|
||||||
|
if err := a.Incoming.check("incoming"); err != nil {
|
||||||
|
return say("%v", err)
|
||||||
|
}
|
||||||
|
if a.Outgoing != nil {
|
||||||
|
if err := a.Outgoing.check("outgoing"); err != nil {
|
||||||
|
return say("%v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, f := range a.Folders {
|
||||||
|
if strings.TrimSpace(f) == "" || strings.EqualFold(f, "INBOX") {
|
||||||
|
return say("lists the folder %q, and the inbox is always watched", f)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s Server) check(which string) error {
|
||||||
|
switch {
|
||||||
|
case strings.TrimSpace(s.Host) == "" || strings.ContainsAny(s.Host, " /:"):
|
||||||
|
return fmt.Errorf("its %s server has no host name", which)
|
||||||
|
case s.Port <= 0 || s.Port > 65535:
|
||||||
|
return fmt.Errorf("its %s server's port %d is not a port", which, s.Port)
|
||||||
|
case s.Security != "tls" && s.Security != "starttls":
|
||||||
|
return fmt.Errorf("its %s server's security is %q, and only tls or starttls signs in", which, s.Security)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// SecretPath is where the account's own secret is placed in the secrets directory, as the manifest declares the
|
||||||
|
// family: <dir>/mail-credential-<account>.secret.
|
||||||
|
func SecretPath(dir string, a Account) string { return filepath.Join(dir, a.Secret()+".secret") }
|
||||||
|
|
||||||
|
// ReadSecret is the account's password, or given false when nobody gave it yet: the mesh never makes a member of the
|
||||||
|
// family, so a member not given is no file (ADR 0283 §6).
|
||||||
|
func ReadSecret(dir string, a Account) (string, bool, error) {
|
||||||
|
raw, err := os.ReadFile(SecretPath(dir, a))
|
||||||
|
if errors.Is(err, os.ErrNotExist) {
|
||||||
|
return "", false, nil
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return "", false, err
|
||||||
|
}
|
||||||
|
pw := strings.TrimRight(string(raw), "\r\n")
|
||||||
|
if pw == "" {
|
||||||
|
return "", false, nil
|
||||||
|
}
|
||||||
|
return pw, true, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// SecretGiven says whether the account's own secret was given.
|
||||||
|
//
|
||||||
|
// It looks at the file and never reads it: whoever judges whether a password was given has no need of the password.
|
||||||
|
// A file that is there and not empty was given; whether it signs in is the keeper's to say.
|
||||||
|
func SecretGiven(dir string, a Account) bool {
|
||||||
|
info, err := os.Stat(SecretPath(dir, a))
|
||||||
|
return err == nil && info.Mode().IsRegular() && info.Size() > 0
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---- grants ------------------------------------------------------------------------------------------------------
|
||||||
|
|
||||||
|
// Grants is the trusted setting `grants`: per account, the callers that may read its mail (ADR 0359 §4). An entry is a
|
||||||
|
// module's name (granted wherever it runs), a module on one machine (`<machine>.<module>`) or a person
|
||||||
|
// (`person.<name>`), as the runtime names callers.
|
||||||
|
type Grants map[string][]string
|
||||||
|
|
||||||
|
// GrantsFile is the shape of the setting as the mesh writes it.
|
||||||
|
type GrantsFile struct {
|
||||||
|
Grants Grants `json:"grants"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// ReadGrants reads the grants setting. A setting that cannot be read grants nothing, and says so.
|
||||||
|
func ReadGrants(path string) (Grants, error) {
|
||||||
|
if path == "" {
|
||||||
|
return Grants{}, errors.New("the module was started without its grants setting (MESH_MAIL_GRANTS)")
|
||||||
|
}
|
||||||
|
raw, err := os.ReadFile(path)
|
||||||
|
if err != nil {
|
||||||
|
return Grants{}, fmt.Errorf("the grants setting cannot be read: %w", err)
|
||||||
|
}
|
||||||
|
var f GrantsFile
|
||||||
|
dec := json.NewDecoder(strings.NewReader(string(raw)))
|
||||||
|
dec.DisallowUnknownFields()
|
||||||
|
if err := dec.Decode(&f); err != nil {
|
||||||
|
return Grants{}, fmt.Errorf("the grants setting is not the module's: %v", err)
|
||||||
|
}
|
||||||
|
if f.Grants == nil {
|
||||||
|
f.Grants = Grants{}
|
||||||
|
}
|
||||||
|
return f.Grants, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Unproven is the caller the runtime proved nobody to be; it is granted nothing.
|
||||||
|
const Unproven = "unproven"
|
||||||
|
|
||||||
|
// Allows says whether the caller may read the account's mail. Nobody is granted an account by default, an unproven
|
||||||
|
// caller never, and a wildcard is no entry.
|
||||||
|
func (g Grants) Allows(caller, account string) bool {
|
||||||
|
if caller == "" || caller == Unproven {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
for _, entry := range g[account] {
|
||||||
|
if entry == "" || strings.ContainsAny(entry, "*>") {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if entry == caller {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
// A module's name alone grants it on every machine: the caller is <machine>.<module>.
|
||||||
|
if !strings.Contains(entry, ".") {
|
||||||
|
if _, module, ok := strings.Cut(caller, "."); ok && module == entry && !strings.HasPrefix(caller, "person.") {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
// Granted is the accounts, of those given, the caller may read.
|
||||||
|
func (g Grants) Granted(caller string, accounts []Account) []string {
|
||||||
|
var out []string
|
||||||
|
for _, a := range accounts {
|
||||||
|
if g.Allows(caller, a.Name) {
|
||||||
|
out = append(out, a.Name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
@@ -0,0 +1,124 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"reflect"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The first stage's account, as the operator sets the setting `accounts` (ADR 0359 §5): its address and its server,
|
||||||
|
// never a credential; its password is the own secret mail-credential-mesh.
|
||||||
|
const stageOne = `{"accounts": {"mesh": {"kind": "imap", "address": "jschoubben@novox.be",
|
||||||
|
"incoming": {"host": "mail.novox.be", "port": 993, "security": "tls"}}}}`
|
||||||
|
|
||||||
|
func TestTheFirstStagesAccountIsReadAndItsSecretNamed(t *testing.T) {
|
||||||
|
got, err := ParseAccounts([]byte(stageOne))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(got) != 1 || got[0].Name != "mesh" || got[0].Kind != KindIMAP || got[0].SignInAs() != "jschoubben@novox.be" ||
|
||||||
|
got[0].Incoming.Address() != "mail.novox.be:993" || got[0].Secret() != "mail-credential-mesh" || got[0].Outgoing != nil {
|
||||||
|
t.Fatalf("%+v", got)
|
||||||
|
}
|
||||||
|
if p := SecretPath("/state", got[0]); p != "/state/mail-credential-mesh.secret" {
|
||||||
|
t.Fatalf("the secret is placed at %s", p)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A second account of the kind imap on another server — Gmail with an app password, another member of the family —
|
||||||
|
// is configuration only: no code changes for it. Each account may also say its outgoing server, which mail_send
|
||||||
|
// uses to send through it (novox/hq ADR 0361 §1).
|
||||||
|
func TestASecondIMAPAccountIsConfigurationOnly(t *testing.T) {
|
||||||
|
got, err := ParseAccounts([]byte(`{"accounts": {
|
||||||
|
"mesh": {"kind": "imap", "address": "jschoubben@novox.be", "incoming": {"host": "mail.novox.be", "port": 993, "security": "tls"}},
|
||||||
|
"gmail": {"kind": "imap", "address": "someone@gmail.com", "incoming": {"host": "imap.gmail.com", "port": 993, "security": "tls"},
|
||||||
|
"outgoing": {"host": "smtp.gmail.com", "port": 465, "security": "tls"}, "folders": ["[Gmail]/Sent Mail"]}}}`))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(got) != 2 || got[0].Name != "gmail" || got[0].Secret() != "mail-credential-gmail" || got[0].Outgoing.Address() != "smtp.gmail.com:465" ||
|
||||||
|
!reflect.DeepEqual(got[0].Folders, []string{"[Gmail]/Sent Mail"}) {
|
||||||
|
t.Fatalf("%+v", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAnAccountThatCannotBeWatchedRefusesTheSetting(t *testing.T) {
|
||||||
|
for name, raw := range map[string]string{
|
||||||
|
"a later kind": `{"accounts": {"work": {"kind": "graph", "address": "a@b.c", "incoming": {"host": "h", "port": 1, "security": "tls"}}}}`,
|
||||||
|
"no encryption": `{"accounts": {"x": {"kind": "imap", "address": "a@b.c", "incoming": {"host": "h", "port": 143, "security": "none"}}}}`,
|
||||||
|
"a name with a dot": `{"accounts": {"my.box": {"kind": "imap", "address": "a@b.c", "incoming": {"host": "h", "port": 993, "security": "tls"}}}}`,
|
||||||
|
"a password in it": `{"accounts": {"x": {"kind": "imap", "address": "a@b.c", "password": "p", "incoming": {"host": "h", "port": 993, "security": "tls"}}}}`,
|
||||||
|
"no address": `{"accounts": {"x": {"kind": "imap", "incoming": {"host": "h", "port": 993, "security": "tls"}}}}`,
|
||||||
|
"the inbox listed": `{"accounts": {"x": {"kind": "imap", "address": "a@b.c", "folders": ["INBOX"], "incoming": {"host": "h", "port": 993, "security": "tls"}}}}`,
|
||||||
|
"a bad outgoing one": `{"accounts": {"x": {"kind": "imap", "address": "a@b.c", "incoming": {"host": "h", "port": 993, "security": "tls"}, "outgoing": {"host": "", "port": 465, "security": "tls"}}}}`,
|
||||||
|
} {
|
||||||
|
if _, err := ParseAccounts([]byte(raw)); err == nil {
|
||||||
|
t.Errorf("%s: accepted", name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if got, err := ParseAccounts([]byte(`{"accounts": {}}`)); err != nil || len(got) != 0 {
|
||||||
|
t.Fatalf("the manifest's default: %v %v", got, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Grants (ADR 0359 §4): nobody by default; a module's name grants it wherever it runs; a module on one machine, or a
|
||||||
|
// person, by its full name; never the unproven, never by a wildcard, never one account for another.
|
||||||
|
func TestGrantsAreByAccountAndByProvenCaller(t *testing.T) {
|
||||||
|
g := Grants{"mesh": {"triage", "novox.desk", "person.jochen", "*"}}
|
||||||
|
for caller, want := range map[string]bool{
|
||||||
|
"novox.triage": true, "shanks.triage": true, "novox.desk": true, "g14.desk": false, "person.jochen": true,
|
||||||
|
"person.triage": false, "novox.other": false, Unproven: false, "": false, "novox.*": false,
|
||||||
|
} {
|
||||||
|
if got := g.Allows(caller, "mesh"); got != want {
|
||||||
|
t.Errorf("%s: %v", caller, got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if g.Allows("novox.triage", "gmail") || (Grants{}).Allows("novox.triage", "mesh") {
|
||||||
|
t.Fatal("a grant reached an account it does not name")
|
||||||
|
}
|
||||||
|
dir := t.TempDir()
|
||||||
|
path := filepath.Join(dir, "grants.json")
|
||||||
|
_ = os.WriteFile(path, []byte(`{"grants": {"mesh": ["triage"]}}`), 0o600)
|
||||||
|
read, err := ReadGrants(path)
|
||||||
|
if err != nil || !read.Allows("novox.triage", "mesh") {
|
||||||
|
t.Fatalf("%v %v", read, err)
|
||||||
|
}
|
||||||
|
_ = os.WriteFile(path, []byte(`{"grants": {"mesh": "triage"}}`), 0o600)
|
||||||
|
if read, err := ReadGrants(path); err == nil || len(read) != 0 {
|
||||||
|
t.Fatal("a grants setting it cannot read granted something")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Whether a password was given is read from the file's presence and size alone (issue 733): an empty file is not
|
||||||
|
// given, a file the judge cannot read is still given, and the password is never read to say so.
|
||||||
|
func TestASecretIsGivenByItsFileNotByReadingIt(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
a := testAccount()
|
||||||
|
if SecretGiven(dir, a) {
|
||||||
|
t.Fatal("given before anything was written")
|
||||||
|
}
|
||||||
|
_ = os.WriteFile(SecretPath(dir, a), nil, 0o600)
|
||||||
|
if SecretGiven(dir, a) {
|
||||||
|
t.Fatal("an empty file is a password")
|
||||||
|
}
|
||||||
|
_ = os.WriteFile(SecretPath(dir, a), []byte("pw\n"), 0o600)
|
||||||
|
if pw, given, _ := ReadSecret(dir, a); !given || pw != "pw" || strings.Contains(pw, "\n") {
|
||||||
|
t.Fatalf("%q %v", pw, given)
|
||||||
|
}
|
||||||
|
// Unreadable to the judge, as a secret owned by root is to the watch: still given.
|
||||||
|
if err := os.Chmod(SecretPath(dir, a), 0); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if !SecretGiven(dir, a) {
|
||||||
|
t.Fatal("a password the judge cannot read is said not given")
|
||||||
|
}
|
||||||
|
src, _ := os.ReadFile("config.go")
|
||||||
|
body := string(src)
|
||||||
|
body = body[strings.Index(body, "func SecretGiven("):]
|
||||||
|
body = body[:strings.Index(body, "\n}\n")]
|
||||||
|
if strings.Contains(body, "ReadSecret") || strings.Contains(body, "ReadFile") || strings.Contains(body, "os.Open") {
|
||||||
|
t.Fatalf("SecretGiven reads the password:\n%s", body)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,170 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
// How mesh-mail is (novox/hq ADR 0359 §6). Its keeper's liveness is its unit's; its watching is judged here, by the
|
||||||
|
// watch process, once a minute, and left in a status file the bundle's tool `mail_health` reads:
|
||||||
|
//
|
||||||
|
// - waits, naming the own secret mail-credential-<account>, for an account whose password nobody gave yet (ADR 0283),
|
||||||
|
// so the condition is needs-operator and the gate passes;
|
||||||
|
// - unhealthy, naming the account and why, when its connection is down or its server has not answered for
|
||||||
|
// SilentAfter (five minutes) — an IDLE that went silent;
|
||||||
|
// - healthy otherwise.
|
||||||
|
//
|
||||||
|
// A fault beside a wait is unhealthy and says both.
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/health"
|
||||||
|
)
|
||||||
|
|
||||||
|
// HealthTool is the module's tool the node-engine asks how it is.
|
||||||
|
const HealthTool = "mail_health"
|
||||||
|
|
||||||
|
// WatchEvery is how often the watch judges; StatusStale how old its verdict may be before the tool says it stopped.
|
||||||
|
const (
|
||||||
|
WatchEvery = time.Minute
|
||||||
|
StatusStale = 3 * WatchEvery
|
||||||
|
// SignInGrace is how long a password just given may wait for its keeper to sign in before that is a fault.
|
||||||
|
SignInGrace = 3 * time.Minute
|
||||||
|
)
|
||||||
|
|
||||||
|
// Judge is the module's health from what the watch reads: the accounts setting (or why it cannot be read), whether the
|
||||||
|
// index serves, which passwords were given, and each account's state as its keeper said it.
|
||||||
|
func Judge(accounts []Account, accountsErr, readyErr error, given func(Account) bool, states map[string]AccountState, now time.Time) health.Answer {
|
||||||
|
if accountsErr != nil {
|
||||||
|
return health.Unhealthy(accountsErr.Error())
|
||||||
|
}
|
||||||
|
var problems, fine []string
|
||||||
|
var waits []health.Wait
|
||||||
|
if readyErr != nil {
|
||||||
|
problems = append(problems, "the index cannot serve: "+readyErr.Error())
|
||||||
|
}
|
||||||
|
for _, a := range accounts {
|
||||||
|
if !given(a) {
|
||||||
|
waits = append(waits, health.Wait{
|
||||||
|
Part: "the mail account " + a.Name,
|
||||||
|
Secret: a.Secret(),
|
||||||
|
What: "the password of the mail account " + a.Name,
|
||||||
|
})
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if readyErr != nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
st, said := states[a.Name]
|
||||||
|
switch {
|
||||||
|
case !said:
|
||||||
|
problems = append(problems, fmt.Sprintf("the account %s: its keeper has said nothing of it", a.Name))
|
||||||
|
case st.State == AccountWaits:
|
||||||
|
if now.Sub(st.Said) > SignInGrace {
|
||||||
|
problems = append(problems, fmt.Sprintf("the account %s: its password was given, and its keeper has not signed in for %s",
|
||||||
|
a.Name, now.Sub(st.Said).Round(time.Second)))
|
||||||
|
} else {
|
||||||
|
fine = append(fine, a.Name+" is signing in")
|
||||||
|
}
|
||||||
|
case st.State == AccountFailing:
|
||||||
|
problems = append(problems, fmt.Sprintf("the account %s is not watched: %s", a.Name, st.Why))
|
||||||
|
case st.Heard.IsZero():
|
||||||
|
problems = append(problems, fmt.Sprintf("the account %s: its server has not answered since its keeper started", a.Name))
|
||||||
|
case now.Sub(st.Heard) > SilentAfter:
|
||||||
|
problems = append(problems, fmt.Sprintf("the account %s: its server has not answered for %s (its IDLE on the inbox is silent)",
|
||||||
|
a.Name, now.Sub(st.Heard).Round(time.Second)))
|
||||||
|
default:
|
||||||
|
fine = append(fine, fmt.Sprintf("%s is %s (%s)", a.Name, st.State, st.Why))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
waitSaid := ""
|
||||||
|
if len(waits) > 0 {
|
||||||
|
var names []string
|
||||||
|
for _, w := range waits {
|
||||||
|
names = append(names, w.Secret)
|
||||||
|
}
|
||||||
|
waitSaid = "waits for its password: " + strings.Join(names, ", ")
|
||||||
|
}
|
||||||
|
switch {
|
||||||
|
case len(problems) > 0:
|
||||||
|
why := strings.Join(problems, "; ")
|
||||||
|
if waitSaid != "" {
|
||||||
|
why += "; and it " + waitSaid
|
||||||
|
}
|
||||||
|
return health.Unhealthy(why, waits...)
|
||||||
|
case len(waits) > 0:
|
||||||
|
why := "nothing is wrong, and it " + waitSaid
|
||||||
|
if len(fine) > 0 {
|
||||||
|
why += "; " + strings.Join(fine, "; ")
|
||||||
|
}
|
||||||
|
return health.Waiting(why, waits...)
|
||||||
|
case len(accounts) == 0:
|
||||||
|
return health.Healthy("no mail account is declared, so nothing is watched")
|
||||||
|
}
|
||||||
|
return health.Healthy(fmt.Sprintf("%d account(s) watched: %s", len(accounts), strings.Join(fine, "; ")))
|
||||||
|
}
|
||||||
|
|
||||||
|
// Status is the watch's verdict as it leaves it: the answer, and when it looked.
|
||||||
|
type Status struct {
|
||||||
|
health.Answer
|
||||||
|
At time.Time `json:"at"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func writeStatus(path string, s Status) error {
|
||||||
|
raw, err := json.Marshal(s)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
tmp := path + ".next"
|
||||||
|
if err := os.WriteFile(tmp, raw, 0o600); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return os.Rename(tmp, path)
|
||||||
|
}
|
||||||
|
|
||||||
|
// ReadStatus is the watch's last verdict, as the tool answers it: unhealthy when there is none, when it cannot be read,
|
||||||
|
// or when it is older than StatusStale.
|
||||||
|
func ReadStatus(path string, now time.Time) health.Answer {
|
||||||
|
raw, err := os.ReadFile(path)
|
||||||
|
if err != nil {
|
||||||
|
return health.Unhealthy("the watch has left no verdict yet: " + err.Error())
|
||||||
|
}
|
||||||
|
var s Status
|
||||||
|
if err := json.Unmarshal(raw, &s); err != nil {
|
||||||
|
return health.Unhealthy("the watch's verdict cannot be read")
|
||||||
|
}
|
||||||
|
if age := now.Sub(s.At); age > StatusStale {
|
||||||
|
return health.Unhealthy(fmt.Sprintf("the watch has not looked for %s (its last verdict: %s)", age.Round(time.Second), s.Why))
|
||||||
|
}
|
||||||
|
// Read through the SDK's reader, so a wait it cannot check is never excused.
|
||||||
|
a, err := health.Read(mustJSON(s.Answer))
|
||||||
|
if err != nil {
|
||||||
|
return health.Unhealthy("the watch's verdict cannot be read: " + err.Error())
|
||||||
|
}
|
||||||
|
return a
|
||||||
|
}
|
||||||
|
|
||||||
|
func mustJSON(v any) []byte {
|
||||||
|
raw, _ := json.Marshal(v)
|
||||||
|
return raw
|
||||||
|
}
|
||||||
|
|
||||||
|
// watchOnce is one look: what the watch reads, judged.
|
||||||
|
func watchOnce(ctx context.Context, env Env, open func(context.Context) (Store, func(), error), now time.Time) health.Answer {
|
||||||
|
accounts, accountsErr := ReadAccounts(env.Accounts)
|
||||||
|
var readyErr error
|
||||||
|
states := map[string]AccountState{}
|
||||||
|
st, done, err := open(ctx)
|
||||||
|
if err != nil {
|
||||||
|
readyErr = err
|
||||||
|
} else {
|
||||||
|
defer done()
|
||||||
|
if readyErr = st.Ready(ctx); readyErr == nil {
|
||||||
|
if states, err = st.States(ctx); err != nil {
|
||||||
|
readyErr = err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return Judge(accounts, accountsErr, readyErr, func(a Account) bool { return SecretGiven(env.Secrets, a) }, states, now)
|
||||||
|
}
|
||||||
@@ -0,0 +1,116 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/health"
|
||||||
|
)
|
||||||
|
|
||||||
|
var judgedAt = time.Date(2026, 10, 14, 18, 0, 0, 0, time.UTC)
|
||||||
|
|
||||||
|
func given(names ...string) func(Account) bool {
|
||||||
|
return func(a Account) bool { return contains(names, a.Name) }
|
||||||
|
}
|
||||||
|
|
||||||
|
// Before the password is given, mail_health answers waits, naming the secret, which the node-engine reads as a wait
|
||||||
|
// the controller can excuse (ADR 0283 §1, §3): the family mail-credential-* is declared issued outside the mesh.
|
||||||
|
func TestItWaitsNamingTheSecretUntilThePasswordIsGiven(t *testing.T) {
|
||||||
|
a := Judge([]Account{testAccount()}, nil, nil, given(), nil, judgedAt)
|
||||||
|
if a.State != health.StateWaits || len(a.Waits) != 1 || a.Waits[0].Secret != "mail-credential-mesh" || a.Healthy {
|
||||||
|
t.Fatalf("%+v", a)
|
||||||
|
}
|
||||||
|
read, err := health.Read(mustJSON(a))
|
||||||
|
if err != nil || read.State != health.StateWaits {
|
||||||
|
t.Fatalf("the node-engine reads it as %+v (%v)", read, err)
|
||||||
|
}
|
||||||
|
if strings.Contains(a.Waits[0].What, "@") || strings.Contains(a.Why, "@") {
|
||||||
|
t.Fatalf("the wait says an address: %+v", a)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Unhealthy, naming the account and why, when its connection is down or its server silent for five minutes.
|
||||||
|
func TestItIsUnhealthyWhenAnAccountIsDownOrSilent(t *testing.T) {
|
||||||
|
acc := []Account{testAccount()}
|
||||||
|
down := map[string]AccountState{"mesh": {Account: "mesh", State: AccountFailing, Why: "its server mail.example cannot be reached", Said: judgedAt}}
|
||||||
|
if a := Judge(acc, nil, nil, given("mesh"), down, judgedAt); a.State != health.StateUnhealthy || !strings.Contains(a.Why, "the account mesh is not watched: its server") {
|
||||||
|
t.Fatalf("down: %+v", a)
|
||||||
|
}
|
||||||
|
silent := map[string]AccountState{"mesh": {Account: "mesh", State: AccountCurrent, Heard: judgedAt.Add(-SilentAfter - time.Second), Said: judgedAt}}
|
||||||
|
if a := Judge(acc, nil, nil, given("mesh"), silent, judgedAt); a.State != health.StateUnhealthy || !strings.Contains(a.Why, "IDLE on the inbox is silent") {
|
||||||
|
t.Fatalf("silent: %+v", a)
|
||||||
|
}
|
||||||
|
fine := map[string]AccountState{"mesh": {Account: "mesh", State: AccountCurrent, Why: "watching", Heard: judgedAt.Add(-time.Minute), Said: judgedAt}}
|
||||||
|
if a := Judge(acc, nil, nil, given("mesh"), fine, judgedAt); a.State != health.StateHealthy || !a.Healthy {
|
||||||
|
t.Fatalf("fine: %+v", a)
|
||||||
|
}
|
||||||
|
if a := Judge(acc, nil, errors.New("the schema is behind"), given("mesh"), fine, judgedAt); a.State != health.StateUnhealthy {
|
||||||
|
t.Fatalf("an index that cannot serve: %+v", a)
|
||||||
|
}
|
||||||
|
if a := Judge(nil, errors.New("the accounts setting is refused"), nil, given(), nil, judgedAt); a.State != health.StateUnhealthy {
|
||||||
|
t.Fatalf("a setting it cannot read: %+v", a)
|
||||||
|
}
|
||||||
|
// A fault beside a wait is unhealthy and says both (ADR 0283 §1).
|
||||||
|
two := []Account{testAccount(), {Name: "gmail", Kind: KindIMAP, Address: "x@example.com", Incoming: Server{Host: "h", Port: 993, Security: "tls"}}}
|
||||||
|
a := Judge(two, nil, nil, given("mesh"), down, judgedAt)
|
||||||
|
if a.State != health.StateUnhealthy || len(a.Waits) != 1 || a.Waits[0].Secret != "mail-credential-gmail" || !strings.Contains(a.Why, "mail-credential-gmail") {
|
||||||
|
t.Fatalf("both: %+v", a)
|
||||||
|
}
|
||||||
|
// A password just given is signing in for a while; not for ever.
|
||||||
|
signing := map[string]AccountState{"mesh": {Account: "mesh", State: AccountWaits, Said: judgedAt.Add(-time.Minute)}}
|
||||||
|
if a := Judge(acc, nil, nil, given("mesh"), signing, judgedAt); a.State != health.StateHealthy {
|
||||||
|
t.Fatalf("signing in: %+v", a)
|
||||||
|
}
|
||||||
|
signing["mesh"] = AccountState{Account: "mesh", State: AccountWaits, Said: judgedAt.Add(-SignInGrace - time.Minute)}
|
||||||
|
if a := Judge(acc, nil, nil, given("mesh"), signing, judgedAt); a.State != health.StateUnhealthy {
|
||||||
|
t.Fatalf("never signed in: %+v", a)
|
||||||
|
}
|
||||||
|
if a := Judge(nil, nil, nil, given(), nil, judgedAt); a.State != health.StateHealthy || !strings.Contains(a.Why, "nothing is watched") {
|
||||||
|
t.Fatalf("no account: %+v", a)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The tool reads the watch's verdict, and says when the watch stopped looking.
|
||||||
|
func TestTheHealthToolReadsTheWatchsVerdictAndSaysWhenItIsStale(t *testing.T) {
|
||||||
|
path := filepath.Join(t.TempDir(), "health.json")
|
||||||
|
if a := ReadStatus(path, judgedAt); a.State != health.StateUnhealthy {
|
||||||
|
t.Fatalf("no verdict: %+v", a)
|
||||||
|
}
|
||||||
|
waits := Judge([]Account{testAccount()}, nil, nil, given(), nil, judgedAt)
|
||||||
|
if err := writeStatus(path, Status{Answer: waits, At: judgedAt}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if a := ReadStatus(path, judgedAt.Add(time.Minute)); a.State != health.StateWaits || a.Waits[0].Secret != "mail-credential-mesh" {
|
||||||
|
t.Fatalf("%+v", a)
|
||||||
|
}
|
||||||
|
if a := ReadStatus(path, judgedAt.Add(StatusStale+time.Second)); a.State != health.StateUnhealthy || !strings.Contains(a.Why, "has not looked") {
|
||||||
|
t.Fatalf("stale: %+v", a)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// One look of the watch, end to end over the files the mesh writes: the accounts setting, the secrets directory and
|
||||||
|
// the index.
|
||||||
|
func TestTheWatchJudgesFromTheFilesAndTheIndex(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
env := Env{Accounts: filepath.Join(dir, "accounts.json"), Secrets: dir}
|
||||||
|
writeFile(t, env.Accounts, stageOne)
|
||||||
|
st := newMemStore()
|
||||||
|
open := func(context.Context) (Store, func(), error) { return st, func() {}, nil }
|
||||||
|
if a := watchOnce(context.Background(), env, open, judgedAt); a.State != health.StateWaits {
|
||||||
|
t.Fatalf("%+v", a)
|
||||||
|
}
|
||||||
|
giveSecret(t, dir, Account{Name: "mesh"}, "pw")
|
||||||
|
_ = st.SetState(context.Background(), AccountState{Account: "mesh", State: AccountCurrent, Heard: judgedAt, Said: judgedAt})
|
||||||
|
if a := watchOnce(context.Background(), env, open, judgedAt); a.State != health.StateHealthy {
|
||||||
|
t.Fatalf("%+v", a)
|
||||||
|
}
|
||||||
|
raw, _ := json.Marshal(Status{Answer: health.Healthy("x"), At: judgedAt})
|
||||||
|
if !strings.Contains(string(raw), `"state":"healthy"`) || !strings.Contains(string(raw), `"at":`) {
|
||||||
|
t.Fatalf("the status file's shape: %s", raw)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,570 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
// The adapter for an account of the kind `imap` (novox/hq ADR 0359 §1, §2, §5). For each account it keeps one
|
||||||
|
// connection: IDLE on the inbox, and every PollEvery a look at every other folder. Each look compares what the server
|
||||||
|
// holds with the index and says each difference as an event: a message added, its flags changed, or removed. The first
|
||||||
|
// look at a folder indexes what is there without saying it — what the bus is told is what changed since the mesh
|
||||||
|
// started watching, never the history of a mailbox.
|
||||||
|
//
|
||||||
|
// **It only reads.** Every folder is opened read-only (EXAMINE), and every message is fetched with BODY.PEEK, so
|
||||||
|
// watching never marks a message seen. The client is used through reader alone, which has no command that flags,
|
||||||
|
// moves, copies, deletes, appends or sends (§5); a test holds the code to that.
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"crypto/tls"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/emersion/go-imap/v2"
|
||||||
|
"github.com/emersion/go-imap/v2/imapclient"
|
||||||
|
)
|
||||||
|
|
||||||
|
// How the keeper paces itself. IdleCycle is how long one IDLE lasts before it is ended and begun again, which proves
|
||||||
|
// the server still answers; SilentAfter is how long without an answer makes the account unhealthy (§6).
|
||||||
|
const (
|
||||||
|
IdleCycle = 2 * time.Minute
|
||||||
|
PollEvery = 5 * time.Minute
|
||||||
|
SilentAfter = 5 * time.Minute
|
||||||
|
RetryFirst = 30 * time.Second
|
||||||
|
RetryMost = 5 * time.Minute
|
||||||
|
FetchBatch = 25
|
||||||
|
// BatchBytes bounds the mail one fetch of bodies holds; SizeBatch is how many sizes one fetch asks.
|
||||||
|
BatchBytes = 64 << 20
|
||||||
|
SizeBatch = 500
|
||||||
|
)
|
||||||
|
|
||||||
|
// Batches cuts the uids, in order, into batches of at most most messages whose sizes add up to at most limit bytes. A
|
||||||
|
// message larger than limit is a batch of its own. A size not known counts as nothing: the server listed the message,
|
||||||
|
// and the fetch says what it holds.
|
||||||
|
func Batches(uids []uint32, sizes map[uint32]int64, limit int64, most int) [][]uint32 {
|
||||||
|
var out [][]uint32
|
||||||
|
var cur []uint32
|
||||||
|
var bytes int64
|
||||||
|
for _, u := range uids {
|
||||||
|
size := sizes[u]
|
||||||
|
if len(cur) > 0 && (len(cur) >= most || bytes+size > limit) {
|
||||||
|
out = append(out, cur)
|
||||||
|
cur, bytes = nil, 0
|
||||||
|
}
|
||||||
|
cur = append(cur, u)
|
||||||
|
bytes += size
|
||||||
|
}
|
||||||
|
if len(cur) > 0 {
|
||||||
|
out = append(out, cur)
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
func uidSet(uids []uint32) imap.UIDSet {
|
||||||
|
set := make([]imap.UID, len(uids))
|
||||||
|
for i, u := range uids {
|
||||||
|
set[i] = imap.UID(u)
|
||||||
|
}
|
||||||
|
return imap.UIDSetNum(set...)
|
||||||
|
}
|
||||||
|
|
||||||
|
func orBytes(n, def int64) int64 {
|
||||||
|
if n > 0 {
|
||||||
|
return n
|
||||||
|
}
|
||||||
|
return def
|
||||||
|
}
|
||||||
|
|
||||||
|
// reader is the part of an IMAP client the keeper uses: nothing in it changes a mailbox.
|
||||||
|
type reader interface {
|
||||||
|
Login(username, password string) *imapclient.Command
|
||||||
|
List(ref, pattern string, options *imap.ListOptions) *imapclient.ListCommand
|
||||||
|
Select(mailbox string, options *imap.SelectOptions) *imapclient.SelectCommand
|
||||||
|
Fetch(numSet imap.NumSet, options *imap.FetchOptions) *imapclient.FetchCommand
|
||||||
|
Idle() (*imapclient.IdleCommand, error)
|
||||||
|
Caps() imap.CapSet
|
||||||
|
Closed() <-chan struct{}
|
||||||
|
Logout() *imapclient.Command
|
||||||
|
Close() error
|
||||||
|
}
|
||||||
|
|
||||||
|
// Dialer opens a connection to an account's server, handing what the server says unasked to the handler.
|
||||||
|
type Dialer func(a Account, handler *imapclient.UnilateralDataHandler) (reader, error)
|
||||||
|
|
||||||
|
// DialServer is the dialer for the mesh: TLS from the first byte, or STARTTLS before signing in, as the account says.
|
||||||
|
func DialServer(a Account, handler *imapclient.UnilateralDataHandler) (reader, error) {
|
||||||
|
opts := &imapclient.Options{UnilateralDataHandler: handler, TLSConfig: &tls.Config{ServerName: a.Incoming.Host, MinVersion: tls.VersionTLS12}}
|
||||||
|
var c *imapclient.Client
|
||||||
|
var err error
|
||||||
|
switch a.Incoming.Security {
|
||||||
|
case "tls":
|
||||||
|
c, err = imapclient.DialTLS(a.Incoming.Address(), opts)
|
||||||
|
case "starttls":
|
||||||
|
c, err = imapclient.DialStartTLS(a.Incoming.Address(), opts)
|
||||||
|
default:
|
||||||
|
err = fmt.Errorf("the account %s's security %q signs in over nothing this version speaks", a.Name, a.Incoming.Security)
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return c, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Keeper watches the accounts and keeps the index.
|
||||||
|
type Keeper struct {
|
||||||
|
Store Store
|
||||||
|
Objects Objects
|
||||||
|
Dial Dialer
|
||||||
|
SecretsDir string
|
||||||
|
Logf func(format string, a ...any)
|
||||||
|
Now func() time.Time
|
||||||
|
|
||||||
|
// Paces, for the tests; the constants above when zero.
|
||||||
|
IdleCycle, PollEvery, RetryFirst time.Duration
|
||||||
|
// BatchBytes, for the tests; the constant when zero.
|
||||||
|
BatchBytes int64
|
||||||
|
}
|
||||||
|
|
||||||
|
func (k *Keeper) now() time.Time {
|
||||||
|
if k.Now != nil {
|
||||||
|
return k.Now()
|
||||||
|
}
|
||||||
|
return time.Now()
|
||||||
|
}
|
||||||
|
|
||||||
|
func (k *Keeper) logf(format string, a ...any) {
|
||||||
|
if k.Logf != nil {
|
||||||
|
k.Logf(format, a...)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func orDefault(d, def time.Duration) time.Duration {
|
||||||
|
if d > 0 {
|
||||||
|
return d
|
||||||
|
}
|
||||||
|
return def
|
||||||
|
}
|
||||||
|
|
||||||
|
// Run watches every account until ctx ends.
|
||||||
|
func (k *Keeper) Run(ctx context.Context, accounts []Account) {
|
||||||
|
var wg sync.WaitGroup
|
||||||
|
for _, a := range accounts {
|
||||||
|
wg.Add(1)
|
||||||
|
go func(a Account) {
|
||||||
|
defer wg.Done()
|
||||||
|
k.Watch(ctx, a)
|
||||||
|
}(a)
|
||||||
|
}
|
||||||
|
wg.Wait()
|
||||||
|
}
|
||||||
|
|
||||||
|
func (k *Keeper) say(ctx context.Context, a Account, state, why string, heard, synced time.Time) {
|
||||||
|
if err := k.Store.SetState(ctx, AccountState{Account: a.Name, State: state, Why: why, Heard: heard, Synced: synced, Said: k.now()}); err != nil {
|
||||||
|
k.logf("[mesh-mail] the account %s's state could not be kept: %v", a.Name, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Watch keeps one account until ctx ends: waits while its secret is not given, signs in, watches, and on any fault
|
||||||
|
// says it, waits a little longer each time, and begins again.
|
||||||
|
func (k *Keeper) Watch(ctx context.Context, a Account) {
|
||||||
|
retry := orDefault(k.RetryFirst, RetryFirst)
|
||||||
|
var heard, synced time.Time
|
||||||
|
for ctx.Err() == nil {
|
||||||
|
pw, given, err := ReadSecret(k.SecretsDir, a)
|
||||||
|
switch {
|
||||||
|
case err != nil:
|
||||||
|
k.say(ctx, a, AccountFailing, "its password was given and cannot be read: "+err.Error(), heard, synced)
|
||||||
|
case !given:
|
||||||
|
k.say(ctx, a, AccountWaits, "its password ("+a.Secret()+") has not been given", heard, synced)
|
||||||
|
default:
|
||||||
|
s := &session{k: k, a: a, wake: make(chan struct{}, 1), heard: heard, synced: synced}
|
||||||
|
err = s.run(ctx, pw)
|
||||||
|
heard, synced = s.heard, s.synced
|
||||||
|
if ctx.Err() != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
k.say(ctx, a, AccountFailing, err.Error(), heard, synced)
|
||||||
|
k.logf("[mesh-mail] the account %s: %v; trying again in %s", a.Name, err, retry)
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return
|
||||||
|
case <-time.After(retry):
|
||||||
|
}
|
||||||
|
if given && err != nil {
|
||||||
|
retry = min(retry*2, RetryMost)
|
||||||
|
} else {
|
||||||
|
retry = orDefault(k.RetryFirst, RetryFirst)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// session is one connection to one account's server.
|
||||||
|
type session struct {
|
||||||
|
k *Keeper
|
||||||
|
a Account
|
||||||
|
c reader
|
||||||
|
wake chan struct{}
|
||||||
|
heard time.Time
|
||||||
|
synced time.Time
|
||||||
|
// state and why are what the session last said of the account, and told when it said it.
|
||||||
|
state, why string
|
||||||
|
told time.Time
|
||||||
|
}
|
||||||
|
|
||||||
|
// answered is the server having answered: kept, and told at least once a minute, so a long first look at a large
|
||||||
|
// mailbox still shows the account alive to the health watch.
|
||||||
|
func (s *session) answered() {
|
||||||
|
s.heard = s.k.now()
|
||||||
|
if s.state != "" && s.heard.Sub(s.told) >= time.Minute {
|
||||||
|
s.tell(s.state, s.why)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// tell says the account's state.
|
||||||
|
func (s *session) tell(state, why string) {
|
||||||
|
s.state, s.why, s.told = state, why, s.k.now()
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
s.k.say(ctx, s.a, state, why, s.heard, s.synced)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *session) poke() {
|
||||||
|
select {
|
||||||
|
case s.wake <- struct{}{}:
|
||||||
|
default:
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *session) run(ctx context.Context, password string) error {
|
||||||
|
handler := &imapclient.UnilateralDataHandler{
|
||||||
|
Expunge: func(uint32) { s.poke() },
|
||||||
|
Mailbox: func(*imapclient.UnilateralDataMailbox) { s.poke() },
|
||||||
|
Fetch: func(*imapclient.FetchMessageData) { s.poke() },
|
||||||
|
}
|
||||||
|
c, err := s.k.Dial(s.a, handler)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("its server %s cannot be reached: %v", s.a.Incoming.Host, err)
|
||||||
|
}
|
||||||
|
s.c = c
|
||||||
|
defer c.Close()
|
||||||
|
if err := c.Login(s.a.SignInAs(), password).Wait(); err != nil {
|
||||||
|
var said *imap.Error
|
||||||
|
if errors.As(err, &said) {
|
||||||
|
return fmt.Errorf("its server refused the sign-in (%s): its password may be wrong, and is given again at the desk", strings.TrimSpace(said.Text))
|
||||||
|
}
|
||||||
|
return fmt.Errorf("signing in failed: %v", err)
|
||||||
|
}
|
||||||
|
s.answered()
|
||||||
|
s.tell(AccountSyncing, "signed in; looking at its folders")
|
||||||
|
folders, err := s.folders()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
idles := c.Caps().Has(imap.CapIdle) || c.Caps().Has(imap.CapIMAP4rev2)
|
||||||
|
var polled time.Time
|
||||||
|
for ctx.Err() == nil {
|
||||||
|
if polled.IsZero() || s.k.now().Sub(polled) >= orDefault(s.k.PollEvery, PollEvery) {
|
||||||
|
for _, f := range folders {
|
||||||
|
if err := s.k.syncFolder(ctx, c, s.a, f, s.answered); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
polled = s.k.now()
|
||||||
|
}
|
||||||
|
// The inbox last, so it is the folder selected while the session idles.
|
||||||
|
if err := s.k.syncFolder(ctx, c, s.a, "INBOX", s.answered); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
s.synced = s.k.now()
|
||||||
|
s.tell(AccountCurrent, fmt.Sprintf("watching the inbox and %d other folder(s)", len(folders)))
|
||||||
|
if err := s.wait(ctx, idles); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
_ = c.Logout().Wait()
|
||||||
|
return ctx.Err()
|
||||||
|
}
|
||||||
|
|
||||||
|
// wait is one IDLE on the inbox, ended by what the server says unasked, by the cycle's end or by ctx. Ending it waits
|
||||||
|
// for the server's answer, which is what proves the connection alive.
|
||||||
|
func (s *session) wait(ctx context.Context, idles bool) error {
|
||||||
|
cycle := time.NewTimer(orDefault(s.k.IdleCycle, IdleCycle))
|
||||||
|
defer cycle.Stop()
|
||||||
|
if !idles {
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
case <-cycle.C:
|
||||||
|
case <-s.c.Closed():
|
||||||
|
return errors.New("its server closed the connection")
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
idle, err := s.c.Idle()
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("its server refused to IDLE on the inbox: %v", err)
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
case <-s.wake:
|
||||||
|
case <-cycle.C:
|
||||||
|
case <-s.c.Closed():
|
||||||
|
}
|
||||||
|
if err := idle.Close(); err != nil {
|
||||||
|
return fmt.Errorf("its IDLE on the inbox ended badly: %v", err)
|
||||||
|
}
|
||||||
|
if err := idle.Wait(); err != nil {
|
||||||
|
return fmt.Errorf("its IDLE on the inbox ended badly: %v", err)
|
||||||
|
}
|
||||||
|
s.answered()
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// folders is what is watched besides the inbox: the account's list, or every folder the server can open.
|
||||||
|
func (s *session) folders() ([]string, error) {
|
||||||
|
if len(s.a.Folders) > 0 {
|
||||||
|
return append([]string(nil), s.a.Folders...), nil
|
||||||
|
}
|
||||||
|
listed, err := s.c.List("", "*", nil).Collect()
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("its folders cannot be listed: %v", err)
|
||||||
|
}
|
||||||
|
s.answered()
|
||||||
|
var out []string
|
||||||
|
for _, l := range listed {
|
||||||
|
if strings.EqualFold(l.Mailbox, "INBOX") || hasAttr(l.Attrs, imap.MailboxAttrNoSelect) || hasAttr(l.Attrs, imap.MailboxAttrNonExistent) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
out = append(out, l.Mailbox)
|
||||||
|
}
|
||||||
|
sort.Strings(out)
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func hasAttr(attrs []imap.MailboxAttr, want imap.MailboxAttr) bool {
|
||||||
|
for _, a := range attrs {
|
||||||
|
if strings.EqualFold(string(a), string(want)) {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
// syncFolder brings the index of one folder to what the server holds, and says each difference, unless this is the
|
||||||
|
// first look at the folder.
|
||||||
|
func (k *Keeper) syncFolder(ctx context.Context, c reader, a Account, folder string, answered func()) error {
|
||||||
|
sel, err := c.Select(folder, &imap.SelectOptions{ReadOnly: true}).Wait()
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("the folder %s cannot be opened: %v", folder, err)
|
||||||
|
}
|
||||||
|
answered()
|
||||||
|
known, seen, err := k.Store.Folder(ctx, a.Name, folder)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if seen && known.UIDValidity != sel.UIDValidity {
|
||||||
|
k.logf("[mesh-mail] the account %s's folder %s was renumbered by its server; it is indexed afresh", a.Name, folder)
|
||||||
|
if err := k.Store.DropFolder(ctx, a.Name, folder); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
seen = false
|
||||||
|
}
|
||||||
|
server := map[uint32][]string{}
|
||||||
|
if sel.NumMessages > 0 {
|
||||||
|
all, err := c.Fetch(imap.UIDSet{{Start: 1, Stop: 0}}, &imap.FetchOptions{UID: true, Flags: true}).Collect()
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("the folder %s's messages cannot be listed: %v", folder, err)
|
||||||
|
}
|
||||||
|
answered()
|
||||||
|
for _, m := range all {
|
||||||
|
server[uint32(m.UID)] = flagStrings(m.Flags)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
index, err := k.Store.Flags(ctx, a.Name, folder)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
say := func(e *Event) *Event {
|
||||||
|
if seen {
|
||||||
|
return e
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
// Removed: in the index, no longer on the server.
|
||||||
|
for uid := range index {
|
||||||
|
if _, still := server[uid]; still {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
m, _, err := k.Store.Get(ctx, a.Name, folder, uid)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if m.UID == 0 {
|
||||||
|
m = Message{Account: a.Name, Folder: folder, UID: uid, UIDValidity: sel.UIDValidity}
|
||||||
|
}
|
||||||
|
if err := k.Store.Remove(ctx, a.Name, folder, uid, say(removedEvent(m))); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// Changed: on both, with other flags.
|
||||||
|
var added []uint32
|
||||||
|
for uid, flags := range server {
|
||||||
|
before, indexed := index[uid]
|
||||||
|
if !indexed {
|
||||||
|
added = append(added, uid)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if sameStrings(before, flags) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
m, _, err := k.Store.Get(ctx, a.Name, folder, uid)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
m.Flags = flags
|
||||||
|
if err := k.Store.SetFlags(ctx, a.Name, folder, uid, flags, say(changedEvent(m, before, flags))); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// Added: on the server, not yet in the index. Their sizes first, then their bodies in batches bounded by bytes as
|
||||||
|
// well as by count, so one batch never holds more than BatchBytes of mail in memory; a message larger than that
|
||||||
|
// is fetched alone, and said. Fetched without marking them seen.
|
||||||
|
sort.Slice(added, func(i, j int) bool { return added[i] < added[j] })
|
||||||
|
sizes := map[uint32]int64{}
|
||||||
|
for start := 0; start < len(added); start += SizeBatch {
|
||||||
|
part := added[start:min(start+SizeBatch, len(added))]
|
||||||
|
got, err := c.Fetch(uidSet(part), &imap.FetchOptions{UID: true, RFC822Size: true}).Collect()
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("the folder %s's new messages cannot be measured: %v", folder, err)
|
||||||
|
}
|
||||||
|
answered()
|
||||||
|
for _, m := range got {
|
||||||
|
sizes[uint32(m.UID)] = m.RFC822Size
|
||||||
|
}
|
||||||
|
}
|
||||||
|
cap := orBytes(k.BatchBytes, BatchBytes)
|
||||||
|
for _, batch := range Batches(added, sizes, cap, FetchBatch) {
|
||||||
|
if len(batch) == 1 && sizes[batch[0]] > cap {
|
||||||
|
k.logf("[mesh-mail] the account %s's message %d in %s is %d bytes, more than a batch holds; it is fetched alone",
|
||||||
|
a.Name, batch[0], folder, sizes[batch[0]])
|
||||||
|
}
|
||||||
|
msgs, err := c.Fetch(uidSet(batch), &imap.FetchOptions{
|
||||||
|
UID: true, Flags: true, Envelope: true, RFC822Size: true, InternalDate: true,
|
||||||
|
BodySection: []*imap.FetchItemBodySection{{Peek: true}},
|
||||||
|
}).Collect()
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("the folder %s's new messages cannot be fetched: %v", folder, err)
|
||||||
|
}
|
||||||
|
answered()
|
||||||
|
for _, fm := range msgs {
|
||||||
|
m, err := k.index(ctx, a, folder, sel.UIDValidity, fm)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := k.Store.Add(ctx, m, say(addedEvent(m))); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return k.Store.SetFolder(ctx, Folder{Account: a.Name, Name: folder, UIDValidity: sel.UIDValidity, Messages: len(server), Synced: k.now()})
|
||||||
|
}
|
||||||
|
|
||||||
|
// index is a fetched message as the index keeps it: its metadata, its text, and its attachments kept in the store.
|
||||||
|
func (k *Keeper) index(ctx context.Context, a Account, folder string, validity uint32, fm *imapclient.FetchMessageBuffer) (Message, error) {
|
||||||
|
m := Message{Account: a.Name, Folder: folder, UIDValidity: validity, UID: uint32(fm.UID), Size: fm.RFC822Size,
|
||||||
|
Flags: flagStrings(fm.Flags), Date: fm.InternalDate.UTC(), From: []Person{}, To: []Person{}, Attachments: []AttachmentRef{}}
|
||||||
|
if env := fm.Envelope; env != nil {
|
||||||
|
m.MessageID, m.Subject = env.MessageID, env.Subject
|
||||||
|
if len(env.InReplyTo) > 0 {
|
||||||
|
m.InReplyTo = env.InReplyTo[0]
|
||||||
|
}
|
||||||
|
if !env.Date.IsZero() {
|
||||||
|
m.Date = env.Date.UTC()
|
||||||
|
}
|
||||||
|
m.From, m.To, m.Cc = people(env.From), people(env.To), people(env.Cc)
|
||||||
|
}
|
||||||
|
var raw []byte
|
||||||
|
if len(fm.BodySection) > 0 {
|
||||||
|
raw = fm.BodySection[0].Bytes
|
||||||
|
}
|
||||||
|
parsed, err := ParseMessage(raw)
|
||||||
|
if err != nil {
|
||||||
|
k.logf("[mesh-mail] the account %s's message %d in %s is read in part: %v", a.Name, m.UID, folder, err)
|
||||||
|
}
|
||||||
|
m.Body = parsed.Text
|
||||||
|
for _, p := range parsed.Attachments {
|
||||||
|
ref, err := Keep(ctx, k.Objects, p.Data, p.Type, p.Name)
|
||||||
|
if err != nil {
|
||||||
|
return m, err
|
||||||
|
}
|
||||||
|
m.Attachments = append(m.Attachments, ref)
|
||||||
|
}
|
||||||
|
return m, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func people(as []imap.Address) []Person {
|
||||||
|
out := []Person{}
|
||||||
|
for _, a := range as {
|
||||||
|
if addr := a.Addr(); addr != "" {
|
||||||
|
out = append(out, Person{Name: a.Name, Address: addr})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
func flagStrings(fs []imap.Flag) []string {
|
||||||
|
out := make([]string, 0, len(fs))
|
||||||
|
for _, f := range fs {
|
||||||
|
if strings.EqualFold(string(f), `\Recent`) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
out = append(out, string(f))
|
||||||
|
}
|
||||||
|
sort.Strings(out)
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
func sameStrings(a, b []string) bool {
|
||||||
|
if len(a) != len(b) {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
for i := range a {
|
||||||
|
if a[i] != b[i] {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---- the events ---------------------------------------------------------------------------------------------------
|
||||||
|
// EventKey and Changes, the event's shape, now live in mailevent.go of their own (issue 800 part A).
|
||||||
|
|
||||||
|
// metadata is what an event says of a message: never its body, never an attachment's bytes (ADR 0359 §2).
|
||||||
|
func metadata(m Message) map[string]any {
|
||||||
|
atts := make([]map[string]any, 0, len(m.Attachments))
|
||||||
|
for _, r := range m.Attachments {
|
||||||
|
atts = append(atts, map[string]any{"bucket": r.Bucket, "key": r.Key, "size": r.Size, "type": r.Type, "name": r.Name})
|
||||||
|
}
|
||||||
|
return map[string]any{
|
||||||
|
"account": m.Account, "folder": m.Folder, "uid": m.UID, "uidvalidity": m.UIDValidity,
|
||||||
|
"message-id": m.MessageID, "in-reply-to": m.InReplyTo, "date": m.Date.UTC().Format(time.RFC3339),
|
||||||
|
"from": m.From, "to": m.To, "cc": m.Cc, "subject": m.Subject, "size": m.Size, "flags": m.Flags,
|
||||||
|
"attachments": atts,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func addedEvent(m Message) *Event {
|
||||||
|
return &Event{Key: EventKey(m.Account, "added"), Body: metadata(m)}
|
||||||
|
}
|
||||||
|
|
||||||
|
func changedEvent(m Message, before, after []string) *Event {
|
||||||
|
body := metadata(m)
|
||||||
|
body["changes"] = map[string]any{"flags": map[string]any{"before": before, "after": after}}
|
||||||
|
return &Event{Key: EventKey(m.Account, "changed"), Body: body}
|
||||||
|
}
|
||||||
|
|
||||||
|
func removedEvent(m Message) *Event {
|
||||||
|
return &Event{Key: EventKey(m.Account, "removed"), Body: map[string]any{
|
||||||
|
"account": m.Account, "folder": m.Folder, "uid": m.UID, "uidvalidity": m.UIDValidity, "message-id": m.MessageID,
|
||||||
|
}}
|
||||||
|
}
|
||||||
@@ -0,0 +1,300 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"crypto/sha256"
|
||||||
|
"encoding/hex"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"regexp"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/emersion/go-imap/v2"
|
||||||
|
)
|
||||||
|
|
||||||
|
// keeping starts a keeper on one account of the test server, its paces short, and stops it when the test ends.
|
||||||
|
func keeping(t *testing.T, s *mailServer, st Store, objects Objects, secrets string) {
|
||||||
|
t.Helper()
|
||||||
|
k := &Keeper{Store: st, Objects: objects, Dial: s.dialer(), SecretsDir: secrets, Logf: t.Logf,
|
||||||
|
IdleCycle: 300 * time.Millisecond, PollEvery: 400 * time.Millisecond, RetryFirst: 100 * time.Millisecond}
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
done := make(chan struct{})
|
||||||
|
go func() { k.Watch(ctx, testAccount()); close(done) }()
|
||||||
|
t.Cleanup(func() { cancel(); <-done })
|
||||||
|
}
|
||||||
|
|
||||||
|
func stateOf(st Store, account string) AccountState {
|
||||||
|
states, _ := st.States(context.Background())
|
||||||
|
return states[account]
|
||||||
|
}
|
||||||
|
|
||||||
|
// float64OrUint reads a uid from an event body whether it came from memory (uint32) or from postgres (float64).
|
||||||
|
type float64OrUint struct{ v any }
|
||||||
|
|
||||||
|
func (f float64OrUint) value() uint32 {
|
||||||
|
switch n := f.v.(type) {
|
||||||
|
case uint32:
|
||||||
|
return n
|
||||||
|
case float64:
|
||||||
|
return uint32(n)
|
||||||
|
}
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
func uidOf(e Event) uint32 { return float64OrUint{e.Body["uid"]}.value() }
|
||||||
|
|
||||||
|
func eventsNamed(st Store, key string) []Event {
|
||||||
|
evs, _ := st.Pending(context.Background(), 1000)
|
||||||
|
var out []Event
|
||||||
|
for _, e := range evs {
|
||||||
|
if e.Key == key {
|
||||||
|
out = append(out, e)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
func describe(st Store) string {
|
||||||
|
evs, _ := st.Pending(context.Background(), 1000)
|
||||||
|
var keys []string
|
||||||
|
for _, e := range evs {
|
||||||
|
keys = append(keys, e.Key)
|
||||||
|
}
|
||||||
|
s := stateOf(st, "mesh")
|
||||||
|
return "state " + s.State + " (" + s.Why + "); events " + strings.Join(keys, ", ")
|
||||||
|
}
|
||||||
|
|
||||||
|
// The whole watch, over a real IMAP conversation (ADR 0359 §2, §3, §5): the first look indexes what is there and says
|
||||||
|
// nothing; then a new mail in the inbox is said within moments of its delivery, through IDLE, with its metadata and
|
||||||
|
// its attachment by reference, never its body; its flags changed are said; its removal is said; a mail in another
|
||||||
|
// folder is said at the next poll; and watching never marks a message seen.
|
||||||
|
func TestTheKeeperSaysEachChangeOnTheBusAndOnlyReads(t *testing.T) {
|
||||||
|
eachStore(t, func(t *testing.T, st Store) {
|
||||||
|
s := newMailServer(t)
|
||||||
|
op := s.operator(t)
|
||||||
|
deliver(t, op, "INBOX", plainMail("before the watch", "an old letter"))
|
||||||
|
deliver(t, op, "Archive", plainMail("archived long ago", "an older letter"))
|
||||||
|
secrets := t.TempDir()
|
||||||
|
giveSecret(t, secrets, testAccount(), testPassword)
|
||||||
|
objects := newMemObjects()
|
||||||
|
keeping(t, s, st, objects, secrets)
|
||||||
|
|
||||||
|
eventually(t, 10*time.Second, "the first look", func() (bool, string) {
|
||||||
|
return stateOf(st, "mesh").State == AccountCurrent, describe(st)
|
||||||
|
})
|
||||||
|
ctx := context.Background()
|
||||||
|
if hits, _ := st.Search(ctx, Query{Text: "letter", Accounts: []string{"mesh"}}); len(hits) != 2 {
|
||||||
|
t.Fatalf("the first look indexed %d messages, not 2", len(hits))
|
||||||
|
}
|
||||||
|
if evs, _ := st.Pending(ctx, 100); len(evs) != 0 {
|
||||||
|
t.Fatalf("the first look said %d event(s): the history of a mailbox is not news", len(evs))
|
||||||
|
}
|
||||||
|
|
||||||
|
// A new mail with an attachment, delivered while the keeper idles on the inbox.
|
||||||
|
deliver(t, op, "INBOX", attachedMail("the invoice"))
|
||||||
|
var added Event
|
||||||
|
eventually(t, 5*time.Second, "the new mail said", func() (bool, string) {
|
||||||
|
evs := eventsNamed(st, "mail.mesh.message.added")
|
||||||
|
if len(evs) == 1 {
|
||||||
|
added = evs[0]
|
||||||
|
}
|
||||||
|
return len(evs) == 1, describe(st)
|
||||||
|
})
|
||||||
|
body := added.Body
|
||||||
|
if body["subject"] != "the invoice" || body["folder"] != "INBOX" || body["account"] != "mesh" || body["message-id"] != "att@example.org" {
|
||||||
|
t.Fatalf("the event says %v", body)
|
||||||
|
}
|
||||||
|
raw, _ := json.Marshal(body)
|
||||||
|
for _, never := range []string{"Please find", "dGhlIGludm9pY2Ug", attachmentData} {
|
||||||
|
if strings.Contains(string(raw), never) {
|
||||||
|
t.Fatalf("the event carries the body or an attachment's bytes (%q): %s", never, raw)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
sum := sha256.Sum256([]byte(attachmentData))
|
||||||
|
key := hex.EncodeToString(sum[:])
|
||||||
|
if !strings.Contains(string(raw), `"key":"`+key+`"`) || !strings.Contains(string(raw), `"name":"invoice.pdf"`) {
|
||||||
|
t.Fatalf("the event does not name the attachment by its SHA-256 %s: %s", key, raw)
|
||||||
|
}
|
||||||
|
if kept, _ := objects.Get(ctx, key); string(kept) != attachmentData {
|
||||||
|
t.Fatalf("the attachment store keeps %q under %s", kept, key)
|
||||||
|
}
|
||||||
|
uid := uidOf(added)
|
||||||
|
m, ok, _ := st.Get(ctx, "mesh", "INBOX", uid)
|
||||||
|
if !ok || !strings.Contains(m.Body, "Please find the invoice attached.") {
|
||||||
|
t.Fatalf("the index keeps the text for mail_read: %+v", m)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Flagged by the operator elsewhere: said as changed, with the flags before and after.
|
||||||
|
if _, err := op.Select("INBOX", nil).Wait(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := op.Store(imap.UIDSetNum(imap.UID(uid)), &imap.StoreFlags{Op: imap.StoreFlagsAdd, Flags: []imap.Flag{imap.FlagFlagged}, Silent: true}, nil).Close(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
eventually(t, 5*time.Second, "the flag said", func() (bool, string) {
|
||||||
|
evs := eventsNamed(st, "mail.mesh.message.changed")
|
||||||
|
if len(evs) != 1 {
|
||||||
|
return false, describe(st)
|
||||||
|
}
|
||||||
|
raw, _ := json.Marshal(evs[0].Body["changes"])
|
||||||
|
return strings.Contains(string(raw), `"after":["\\Flagged"]`), string(raw)
|
||||||
|
})
|
||||||
|
|
||||||
|
// Never marked seen by the watching: the message is still unseen on the server.
|
||||||
|
fetched, err := op.Fetch(imap.UIDSetNum(imap.UID(uid)), &imap.FetchOptions{Flags: true, UID: true}).Collect()
|
||||||
|
if err != nil || len(fetched) != 1 {
|
||||||
|
t.Fatalf("%v %v", fetched, err)
|
||||||
|
}
|
||||||
|
for _, f := range fetched[0].Flags {
|
||||||
|
if f == imap.FlagSeen {
|
||||||
|
t.Fatal("watching marked the message seen")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Deleted by the operator elsewhere: said as removed.
|
||||||
|
if err := op.Store(imap.UIDSetNum(imap.UID(uid)), &imap.StoreFlags{Op: imap.StoreFlagsAdd, Flags: []imap.Flag{imap.FlagDeleted}, Silent: true}, nil).Close(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := op.Expunge().Close(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
eventually(t, 5*time.Second, "the removal said", func() (bool, string) {
|
||||||
|
evs := eventsNamed(st, "mail.mesh.message.removed")
|
||||||
|
return len(evs) == 1 && uidOf(evs[0]) == uid && evs[0].Body["message-id"] == "att@example.org", describe(st)
|
||||||
|
})
|
||||||
|
if _, ok, _ := st.Get(ctx, "mesh", "INBOX", uid); ok {
|
||||||
|
t.Fatal("the removed message is still indexed")
|
||||||
|
}
|
||||||
|
|
||||||
|
// A mail in another folder is said at the next poll.
|
||||||
|
deliver(t, op, "Archive", plainMail("filed away", "a letter filed"))
|
||||||
|
eventually(t, 5*time.Second, "the archived mail said", func() (bool, string) {
|
||||||
|
for _, e := range eventsNamed(st, "mail.mesh.message.added") {
|
||||||
|
if e.Body["folder"] == "Archive" && e.Body["subject"] == "filed away" {
|
||||||
|
return true, ""
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false, describe(st)
|
||||||
|
})
|
||||||
|
if s := stateOf(st, "mesh"); s.State != AccountCurrent || time.Since(s.Heard) > time.Minute {
|
||||||
|
t.Fatalf("the account is %+v", s)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// No password given: the account waits, and nothing is dialled. A wrong one: the account fails, saying the server
|
||||||
|
// refused the sign-in. Given later: it is picked up without a restart.
|
||||||
|
func TestTheKeeperWaitsForThePasswordAndSaysARefusal(t *testing.T) {
|
||||||
|
s := newMailServer(t)
|
||||||
|
st := newMemStore()
|
||||||
|
secrets := t.TempDir()
|
||||||
|
keeping(t, s, st, newMemObjects(), secrets)
|
||||||
|
eventually(t, 5*time.Second, "the wait", func() (bool, string) {
|
||||||
|
got := stateOf(st, "mesh")
|
||||||
|
return got.State == AccountWaits && strings.Contains(got.Why, "mail-credential-mesh"), describe(st)
|
||||||
|
})
|
||||||
|
giveSecret(t, secrets, testAccount(), "not-the-password")
|
||||||
|
eventually(t, 5*time.Second, "the refusal", func() (bool, string) {
|
||||||
|
got := stateOf(st, "mesh")
|
||||||
|
return got.State == AccountFailing && strings.Contains(got.Why, "refused the sign-in"), describe(st)
|
||||||
|
})
|
||||||
|
giveSecret(t, secrets, testAccount(), testPassword)
|
||||||
|
eventually(t, 10*time.Second, "the sign-in", func() (bool, string) {
|
||||||
|
return stateOf(st, "mesh").State == AccountCurrent, describe(st)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// It only reads a mailbox (ADR 0359 §5): no code of the module, but its tests, issues a command that flags, moves,
|
||||||
|
// copies, deletes, appends, creates, renames or otherwise writes to one over IMAP, and every folder is opened
|
||||||
|
// read-only and every body peeked. mail_send is the one exception the operator's own Approve carves out (ADR 0361
|
||||||
|
// §2): it never touches a mailbox either — it speaks SMTP, in smtp.go alone, to the account's outgoing server, and
|
||||||
|
// only once the operator approved it (send.go).
|
||||||
|
func TestNothingButTheTestsWritesToAMailbox(t *testing.T) {
|
||||||
|
files, err := filepath.Glob("*.go")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
writes := regexp.MustCompile(`([A-Za-z_][A-Za-z0-9_]*)\.(Store|Move|Copy|Expunge|UIDExpunge|Append|Create|Delete|Rename|Subscribe|Unsubscribe|SetMetadata|SetQuota)\(`)
|
||||||
|
for _, f := range files {
|
||||||
|
if strings.HasSuffix(f, "_test.go") || f == "send.go" || f == "smtp.go" {
|
||||||
|
// Neither touches a mailbox: send.go keeps the `sends` state on the bus, never IMAP; smtp.go speaks SMTP
|
||||||
|
// to the account's outgoing server, never to a mailbox, and only on the operator's Approve (ADR 0361 §2).
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
raw, err := os.ReadFile(f)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for _, m := range writes.FindAllSubmatch(raw, -1) {
|
||||||
|
// os is a file of the module's own, never a mailbox; stdio is the bus SDK (main.go's hear subscribes to
|
||||||
|
// the router's own decided subject, exactly as mesh-knowledge does, novox/hq issue 790 review round 1)
|
||||||
|
// — its Subscribe has nothing to do with an IMAP mailbox's.
|
||||||
|
if string(m[1]) != "os" && string(m[1]) != "stdio" {
|
||||||
|
t.Errorf("%s calls %s, which writes to a mailbox", f, m[0])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if strings.Contains(string(raw), "net/smtp") {
|
||||||
|
t.Errorf("%s speaks SMTP: only smtp.go does, and only on the operator's Approve (ADR 0361 §2)", f)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
src, _ := os.ReadFile("imap.go")
|
||||||
|
if !strings.Contains(string(src), "&imap.SelectOptions{ReadOnly: true}") || !strings.Contains(string(src), "{{Peek: true}}") {
|
||||||
|
t.Fatal("folders are not opened read-only, or bodies not peeked")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A fetch of bodies is bounded by bytes (issue 733): batches cut by size and by count, and a message larger than a
|
||||||
|
// batch holds alone.
|
||||||
|
func TestBatchesAreBoundedByBytesAndALargerMessageGoesAlone(t *testing.T) {
|
||||||
|
sizes := map[uint32]int64{1: 40, 2: 40, 3: 500, 4: 10, 5: 10, 6: 10}
|
||||||
|
got := Batches([]uint32{1, 2, 3, 4, 5, 6}, sizes, 100, 2)
|
||||||
|
want := [][]uint32{{1, 2}, {3}, {4, 5}, {6}}
|
||||||
|
if fmt.Sprint(got) != fmt.Sprint(want) {
|
||||||
|
t.Fatalf("%v, not %v", got, want)
|
||||||
|
}
|
||||||
|
if got := Batches(nil, nil, 100, 25); len(got) != 0 {
|
||||||
|
t.Fatalf("%v", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Over a real conversation: with a cap of a few hundred bytes, a large mail is fetched alone and said so, and every
|
||||||
|
// mail is still indexed and said.
|
||||||
|
func TestTheKeeperFetchesUnderAByteCap(t *testing.T) {
|
||||||
|
s := newMailServer(t)
|
||||||
|
op := s.operator(t)
|
||||||
|
st := newMemStore()
|
||||||
|
secrets := t.TempDir()
|
||||||
|
giveSecret(t, secrets, testAccount(), testPassword)
|
||||||
|
var mu sync.Mutex
|
||||||
|
var said []string
|
||||||
|
k := &Keeper{Store: st, Objects: newMemObjects(), Dial: s.dialer(), SecretsDir: secrets,
|
||||||
|
Logf: func(f string, a ...any) { mu.Lock(); said = append(said, fmt.Sprintf(f, a...)); mu.Unlock() },
|
||||||
|
IdleCycle: 300 * time.Millisecond, PollEvery: 400 * time.Millisecond, RetryFirst: 100 * time.Millisecond, BatchBytes: 600}
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
done := make(chan struct{})
|
||||||
|
go func() { k.Watch(ctx, testAccount()); close(done) }()
|
||||||
|
t.Cleanup(func() { cancel(); <-done })
|
||||||
|
eventually(t, 10*time.Second, "the first look", func() (bool, string) { return stateOf(st, "mesh").State == AccountCurrent, describe(st) })
|
||||||
|
deliver(t, op, "INBOX", plainMail("small one", "short"))
|
||||||
|
deliver(t, op, "INBOX", plainMail("large one", strings.Repeat("a long letter ", 200)))
|
||||||
|
deliver(t, op, "INBOX", plainMail("small two", "short"))
|
||||||
|
eventually(t, 5*time.Second, "three mails said", func() (bool, string) {
|
||||||
|
return len(eventsNamed(st, "mail.mesh.message.added")) == 3, describe(st)
|
||||||
|
})
|
||||||
|
mu.Lock()
|
||||||
|
defer mu.Unlock()
|
||||||
|
alone := 0
|
||||||
|
for _, line := range said {
|
||||||
|
if strings.Contains(line, "fetched alone") {
|
||||||
|
alone++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if alone != 1 {
|
||||||
|
t.Fatalf("said %d message(s) fetched alone: %v", alone, said)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,252 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
// What the tests stand on: an IMAP server in this process (go-imap's own in-memory server, speaking the protocol over
|
||||||
|
// a real socket), the index in memory always and on a real postgres when one can be had (MESH_TEST_POSTGRES, or a
|
||||||
|
// private server started here from the machine's initdb on a socket of its own, listening on no port), and the
|
||||||
|
// attachment store in memory.
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"net"
|
||||||
|
"net/url"
|
||||||
|
"os"
|
||||||
|
"os/exec"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/emersion/go-imap/v2"
|
||||||
|
"github.com/emersion/go-imap/v2/imapclient"
|
||||||
|
"github.com/emersion/go-imap/v2/imapserver"
|
||||||
|
"github.com/emersion/go-imap/v2/imapserver/imapmemserver"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestMain(m *testing.M) {
|
||||||
|
code := m.Run()
|
||||||
|
stopPostgres()
|
||||||
|
os.Exit(code)
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---- the IMAP server --------------------------------------------------------------------------------------------
|
||||||
|
|
||||||
|
const (
|
||||||
|
testUser = "operator@example.org"
|
||||||
|
testPassword = "the-password"
|
||||||
|
)
|
||||||
|
|
||||||
|
type mailServer struct {
|
||||||
|
addr string
|
||||||
|
srv *imapserver.Server
|
||||||
|
}
|
||||||
|
|
||||||
|
// newMailServer is a server holding one user with an inbox and an Archive folder.
|
||||||
|
func newMailServer(t *testing.T) *mailServer {
|
||||||
|
t.Helper()
|
||||||
|
mem := imapmemserver.New()
|
||||||
|
user := imapmemserver.NewUser(testUser, testPassword)
|
||||||
|
for _, f := range []string{"INBOX", "Archive"} {
|
||||||
|
if err := user.Create(f, nil); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
mem.AddUser(user)
|
||||||
|
srv := imapserver.New(&imapserver.Options{
|
||||||
|
NewSession: func(*imapserver.Conn) (imapserver.Session, *imapserver.GreetingData, error) {
|
||||||
|
return mem.NewSession(), nil, nil
|
||||||
|
},
|
||||||
|
InsecureAuth: true,
|
||||||
|
Caps: imap.CapSet{imap.CapIMAP4rev1: {}, imap.CapIMAP4rev2: {}},
|
||||||
|
})
|
||||||
|
ln, err := net.Listen("tcp", "127.0.0.1:0")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
go func() { _ = srv.Serve(ln) }()
|
||||||
|
t.Cleanup(func() { _ = srv.Close() })
|
||||||
|
return &mailServer{addr: ln.Addr().String(), srv: srv}
|
||||||
|
}
|
||||||
|
|
||||||
|
// dialer reaches the test server without TLS, which only a test does: the mesh's dialer signs in over TLS alone.
|
||||||
|
func (s *mailServer) dialer() Dialer {
|
||||||
|
return func(_ Account, h *imapclient.UnilateralDataHandler) (reader, error) {
|
||||||
|
c, err := imapclient.DialInsecure(s.addr, &imapclient.Options{UnilateralDataHandler: h})
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return c, nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// operator is a second client, as the operator's mail program or another sender is: it appends, flags and deletes.
|
||||||
|
func (s *mailServer) operator(t *testing.T) *imapclient.Client {
|
||||||
|
t.Helper()
|
||||||
|
c, err := imapclient.DialInsecure(s.addr, nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
t.Cleanup(func() { _ = c.Close() })
|
||||||
|
if err := c.Login(testUser, testPassword).Wait(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return c
|
||||||
|
}
|
||||||
|
|
||||||
|
func deliver(t *testing.T, c *imapclient.Client, folder, raw string) {
|
||||||
|
t.Helper()
|
||||||
|
raw = strings.ReplaceAll(raw, "\n", "\r\n")
|
||||||
|
cmd := c.Append(folder, int64(len(raw)), nil)
|
||||||
|
if _, err := cmd.Write([]byte(raw)); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := cmd.Close(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := cmd.Wait(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func plainMail(subject, body string) string {
|
||||||
|
return "From: Alice <alice@example.org>\nTo: Operator <operator@example.org>\nSubject: " + subject +
|
||||||
|
"\nMessage-Id: <" + strings.ReplaceAll(subject, " ", "-") + "@example.org>\nDate: Wed, 14 Oct 2026 10:00:00 +0000\n" +
|
||||||
|
"MIME-Version: 1.0\nContent-Type: text/plain; charset=utf-8\n\n" + body + "\n"
|
||||||
|
}
|
||||||
|
|
||||||
|
// attachmentData is an attachment's content, and attachedMail a message carrying it.
|
||||||
|
const attachmentData = "the invoice, in full"
|
||||||
|
|
||||||
|
func attachedMail(subject string) string {
|
||||||
|
return "From: Bob <bob@example.org>\nTo: operator@example.org\nSubject: " + subject + "\nMessage-Id: <att@example.org>\n" +
|
||||||
|
"Date: Wed, 14 Oct 2026 11:00:00 +0000\nMIME-Version: 1.0\nContent-Type: multipart/mixed; boundary=XYZ\n\n" +
|
||||||
|
"--XYZ\nContent-Type: text/plain; charset=utf-8\n\nPlease find the invoice attached.\n" +
|
||||||
|
"--XYZ\nContent-Type: application/pdf; name=invoice.pdf\nContent-Disposition: attachment; filename=invoice.pdf\n" +
|
||||||
|
"Content-Transfer-Encoding: base64\n\ndGhlIGludm9pY2UsIGluIGZ1bGw=\n--XYZ--\n"
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---- the account ------------------------------------------------------------------------------------------------
|
||||||
|
|
||||||
|
func testAccount() Account {
|
||||||
|
return Account{Name: "mesh", Kind: KindIMAP, Address: testUser, Incoming: Server{Host: "localhost", Port: 993, Security: "tls"}}
|
||||||
|
}
|
||||||
|
|
||||||
|
func giveSecret(t *testing.T, dir string, a Account, pw string) {
|
||||||
|
t.Helper()
|
||||||
|
if err := os.WriteFile(SecretPath(dir, a), []byte(pw+"\n"), 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// eventually waits for a condition, failing with what it last saw.
|
||||||
|
func eventually(t *testing.T, within time.Duration, what string, ok func() (bool, string)) {
|
||||||
|
t.Helper()
|
||||||
|
deadline := time.Now().Add(within)
|
||||||
|
last := ""
|
||||||
|
for time.Now().Before(deadline) {
|
||||||
|
var done bool
|
||||||
|
if done, last = ok(); done {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
time.Sleep(20 * time.Millisecond)
|
||||||
|
}
|
||||||
|
t.Fatalf("%s did not happen within %s; last seen: %s", what, within, last)
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---- postgres ---------------------------------------------------------------------------------------------------
|
||||||
|
|
||||||
|
var (
|
||||||
|
pgOnce sync.Once
|
||||||
|
pgBase string
|
||||||
|
pgDir string
|
||||||
|
pgWhy string
|
||||||
|
pgCount int
|
||||||
|
pgMu sync.Mutex
|
||||||
|
)
|
||||||
|
|
||||||
|
func startPostgres() {
|
||||||
|
if u := os.Getenv("MESH_TEST_POSTGRES"); u != "" {
|
||||||
|
pgBase = u
|
||||||
|
return
|
||||||
|
}
|
||||||
|
initdb, err := exec.LookPath("initdb")
|
||||||
|
if err != nil {
|
||||||
|
pgWhy = "no postgres: MESH_TEST_POSTGRES is unset and the machine has no initdb"
|
||||||
|
return
|
||||||
|
}
|
||||||
|
pgctl := filepath.Join(filepath.Dir(initdb), "pg_ctl")
|
||||||
|
dir, err := os.MkdirTemp("", "mmpg")
|
||||||
|
if err != nil {
|
||||||
|
pgWhy = err.Error()
|
||||||
|
return
|
||||||
|
}
|
||||||
|
pgDir = dir
|
||||||
|
if out, err := exec.Command(initdb, "-D", dir+"/data", "-U", "postgres", "-A", "trust", "-E", "UTF8", "--no-locale").CombinedOutput(); err != nil {
|
||||||
|
pgWhy = fmt.Sprintf("initdb: %v: %s", err, out)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if out, err := exec.Command(pgctl, "-D", dir+"/data", "-l", dir+"/log", "-w", "-o",
|
||||||
|
"-k "+dir+" -c listen_addresses='' -F", "start").CombinedOutput(); err != nil {
|
||||||
|
pgWhy = fmt.Sprintf("pg_ctl start: %v: %s", err, out)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
pgBase = "postgres://postgres@/postgres?host=" + dir
|
||||||
|
}
|
||||||
|
|
||||||
|
func stopPostgres() {
|
||||||
|
if pgDir == "" {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if initdb, err := exec.LookPath("initdb"); err == nil {
|
||||||
|
_ = exec.Command(filepath.Join(filepath.Dir(initdb), "pg_ctl"), "-D", pgDir+"/data", "-m", "immediate", "stop").Run()
|
||||||
|
}
|
||||||
|
_ = os.RemoveAll(pgDir)
|
||||||
|
}
|
||||||
|
|
||||||
|
// freshDatabase is a database of its own for one test, or a skip saying why there is none.
|
||||||
|
func freshDatabase(t *testing.T) string {
|
||||||
|
t.Helper()
|
||||||
|
pgOnce.Do(startPostgres)
|
||||||
|
if pgBase == "" {
|
||||||
|
t.Skipf("NOT TESTED ON POSTGRES: %s", pgWhy)
|
||||||
|
}
|
||||||
|
pgMu.Lock()
|
||||||
|
pgCount++
|
||||||
|
name := fmt.Sprintf("mail_%d_%d", os.Getpid(), pgCount)
|
||||||
|
pgMu.Unlock()
|
||||||
|
ctx := context.Background()
|
||||||
|
admin, err := OpenPgStore(ctx, pgBase)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer admin.Close()
|
||||||
|
if _, err := admin.pool.Exec(ctx, "create database "+name); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
u, err := url.Parse(pgBase)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
u.Path = "/" + name
|
||||||
|
return u.String()
|
||||||
|
}
|
||||||
|
|
||||||
|
func pgStore(t *testing.T) *PgStore {
|
||||||
|
t.Helper()
|
||||||
|
s, err := OpenPgStore(context.Background(), freshDatabase(t))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
t.Cleanup(s.Close)
|
||||||
|
if _, err := s.Prepare(context.Background()); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return s
|
||||||
|
}
|
||||||
|
|
||||||
|
// eachStore runs a test over the index in memory and on postgres.
|
||||||
|
func eachStore(t *testing.T, run func(t *testing.T, s Store)) {
|
||||||
|
t.Run("memory", func(t *testing.T) { run(t, newMemStore()) })
|
||||||
|
t.Run("postgres", func(t *testing.T) { run(t, pgStore(t)) })
|
||||||
|
}
|
||||||
@@ -0,0 +1,13 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
// The shape of a mail-store message event (novox/hq ADR 0378 §2, issue 800 part A), moved here on its own so
|
||||||
|
// mesh-mail-notifier (issue 800 part B) carries a byte-identical copy, the way modules/networkmanager's
|
||||||
|
// uplink-tools carries ifwatchd's own file (novox/hq ADR 0241, copies_test.go). Nothing here reaches a mailbox or
|
||||||
|
// a server; it only says where an account's change is told on the bus and what the three changes are called.
|
||||||
|
|
||||||
|
// EventKey is where an account's change is said: mail.<account>.message.<added|changed|removed> (research 049
|
||||||
|
// document 02 §5), the account by its name in the setting, never its address.
|
||||||
|
func EventKey(account, change string) string { return "mail." + account + ".message." + change }
|
||||||
|
|
||||||
|
// Changes are the three things said of a message.
|
||||||
|
var Changes = []string{"added", "changed", "removed"}
|
||||||
@@ -0,0 +1,291 @@
|
|||||||
|
// mesh-mail: the operator's mail accounts, watched, and each mail said on the bus (novox/hq ADR 0359). One instance,
|
||||||
|
// on the control-node. Reading is read-only: no flag, move, delete or draft. mail_send is the one way out (novox/hq
|
||||||
|
// ADR 0361 §2): it sends nothing itself, and reaches an account's outgoing server only on the operator's Approve.
|
||||||
|
//
|
||||||
|
// mesh-mail the bundle the node's runtime launches: the tools mail_accounts, mail_search, mail_read,
|
||||||
|
// mail_health, mail_send and mail_send_status, and the events the keeper wrote, said on the
|
||||||
|
// bus as the module
|
||||||
|
// mesh-mail prepare the preparation step: the index's schema brought to this version's
|
||||||
|
// mesh-mail keep acts: watches every account (IDLE on the inbox, a look at the other folders every five
|
||||||
|
// minutes), keeps the index and the attachment store, and writes each change's event beside it
|
||||||
|
// mesh-mail watch judges: reads every minute how each account is, and leaves its verdict for mail_health
|
||||||
|
//
|
||||||
|
// The two long-running processes are the act and the judge, as mesh-knowledge's are: keep is judged by its unit, watch
|
||||||
|
// by the tool mail_health (ADR 0359 §6, ADR 0240 §3 and §7). Only the bundle reaches the bus, through the runtime that
|
||||||
|
// launched it, so the keeper hands its events over in the index, written in the same step as the change they say,
|
||||||
|
// and the bundle says them and removes each once the bus took it. stdout is the bundle's MCP channel; what this module
|
||||||
|
// says, it says on stderr.
|
||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"os/signal"
|
||||||
|
"sync"
|
||||||
|
"syscall"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Module is this module's name.
|
||||||
|
const Module = "mesh-mail"
|
||||||
|
|
||||||
|
// Env is where the mesh puts what the module reads, as the manifest's environment names it.
|
||||||
|
type Env struct {
|
||||||
|
Accounts string // the trusted setting accounts
|
||||||
|
Grants string // the trusted setting grants
|
||||||
|
Secrets string // the directory the own secrets mail-credential-<account> are placed in
|
||||||
|
Store string // the attachment store's facts
|
||||||
|
Status string // the watch's verdict
|
||||||
|
}
|
||||||
|
|
||||||
|
func envFromOS() Env {
|
||||||
|
return Env{
|
||||||
|
Accounts: os.Getenv("MESH_MAIL_ACCOUNTS"),
|
||||||
|
Grants: os.Getenv("MESH_MAIL_GRANTS"),
|
||||||
|
Secrets: os.Getenv("MESH_MAIL_SECRETS"),
|
||||||
|
Store: os.Getenv("MESH_MAIL_STORE"),
|
||||||
|
Status: os.Getenv("MESH_MAIL_STATUS"),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func logf(format string, a ...any) { fmt.Fprintf(os.Stderr, format+"\n", a...) }
|
||||||
|
|
||||||
|
func main() {
|
||||||
|
if len(os.Args) > 1 {
|
||||||
|
switch os.Args[1] {
|
||||||
|
case "prepare":
|
||||||
|
os.Exit(prepare())
|
||||||
|
case "keep":
|
||||||
|
os.Exit(keep(envFromOS()))
|
||||||
|
case "watch":
|
||||||
|
os.Exit(watch(envFromOS()))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
serve(envFromOS())
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---- prepare ------------------------------------------------------------------------------------------------------
|
||||||
|
|
||||||
|
func prepare() int {
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
|
||||||
|
defer cancel()
|
||||||
|
s, err := PgStoreFromEnv(ctx)
|
||||||
|
if err != nil {
|
||||||
|
logf("[mesh-mail] preparing the index: %v", err)
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
defer s.Close()
|
||||||
|
said, err := s.Prepare(ctx)
|
||||||
|
for _, line := range said {
|
||||||
|
logf("[mesh-mail] %s", line)
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
logf("[mesh-mail] preparing the index failed: %v", err)
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---- keep ---------------------------------------------------------------------------------------------------------
|
||||||
|
|
||||||
|
// keep watches every account until it is stopped. A setting it cannot read, an index that is behind or a store it
|
||||||
|
// cannot reach ends it with a fault, which its unit says; the mesh starts it again.
|
||||||
|
func keep(env Env) int {
|
||||||
|
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
|
||||||
|
defer stop()
|
||||||
|
accounts, err := ReadAccounts(env.Accounts)
|
||||||
|
if err != nil {
|
||||||
|
logf("[mesh-mail] %v", err)
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
st, err := PgStoreFromEnv(ctx)
|
||||||
|
if err != nil {
|
||||||
|
logf("[mesh-mail] the index cannot be opened: %v", err)
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
defer st.Close()
|
||||||
|
if err := st.Ready(ctx); err != nil {
|
||||||
|
logf("[mesh-mail] %v", err)
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
facts, err := ReadStoreFacts(env.Store)
|
||||||
|
if err != nil {
|
||||||
|
logf("[mesh-mail] %v", err)
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
k := &Keeper{Store: st, Objects: NewS3(facts), Dial: DialServer, SecretsDir: env.Secrets, Logf: logf}
|
||||||
|
logf("[mesh-mail] keeping %d account(s)", len(accounts))
|
||||||
|
k.Run(ctx, accounts)
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---- watch --------------------------------------------------------------------------------------------------------
|
||||||
|
|
||||||
|
func watch(env Env) int {
|
||||||
|
if env.Status == "" {
|
||||||
|
logf("[mesh-mail] watch was started without MESH_MAIL_STATUS")
|
||||||
|
return 2
|
||||||
|
}
|
||||||
|
open := func(ctx context.Context) (Store, func(), error) {
|
||||||
|
s, err := PgStoreFromEnv(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, nil, err
|
||||||
|
}
|
||||||
|
return s, s.Close, nil
|
||||||
|
}
|
||||||
|
for {
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||||
|
now := time.Now().UTC()
|
||||||
|
answer := watchOnce(ctx, env, open, now)
|
||||||
|
cancel()
|
||||||
|
if err := writeStatus(env.Status, Status{Answer: answer, At: now}); err != nil {
|
||||||
|
logf("[mesh-mail] the verdict could not be written: %v", err)
|
||||||
|
}
|
||||||
|
time.Sleep(WatchEvery)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---- the bundle ---------------------------------------------------------------------------------------------------
|
||||||
|
|
||||||
|
func serve(env Env) {
|
||||||
|
var mu sync.Mutex
|
||||||
|
var st *PgStore
|
||||||
|
b := &Bundle{
|
||||||
|
Env: env,
|
||||||
|
Index: func(ctx context.Context) (Store, error) {
|
||||||
|
mu.Lock()
|
||||||
|
defer mu.Unlock()
|
||||||
|
if st == nil {
|
||||||
|
s, err := PgStoreFromEnv(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
st = s
|
||||||
|
}
|
||||||
|
if err := st.Ready(ctx); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return st, nil
|
||||||
|
},
|
||||||
|
Objects: func() (Objects, error) {
|
||||||
|
facts, err := ReadStoreFacts(env.Store)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return NewS3(facts), nil
|
||||||
|
},
|
||||||
|
}
|
||||||
|
b.Logf = logf
|
||||||
|
p := &Publisher{Index: b.Index, Emit: func(key string, body any) error { return stdio.Emit(key, body) }, Logf: logf}
|
||||||
|
go p.Run(context.Background())
|
||||||
|
go hear(b)
|
||||||
|
go sweepSends(b)
|
||||||
|
if err := stdio.Serve("", b.Tools()); err != nil {
|
||||||
|
logf("%v", err)
|
||||||
|
os.Exit(1)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// hear subscribes to the router's word on this module's own asks, asked again until the runtime takes it, exactly
|
||||||
|
// as mesh-knowledge does. Each warrant heard goes to Heard, the same Decide that mail_send_status uses: an
|
||||||
|
// approved send goes out as soon as the operator answers it, never only when something later calls
|
||||||
|
// mail_send_status (novox/hq ADR 0361 §2, review round 1 of issue 790).
|
||||||
|
func hear(b *Bundle) {
|
||||||
|
for wait := 2 * time.Second; ; wait = min(wait*2, time.Minute) {
|
||||||
|
time.Sleep(wait)
|
||||||
|
err := stdio.Subscribe(asks.Seat+".decided."+Module, func(e stdio.Envelope) error {
|
||||||
|
var w asks.Warrant
|
||||||
|
if json.Unmarshal(e.Body, &w) == nil {
|
||||||
|
b.Heard(w)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
if err == nil {
|
||||||
|
logf("[mesh-mail] hears the operator's answers")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
logf("[mesh-mail] does not hear the operator's answers yet (%v); asking again in %s", err, wait)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// sweepSends reads every send still undecided once a minute and asks the router's own record for its answer, so
|
||||||
|
// an Approve answered while this module could not hear it directly is still acted on (novox/hq issue 790, review
|
||||||
|
// round 1).
|
||||||
|
func sweepSends(b *Bundle) {
|
||||||
|
tick := time.NewTicker(SweepEvery)
|
||||||
|
defer tick.Stop()
|
||||||
|
for {
|
||||||
|
b.Sweep(time.Now())
|
||||||
|
<-tick.C
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Publisher says the events the keeper wrote, oldest first, and removes each once the bus took it. One the bus did not
|
||||||
|
// take stays, and is said again on the next round: delivery is at least once, and a consumer reads the uid and the
|
||||||
|
// folder to know a repeat.
|
||||||
|
type Publisher struct {
|
||||||
|
Index func(ctx context.Context) (Store, error)
|
||||||
|
Emit func(key string, body any) error
|
||||||
|
Logf func(format string, a ...any)
|
||||||
|
Every time.Duration
|
||||||
|
}
|
||||||
|
|
||||||
|
// PublishBatch is how many events one round says at most.
|
||||||
|
const PublishBatch = 100
|
||||||
|
|
||||||
|
// Once says what is pending, and answers how many it said.
|
||||||
|
func (p *Publisher) Once(ctx context.Context) (int, error) {
|
||||||
|
st, err := p.Index(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
pending, err := st.Pending(ctx, PublishBatch)
|
||||||
|
if err != nil {
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
var said []int64
|
||||||
|
var failed error
|
||||||
|
for _, e := range pending {
|
||||||
|
if err := p.Emit(e.Key, e.Body); err != nil {
|
||||||
|
failed = fmt.Errorf("the bus did not take %s: %v", e.Key, err)
|
||||||
|
break
|
||||||
|
}
|
||||||
|
said = append(said, e.ID)
|
||||||
|
}
|
||||||
|
if len(said) > 0 {
|
||||||
|
if err := st.Said(ctx, said); err != nil {
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return len(said), failed
|
||||||
|
}
|
||||||
|
|
||||||
|
// Run says pending events until ctx ends: every few seconds, at once again while a round was full.
|
||||||
|
func (p *Publisher) Run(ctx context.Context) {
|
||||||
|
every := p.Every
|
||||||
|
if every <= 0 {
|
||||||
|
every = 3 * time.Second
|
||||||
|
}
|
||||||
|
// The runtime takes the bundle's first calls only once Serve runs.
|
||||||
|
time.Sleep(every)
|
||||||
|
for ctx.Err() == nil {
|
||||||
|
c, cancel := context.WithTimeout(ctx, time.Minute)
|
||||||
|
n, err := p.Once(c)
|
||||||
|
cancel()
|
||||||
|
if err != nil && p.Logf != nil {
|
||||||
|
p.Logf("[mesh-mail] events wait to be said: %v", err)
|
||||||
|
}
|
||||||
|
if n == PublishBatch {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
case <-time.After(every):
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,182 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"reflect"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The manifest says what the code does (novox/hq ADR 0359): the database and the bucket it requires (§3), the own
|
||||||
|
// secret family issued outside the mesh (§5, ADR 0283 §6), the two trusted settings (§4), the tools (§5), the events,
|
||||||
|
// its two long-running processes with their health (§6), and — ADR 0378 as amended by ADR 0379, issue 800 part A —
|
||||||
|
// the three mail seats it declares and the one, mail-store, it claims. It is assigned to no machine by itself.
|
||||||
|
|
||||||
|
type seat struct {
|
||||||
|
Name string `json:"name"`
|
||||||
|
Scope string `json:"scope"`
|
||||||
|
Serves []string `json:"serves"`
|
||||||
|
Needs []string `json:"needs"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type manifest struct {
|
||||||
|
Module string `json:"module"`
|
||||||
|
Slug string `json:"slug"`
|
||||||
|
Seats []seat `json:"seats"`
|
||||||
|
Requires []string `json:"requires"`
|
||||||
|
Contributes map[string]map[string]any `json:"contributes"`
|
||||||
|
OwnSecrets map[string]map[string]string `json:"own-secrets"`
|
||||||
|
Tools []string `json:"tools"`
|
||||||
|
Emits []string `json:"emits"`
|
||||||
|
Streams []struct {
|
||||||
|
Name string `json:"name"`
|
||||||
|
Events []string `json:"events"`
|
||||||
|
MaxAgeDays int `json:"max-age-days"`
|
||||||
|
Grant struct {
|
||||||
|
Config string `json:"config"`
|
||||||
|
Events string `json:"events"`
|
||||||
|
} `json:"grant"`
|
||||||
|
} `json:"streams"`
|
||||||
|
Consumes []string `json:"consumes"`
|
||||||
|
Invokes []string `json:"invokes"`
|
||||||
|
Claims []seat `json:"claims"`
|
||||||
|
Resources []map[string]any `json:"resources"`
|
||||||
|
Build struct {
|
||||||
|
Artifacts []struct {
|
||||||
|
From string `json:"from"`
|
||||||
|
Binary string `json:"binary"`
|
||||||
|
Loads []string `json:"loads"`
|
||||||
|
Env map[string]string `json:"env"`
|
||||||
|
} `json:"artifacts"`
|
||||||
|
} `json:"build"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func readManifest(t *testing.T) (manifest, map[string]map[string]any) {
|
||||||
|
t.Helper()
|
||||||
|
raw, err := os.ReadFile(filepath.Join("..", "..", "module.json"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var m manifest
|
||||||
|
if err := json.Unmarshal(raw, &m); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
byID := map[string]map[string]any{}
|
||||||
|
for _, r := range m.Resources {
|
||||||
|
byID[r["id"].(string)] = r
|
||||||
|
}
|
||||||
|
return m, byID
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTheManifestSaysWhatTheCodeDoes(t *testing.T) {
|
||||||
|
m, byID := readManifest(t)
|
||||||
|
if m.Module != Module || !reflect.DeepEqual(m.Requires, []string{"postgres-database", "s3-bucket"}) || m.Contributes["postgres-database"]["name"] != "mail" {
|
||||||
|
t.Fatalf("module %s, requires %v, contributes %v", m.Module, m.Requires, m.Contributes)
|
||||||
|
}
|
||||||
|
// mesh_<node>_<slug> fits the S3 access key's 20 characters on every node.
|
||||||
|
if len("mesh_shanks_"+m.Slug) > 20 {
|
||||||
|
t.Fatalf("the slug %q overflows an S3 access key", m.Slug)
|
||||||
|
}
|
||||||
|
fam := m.OwnSecrets[SecretFamily+"*"]
|
||||||
|
if fam["issued-by"] != "outside" || fam["path"] != "${dir:state}/"+SecretFamily+"*.secret" || len(m.OwnSecrets) != 1 {
|
||||||
|
t.Fatalf("own secrets %v", m.OwnSecrets)
|
||||||
|
}
|
||||||
|
var names []string
|
||||||
|
for _, tool := range (&Bundle{}).Tools() {
|
||||||
|
names = append(names, tool.Name)
|
||||||
|
}
|
||||||
|
wantTools := []string{
|
||||||
|
"mail_accounts", "mail_search", "mail_read", "mail_health", "mail_send", "mail_send_status",
|
||||||
|
"mail-store.accounts", "mail-store.search", "mail-store.read", "mail-store.send",
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(m.Tools, names) || !reflect.DeepEqual(names, wantTools) {
|
||||||
|
t.Fatalf("tools %v, the code serves %v", m.Tools, names)
|
||||||
|
}
|
||||||
|
// Its events go to its own stream MAIL, kept a week, each account's part heard only by the consumers the trusted
|
||||||
|
// config value grants names for it (ADR 0359 §2, §4; ADR 0363): none on the shared EVENTS stream.
|
||||||
|
var events []string
|
||||||
|
for _, c := range Changes {
|
||||||
|
events = append(events, EventKey("*", c))
|
||||||
|
}
|
||||||
|
if len(m.Emits) != 0 || len(m.Streams) != 1 || m.Streams[0].Name != "mail" || !reflect.DeepEqual(m.Streams[0].Events, events) ||
|
||||||
|
m.Streams[0].MaxAgeDays != 7 || m.Streams[0].Grant.Config != "grants" || m.Streams[0].Grant.Events != "mail.{key}.**" {
|
||||||
|
t.Fatalf("emits %v, streams %+v; the code says %v", m.Emits, m.Streams, events)
|
||||||
|
}
|
||||||
|
// It calls nothing and consumes nothing: mesh-mail-notifier (issue 800 part B) is the one that invokes and
|
||||||
|
// consumes, not the store.
|
||||||
|
if len(m.Consumes)+len(m.Invokes) != 0 {
|
||||||
|
t.Fatalf("consumes %v, invokes %v", m.Consumes, m.Invokes)
|
||||||
|
}
|
||||||
|
// It declares exactly the three mail seats (ADR 0378 §1 as amended by ADR 0379 §4) — mail-store and
|
||||||
|
// mail-webclient each held once for the mesh, mail-notifier held once per machine and needing node-notifier
|
||||||
|
// there — and claims only mail-store, serving the same four verbs as its own tools (§2, §3).
|
||||||
|
wantSeats := []seat{
|
||||||
|
{Name: "mail-store", Scope: "mesh", Serves: []string{"accounts", "search", "read", "send"}},
|
||||||
|
{Name: "mail-webclient", Scope: "mesh", Serves: []string{"open", "status"}},
|
||||||
|
{Name: "mail-notifier", Scope: "node", Serves: []string{"status"}, Needs: []string{"node-notifier"}},
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(m.Seats, wantSeats) {
|
||||||
|
t.Fatalf("seats %+v, want %+v", m.Seats, wantSeats)
|
||||||
|
}
|
||||||
|
wantClaims := []seat{
|
||||||
|
{Name: "mail-store", Scope: "mesh", Serves: []string{"accounts", "search", "read", "send"}},
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(m.Claims, wantClaims) {
|
||||||
|
t.Fatalf("claims %+v, want %+v", m.Claims, wantClaims)
|
||||||
|
}
|
||||||
|
// The two settings are trusted, and their defaults are an empty account list and no grant.
|
||||||
|
for id, want := range map[string]string{"accounts": `{"accounts":{}}`, "grants": `{"grants":{}}`} {
|
||||||
|
r := byID[id]
|
||||||
|
if r == nil || r["trusted"] != true || r["merge"] != "json" || !reflect.DeepEqual(r["takes-settings"], []any{id}) {
|
||||||
|
t.Fatalf("%s: %v", id, r)
|
||||||
|
}
|
||||||
|
if strings.Join(strings.Fields(r["content"].(string)), "") != want {
|
||||||
|
t.Fatalf("%s's default is %q", id, r["content"])
|
||||||
|
}
|
||||||
|
if _, err := ParseAccounts([]byte(byID["accounts"]["content"].(string))); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
a := m.Build.Artifacts
|
||||||
|
if len(a) != 1 || a[0].From != "cmd/mesh-mail" || a[0].Binary != Module || !reflect.DeepEqual(a[0].Loads, []string{Module}) {
|
||||||
|
t.Fatalf("build %+v", m.Build)
|
||||||
|
}
|
||||||
|
for k, v := range map[string]string{"MESH_MAIL_ACCOUNTS": "accounts", "MESH_MAIL_GRANTS": "grants", "MESH_MAIL_STORE": "store", "DATABASE_URL_FILE": "database-url"} {
|
||||||
|
if a[0].Env[k] != byID[v]["path"] {
|
||||||
|
t.Errorf("the bundle reads %s at %s, and the manifest writes %s at %v", k, a[0].Env[k], v, byID[v]["path"])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if a[0].Env["MESH_MAIL_SECRETS"] != "${dir:state}" {
|
||||||
|
t.Errorf("the secrets directory is %s", a[0].Env["MESH_MAIL_SECRETS"])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Its processes: the preparation once, keep judged by its unit, watch by mail_health (ADR 0359 §6), the watch writing
|
||||||
|
// the verdict the bundle's tool reads, as the account the bundle runs as.
|
||||||
|
func TestItsProcessesAreThePreparationTheActAndTheJudge(t *testing.T) {
|
||||||
|
m, byID := readManifest(t)
|
||||||
|
prep, keep, watch := byID["prepare"], byID["keep"], byID["watch"]
|
||||||
|
if prep == nil || prep["run-once"] != true || prep["run"].([]any)[1] != "prepare" {
|
||||||
|
t.Fatalf("prepare %v", prep)
|
||||||
|
}
|
||||||
|
if keep == nil || keep["run"].([]any)[1] != "keep" || keep["health"].(map[string]any)["kind"] != "unit" {
|
||||||
|
t.Fatalf("keep %v", keep)
|
||||||
|
}
|
||||||
|
h, _ := watch["health"].(map[string]any)
|
||||||
|
if watch == nil || watch["run"].([]any)[1] != "watch" || h["kind"] != "tool" || h["tool"] != HealthTool || h["needs"] != "postgres-database" {
|
||||||
|
t.Fatalf("watch %v", watch)
|
||||||
|
}
|
||||||
|
if watch["user"] != "${machine:account}" || watch["env"].(map[string]any)["MESH_MAIL_STATUS"] != m.Build.Artifacts[0].Env["MESH_MAIL_STATUS"] {
|
||||||
|
t.Fatalf("watch writes another status than the tool reads, or as another account: %v", watch)
|
||||||
|
}
|
||||||
|
// keep runs as the machine's root, which the own secrets' files belong to; it starts again when its settings or
|
||||||
|
// its store change.
|
||||||
|
if _, as := keep["user"]; as {
|
||||||
|
t.Fatalf("keep runs as %v", keep["user"])
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(keep["restart-on"], []any{"database-url", "store", "accounts"}) {
|
||||||
|
t.Fatalf("keep restarts on %v", keep["restart-on"])
|
||||||
|
}
|
||||||
|
}
|
||||||
+46
@@ -0,0 +1,46 @@
|
|||||||
|
-- mesh-mail's index (novox/hq ADR 0359 §3): one row per message and folder, what is known of each folder, each
|
||||||
|
-- account's state as its keeper said it, and the events written with the change they say, until they are said.
|
||||||
|
create table folder (
|
||||||
|
account text not null,
|
||||||
|
name text not null,
|
||||||
|
uidvalidity bigint not null,
|
||||||
|
messages integer not null default 0,
|
||||||
|
synced timestamptz not null default now(),
|
||||||
|
primary key (account, name));
|
||||||
|
|
||||||
|
create table message (
|
||||||
|
account text not null,
|
||||||
|
folder text not null,
|
||||||
|
uid bigint not null,
|
||||||
|
uidvalidity bigint not null,
|
||||||
|
message_id text not null default '',
|
||||||
|
in_reply_to text not null default '',
|
||||||
|
date timestamptz not null,
|
||||||
|
sender jsonb not null default '[]',
|
||||||
|
recipients jsonb not null default '[]',
|
||||||
|
copied jsonb not null default '[]',
|
||||||
|
people text not null default '',
|
||||||
|
subject text not null default '',
|
||||||
|
size bigint not null default 0,
|
||||||
|
flags text[] not null default '{}',
|
||||||
|
attachments jsonb not null default '[]',
|
||||||
|
body text not null default '',
|
||||||
|
search tsvector generated always as (
|
||||||
|
to_tsvector('simple'::regconfig, subject || ' ' || people || ' ' || body)) stored,
|
||||||
|
primary key (account, folder, uid));
|
||||||
|
create index message_search on message using gin (search);
|
||||||
|
create index message_date on message (account, date desc);
|
||||||
|
|
||||||
|
create table account_state (
|
||||||
|
account text primary key,
|
||||||
|
state text not null,
|
||||||
|
why text not null default '',
|
||||||
|
heard timestamptz,
|
||||||
|
synced timestamptz,
|
||||||
|
said timestamptz not null default now());
|
||||||
|
|
||||||
|
create table outbox (
|
||||||
|
id bigserial primary key,
|
||||||
|
key text not null,
|
||||||
|
body jsonb not null,
|
||||||
|
made timestamptz not null default now());
|
||||||
@@ -0,0 +1,125 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
// A message's text and its attachments, read from its MIME source. The text is kept in the index for search and served
|
||||||
|
// through mail_read; each attachment goes to the attachment store once, by its SHA-256 (novox/hq ADR 0359 §3).
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"errors"
|
||||||
|
"html"
|
||||||
|
"io"
|
||||||
|
"mime"
|
||||||
|
"regexp"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
_ "github.com/emersion/go-message/charset" // the charsets mail is written in, beyond UTF-8 and ASCII
|
||||||
|
"github.com/emersion/go-message/mail"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Part is one attachment as read from a message.
|
||||||
|
type Part struct {
|
||||||
|
Name string
|
||||||
|
Type string
|
||||||
|
Data []byte
|
||||||
|
}
|
||||||
|
|
||||||
|
// MostBodyText bounds the text kept per message: a longer body is cut, and says so.
|
||||||
|
const MostBodyText = 256 << 10
|
||||||
|
|
||||||
|
// Parsed is a message's text and its attachments.
|
||||||
|
type Parsed struct {
|
||||||
|
Text string
|
||||||
|
Attachments []Part
|
||||||
|
}
|
||||||
|
|
||||||
|
// ParseMessage reads a message's text (its plain part; else its HTML part as text) and its attachments. A message the
|
||||||
|
// reader cannot follow keeps what was read before the fault, and the error says what.
|
||||||
|
func ParseMessage(raw []byte) (Parsed, error) {
|
||||||
|
var p Parsed
|
||||||
|
r, err := mail.CreateReader(bytes.NewReader(raw))
|
||||||
|
if r == nil {
|
||||||
|
return p, err
|
||||||
|
}
|
||||||
|
var plain, htmlText string
|
||||||
|
for {
|
||||||
|
part, err := r.NextPart()
|
||||||
|
if errors.Is(err, io.EOF) {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return finish(p, plain, htmlText), err
|
||||||
|
}
|
||||||
|
switch h := part.Header.(type) {
|
||||||
|
case *mail.InlineHeader:
|
||||||
|
ct, params, _ := h.ContentType()
|
||||||
|
_, disp, _ := h.ContentDisposition()
|
||||||
|
body, readErr := io.ReadAll(io.LimitReader(part.Body, 64<<20))
|
||||||
|
if readErr != nil {
|
||||||
|
return finish(p, plain, htmlText), readErr
|
||||||
|
}
|
||||||
|
name := params["name"]
|
||||||
|
if name == "" {
|
||||||
|
name = disp["filename"]
|
||||||
|
}
|
||||||
|
switch {
|
||||||
|
case ct == "text/plain" && plain == "" && name == "":
|
||||||
|
plain = string(body)
|
||||||
|
case ct == "text/html" && htmlText == "" && name == "":
|
||||||
|
htmlText = HTMLText(string(body))
|
||||||
|
case !strings.HasPrefix(ct, "text/") || name != "":
|
||||||
|
// An inline image or a named inline file is an attachment too.
|
||||||
|
p.Attachments = append(p.Attachments, Part{Name: name, Type: orOctets(ct), Data: body})
|
||||||
|
}
|
||||||
|
case *mail.AttachmentHeader:
|
||||||
|
ct, _, _ := h.ContentType()
|
||||||
|
name, _ := h.Filename()
|
||||||
|
body, readErr := io.ReadAll(io.LimitReader(part.Body, 64<<20))
|
||||||
|
if readErr != nil {
|
||||||
|
return finish(p, plain, htmlText), readErr
|
||||||
|
}
|
||||||
|
p.Attachments = append(p.Attachments, Part{Name: name, Type: orOctets(ct), Data: body})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return finish(p, plain, htmlText), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func finish(p Parsed, plain, htmlText string) Parsed {
|
||||||
|
p.Text = plain
|
||||||
|
if strings.TrimSpace(p.Text) == "" {
|
||||||
|
p.Text = htmlText
|
||||||
|
}
|
||||||
|
p.Text = strings.ToValidUTF8(p.Text, "�")
|
||||||
|
if len(p.Text) > MostBodyText {
|
||||||
|
p.Text = strings.ToValidUTF8(p.Text[:MostBodyText], "") + "\n[the text goes on; it was cut at 256 KiB]"
|
||||||
|
}
|
||||||
|
return p
|
||||||
|
}
|
||||||
|
|
||||||
|
func orOctets(ct string) string {
|
||||||
|
if ct == "" {
|
||||||
|
return "application/octet-stream"
|
||||||
|
}
|
||||||
|
if t, _, err := mime.ParseMediaType(ct); err == nil {
|
||||||
|
return t
|
||||||
|
}
|
||||||
|
return ct
|
||||||
|
}
|
||||||
|
|
||||||
|
var (
|
||||||
|
scriptsAndStyles = regexp.MustCompile(`(?is)<(script|style|head)\b.*?</(script|style|head)>`)
|
||||||
|
breaks = regexp.MustCompile(`(?i)<(br|/p|/div|/tr|/li|/h[1-6])\b[^>]*>`)
|
||||||
|
tags = regexp.MustCompile(`(?s)<[^>]*>`)
|
||||||
|
blankLines = regexp.MustCompile(`\n[ \t]*\n([ \t]*\n)+`)
|
||||||
|
)
|
||||||
|
|
||||||
|
// HTMLText is an HTML body as plain text: no script, style or markup, the entities read, paragraphs kept as lines.
|
||||||
|
// Text only, so nothing of the sender's markup ever reaches whoever reads it.
|
||||||
|
func HTMLText(s string) string {
|
||||||
|
s = scriptsAndStyles.ReplaceAllString(s, "")
|
||||||
|
s = breaks.ReplaceAllString(s, "\n")
|
||||||
|
s = tags.ReplaceAllString(s, "")
|
||||||
|
s = html.UnescapeString(s)
|
||||||
|
s = strings.ReplaceAll(s, "\r", "")
|
||||||
|
s = blankLines.ReplaceAllString(s, "\n\n")
|
||||||
|
return strings.TrimSpace(s)
|
||||||
|
}
|
||||||
@@ -0,0 +1,275 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
// The attachment store (novox/hq ADR 0359 §3): each attachment kept once, by its SHA-256, in the bucket the `minio`
|
||||||
|
// module provides this module through the `s3-bucket` provision. The mesh writes the bucket's facts and its key into
|
||||||
|
// one file (MESH_MAIL_STORE); this speaks S3 to it with the standard library alone: a HEAD, a PUT and a GET, each
|
||||||
|
// signed with AWS Signature Version 4, path-style, as an S3-compatible object store answers them.
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"crypto/hmac"
|
||||||
|
"crypto/sha256"
|
||||||
|
"encoding/hex"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"net/http"
|
||||||
|
"net/url"
|
||||||
|
"os"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Objects is the attachment store.
|
||||||
|
type Objects interface {
|
||||||
|
// Bucket is the bucket's name, as an event names it.
|
||||||
|
Bucket() string
|
||||||
|
// Has says whether the key is kept.
|
||||||
|
Has(ctx context.Context, key string) (bool, error)
|
||||||
|
// Put keeps the bytes under the key.
|
||||||
|
Put(ctx context.Context, key, mediaType string, data []byte) error
|
||||||
|
// Get is the bytes kept under the key.
|
||||||
|
Get(ctx context.Context, key string) ([]byte, error)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Keep stores an attachment once by its SHA-256 and answers its reference: the same file mailed ten times is one
|
||||||
|
// object.
|
||||||
|
func Keep(ctx context.Context, o Objects, data []byte, mediaType, name string) (AttachmentRef, error) {
|
||||||
|
sum := sha256.Sum256(data)
|
||||||
|
key := hex.EncodeToString(sum[:])
|
||||||
|
has, err := o.Has(ctx, key)
|
||||||
|
if err != nil {
|
||||||
|
return AttachmentRef{}, err
|
||||||
|
}
|
||||||
|
if !has {
|
||||||
|
if err := o.Put(ctx, key, mediaType, data); err != nil {
|
||||||
|
return AttachmentRef{}, err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return AttachmentRef{Bucket: o.Bucket(), Key: key, Size: int64(len(data)), Type: mediaType, Name: name}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// StoreFacts is the file the mesh writes from the s3-bucket binding and its secret.
|
||||||
|
type StoreFacts struct {
|
||||||
|
Scheme string `json:"scheme"`
|
||||||
|
At string `json:"at"`
|
||||||
|
Port string `json:"port"`
|
||||||
|
Bucket string `json:"bucket"`
|
||||||
|
Region string `json:"region"`
|
||||||
|
Key string `json:"key"`
|
||||||
|
Secret string `json:"secret"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// ReadStoreFacts reads the store's facts.
|
||||||
|
func ReadStoreFacts(path string) (StoreFacts, error) {
|
||||||
|
var f StoreFacts
|
||||||
|
if path == "" {
|
||||||
|
return f, errors.New("the module was started without its store's facts (MESH_MAIL_STORE)")
|
||||||
|
}
|
||||||
|
raw, err := os.ReadFile(path)
|
||||||
|
if err != nil {
|
||||||
|
return f, fmt.Errorf("the store's facts cannot be read: %w", err)
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal(raw, &f); err != nil {
|
||||||
|
return f, fmt.Errorf("the store's facts are not the module's: %v", err)
|
||||||
|
}
|
||||||
|
if f.At == "" || f.Bucket == "" || f.Key == "" || f.Secret == "" {
|
||||||
|
return f, errors.New("the store's facts name no address, bucket or key")
|
||||||
|
}
|
||||||
|
if f.Scheme == "" {
|
||||||
|
f.Scheme = "http"
|
||||||
|
}
|
||||||
|
if f.Region == "" {
|
||||||
|
f.Region = "us-east-1"
|
||||||
|
}
|
||||||
|
return f, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// S3 is the attachment store over S3.
|
||||||
|
type S3 struct {
|
||||||
|
Endpoint string // scheme://host[:port]
|
||||||
|
Name string
|
||||||
|
Region string
|
||||||
|
Access string
|
||||||
|
Secret string
|
||||||
|
Client *http.Client
|
||||||
|
Now func() time.Time
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewS3 is the store the facts name.
|
||||||
|
func NewS3(f StoreFacts) *S3 {
|
||||||
|
host := f.At
|
||||||
|
if f.Port != "" {
|
||||||
|
host += ":" + f.Port
|
||||||
|
}
|
||||||
|
return &S3{Endpoint: f.Scheme + "://" + host, Name: f.Bucket, Region: f.Region, Access: f.Key, Secret: f.Secret,
|
||||||
|
Client: &http.Client{Timeout: 2 * time.Minute}, Now: time.Now}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *S3) Bucket() string { return s.Name }
|
||||||
|
|
||||||
|
func (s *S3) do(ctx context.Context, method, key, mediaType string, body []byte) (*http.Response, error) {
|
||||||
|
u, err := url.Parse(s.Endpoint + "/" + s.Name + "/" + key)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
req, err := http.NewRequestWithContext(ctx, method, u.String(), bytes.NewReader(body))
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if mediaType != "" {
|
||||||
|
req.Header.Set("Content-Type", mediaType)
|
||||||
|
}
|
||||||
|
req.ContentLength = int64(len(body))
|
||||||
|
Sign(req, body, s.Access, s.Secret, s.Region, s.Now().UTC())
|
||||||
|
return s.Client.Do(req)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *S3) Has(ctx context.Context, key string) (bool, error) {
|
||||||
|
resp, err := s.do(ctx, http.MethodHead, key, "", nil)
|
||||||
|
if err != nil {
|
||||||
|
return false, fmt.Errorf("the attachment store cannot be reached: %w", err)
|
||||||
|
}
|
||||||
|
resp.Body.Close()
|
||||||
|
switch resp.StatusCode {
|
||||||
|
case http.StatusOK:
|
||||||
|
return true, nil
|
||||||
|
case http.StatusNotFound:
|
||||||
|
return false, nil
|
||||||
|
}
|
||||||
|
return false, fmt.Errorf("the attachment store answered %s to asking for %s", resp.Status, key)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *S3) Put(ctx context.Context, key, mediaType string, data []byte) error {
|
||||||
|
resp, err := s.do(ctx, http.MethodPut, key, mediaType, data)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("the attachment store cannot be reached: %w", err)
|
||||||
|
}
|
||||||
|
defer resp.Body.Close()
|
||||||
|
if resp.StatusCode/100 != 2 {
|
||||||
|
said, _ := io.ReadAll(io.LimitReader(resp.Body, 512))
|
||||||
|
return fmt.Errorf("the attachment store refused %s: %s %s", key, resp.Status, strings.TrimSpace(string(said)))
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *S3) Get(ctx context.Context, key string) ([]byte, error) {
|
||||||
|
resp, err := s.do(ctx, http.MethodGet, key, "", nil)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("the attachment store cannot be reached: %w", err)
|
||||||
|
}
|
||||||
|
defer resp.Body.Close()
|
||||||
|
if resp.StatusCode != http.StatusOK {
|
||||||
|
return nil, fmt.Errorf("the attachment store answered %s for %s", resp.Status, key)
|
||||||
|
}
|
||||||
|
return io.ReadAll(resp.Body)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Sign signs a request with AWS Signature Version 4 for S3: every header it carries but the agent and the length is
|
||||||
|
// signed, with the payload's hash.
|
||||||
|
func Sign(req *http.Request, body []byte, access, secret, region string, now time.Time) {
|
||||||
|
payload := sha256.Sum256(body)
|
||||||
|
payloadHex := hex.EncodeToString(payload[:])
|
||||||
|
amzDate := now.Format("20060102T150405Z")
|
||||||
|
day := now.Format("20060102")
|
||||||
|
req.Header.Set("x-amz-date", amzDate)
|
||||||
|
req.Header.Set("x-amz-content-sha256", payloadHex)
|
||||||
|
|
||||||
|
headers := map[string]string{"host": req.URL.Host}
|
||||||
|
for k, v := range req.Header {
|
||||||
|
lk := strings.ToLower(k)
|
||||||
|
if lk == "user-agent" || lk == "content-length" || lk == "authorization" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
headers[lk] = strings.TrimSpace(strings.Join(v, ","))
|
||||||
|
}
|
||||||
|
names := make([]string, 0, len(headers))
|
||||||
|
for k := range headers {
|
||||||
|
names = append(names, k)
|
||||||
|
}
|
||||||
|
sort.Strings(names)
|
||||||
|
var canonHeaders strings.Builder
|
||||||
|
for _, k := range names {
|
||||||
|
canonHeaders.WriteString(k + ":" + headers[k] + "\n")
|
||||||
|
}
|
||||||
|
signed := strings.Join(names, ";")
|
||||||
|
path := req.URL.EscapedPath()
|
||||||
|
if path == "" {
|
||||||
|
path = "/"
|
||||||
|
}
|
||||||
|
canonical := strings.Join([]string{req.Method, path, canonicalQuery(req.URL.Query()), canonHeaders.String(), signed, payloadHex}, "\n")
|
||||||
|
scope := day + "/" + region + "/s3/aws4_request"
|
||||||
|
sum := sha256.Sum256([]byte(canonical))
|
||||||
|
toSign := "AWS4-HMAC-SHA256\n" + amzDate + "\n" + scope + "\n" + hex.EncodeToString(sum[:])
|
||||||
|
key := hmacOf([]byte("AWS4"+secret), day)
|
||||||
|
key = hmacOf(key, region)
|
||||||
|
key = hmacOf(key, "s3")
|
||||||
|
key = hmacOf(key, "aws4_request")
|
||||||
|
sig := hex.EncodeToString(hmacOf(key, toSign))
|
||||||
|
req.Header.Set("Authorization", "AWS4-HMAC-SHA256 Credential="+access+"/"+scope+", SignedHeaders="+signed+", Signature="+sig)
|
||||||
|
}
|
||||||
|
|
||||||
|
func canonicalQuery(q url.Values) string {
|
||||||
|
keys := make([]string, 0, len(q))
|
||||||
|
for k := range q {
|
||||||
|
keys = append(keys, k)
|
||||||
|
}
|
||||||
|
sort.Strings(keys)
|
||||||
|
var parts []string
|
||||||
|
for _, k := range keys {
|
||||||
|
vs := append([]string(nil), q[k]...)
|
||||||
|
sort.Strings(vs)
|
||||||
|
for _, v := range vs {
|
||||||
|
parts = append(parts, url.QueryEscape(k)+"="+strings.ReplaceAll(url.QueryEscape(v), "+", "%20"))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return strings.Join(parts, "&")
|
||||||
|
}
|
||||||
|
|
||||||
|
func hmacOf(key []byte, data string) []byte {
|
||||||
|
h := hmac.New(sha256.New, key)
|
||||||
|
h.Write([]byte(data))
|
||||||
|
return h.Sum(nil)
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---- in memory ----------------------------------------------------------------------------------------------------
|
||||||
|
|
||||||
|
type memObjects struct {
|
||||||
|
mu sync.Mutex
|
||||||
|
name string
|
||||||
|
kept map[string][]byte
|
||||||
|
puts int
|
||||||
|
}
|
||||||
|
|
||||||
|
func newMemObjects() *memObjects { return &memObjects{name: "mail-test", kept: map[string][]byte{}} }
|
||||||
|
|
||||||
|
func (m *memObjects) Bucket() string { return m.name }
|
||||||
|
|
||||||
|
func (m *memObjects) Has(_ context.Context, key string) (bool, error) {
|
||||||
|
m.mu.Lock()
|
||||||
|
defer m.mu.Unlock()
|
||||||
|
_, ok := m.kept[key]
|
||||||
|
return ok, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *memObjects) Put(_ context.Context, key, _ string, data []byte) error {
|
||||||
|
m.mu.Lock()
|
||||||
|
defer m.mu.Unlock()
|
||||||
|
m.kept[key] = append([]byte(nil), data...)
|
||||||
|
m.puts++
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *memObjects) Get(_ context.Context, key string) ([]byte, error) {
|
||||||
|
m.mu.Lock()
|
||||||
|
defer m.mu.Unlock()
|
||||||
|
b, ok := m.kept[key]
|
||||||
|
if !ok {
|
||||||
|
return nil, fmt.Errorf("%s is not kept", key)
|
||||||
|
}
|
||||||
|
return b, nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,104 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"io"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The signature is AWS Signature Version 4 as S3 documents it: its own worked example, "GET Object", signs to the
|
||||||
|
// signature AWS publishes.
|
||||||
|
func TestTheSignatureIsTheOneS3Documents(t *testing.T) {
|
||||||
|
req, _ := http.NewRequest(http.MethodGet, "https://examplebucket.s3.amazonaws.com/test.txt", nil)
|
||||||
|
req.Header.Set("Range", "bytes=0-9")
|
||||||
|
Sign(req, nil, "AKIAIOSFODNN7EXAMPLE", "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", "us-east-1", time.Date(2013, 5, 24, 0, 0, 0, 0, time.UTC))
|
||||||
|
want := "AWS4-HMAC-SHA256 Credential=AKIAIOSFODNN7EXAMPLE/20130524/us-east-1/s3/aws4_request, " +
|
||||||
|
"SignedHeaders=host;range;x-amz-content-sha256;x-amz-date, Signature=f0e8bdb87c964420e857bd35b5d6ed310bd44f0170aba48dd91039c6036bdb41"
|
||||||
|
if got := req.Header.Get("Authorization"); got != want {
|
||||||
|
t.Fatalf("signed\n%s\nnot\n%s", got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// An attachment is kept once by its SHA-256: the same file twice is one PUT, and it reads back.
|
||||||
|
func TestAnAttachmentIsKeptOnceByItsHash(t *testing.T) {
|
||||||
|
var mu sync.Mutex
|
||||||
|
kept := map[string][]byte{}
|
||||||
|
puts := 0
|
||||||
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
if !strings.HasPrefix(r.Header.Get("Authorization"), "AWS4-HMAC-SHA256 Credential=mesh_novox_mail/") {
|
||||||
|
w.WriteHeader(http.StatusForbidden)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
mu.Lock()
|
||||||
|
defer mu.Unlock()
|
||||||
|
switch r.Method {
|
||||||
|
case http.MethodHead:
|
||||||
|
if _, ok := kept[r.URL.Path]; !ok {
|
||||||
|
w.WriteHeader(http.StatusNotFound)
|
||||||
|
}
|
||||||
|
case http.MethodPut:
|
||||||
|
body, _ := io.ReadAll(r.Body)
|
||||||
|
kept[r.URL.Path] = body
|
||||||
|
puts++
|
||||||
|
case http.MethodGet:
|
||||||
|
b, ok := kept[r.URL.Path]
|
||||||
|
if !ok {
|
||||||
|
w.WriteHeader(http.StatusNotFound)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
_, _ = w.Write(b)
|
||||||
|
}
|
||||||
|
}))
|
||||||
|
defer srv.Close()
|
||||||
|
dir := t.TempDir()
|
||||||
|
path := filepath.Join(dir, "store.json")
|
||||||
|
host, port, _ := strings.Cut(strings.TrimPrefix(srv.URL, "http://"), ":")
|
||||||
|
writeFile(t, path, `{"scheme": "http", "at": "`+host+`", "port": "`+port+`", "bucket": "mesh-novox-mail", "region": "eu-west",
|
||||||
|
"key": "mesh_novox_mail", "secret": "s3cr3t"}`)
|
||||||
|
facts, err := ReadStoreFacts(path)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
s3 := NewS3(facts)
|
||||||
|
ctx := context.Background()
|
||||||
|
a, err := Keep(ctx, s3, []byte("same file"), "text/plain", "a.txt")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
b, err := Keep(ctx, s3, []byte("same file"), "text/plain", "b.txt")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if a.Key != b.Key || a.Bucket != "mesh-novox-mail" || puts != 1 || len(a.Key) != 64 {
|
||||||
|
t.Fatalf("%+v %+v, %d put(s)", a, b, puts)
|
||||||
|
}
|
||||||
|
if _, ok := kept["/mesh-novox-mail/"+a.Key]; !ok {
|
||||||
|
t.Fatalf("kept %v", kept)
|
||||||
|
}
|
||||||
|
got, err := s3.Get(ctx, a.Key)
|
||||||
|
if err != nil || string(got) != "same file" {
|
||||||
|
t.Fatalf("%q %v", got, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAMessagesTextAndAttachmentsAreRead(t *testing.T) {
|
||||||
|
p, err := ParseMessage([]byte(strings.ReplaceAll(attachedMail("x"), "\n", "\r\n")))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(p.Text) != "Please find the invoice attached." || len(p.Attachments) != 1 ||
|
||||||
|
p.Attachments[0].Name != "invoice.pdf" || p.Attachments[0].Type != "application/pdf" || string(p.Attachments[0].Data) != attachmentData {
|
||||||
|
t.Fatalf("%+v", p)
|
||||||
|
}
|
||||||
|
html := "From: a@b.c\r\nSubject: s\r\nContent-Type: text/html; charset=utf-8\r\n\r\n<html><head><style>p{}</style></head><body><p>Hello & welcome</p><script>alert(1)</script><p>Bye</p></body></html>"
|
||||||
|
p, err = ParseMessage([]byte(html))
|
||||||
|
if err != nil || p.Text != "Hello & welcome\nBye" {
|
||||||
|
t.Fatalf("%q %v", p.Text, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,504 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
// The index on the mesh's postgres (novox/hq ADR 0359 §3): the database the mesh provisions for this module through
|
||||||
|
// the postgres-database provision, its address in the file DATABASE_URL_FILE names. Its schema is numbered migrations
|
||||||
|
// in migrations/, applied by the preparation step (`mesh-mail prepare`, ADR 0135) and recorded in the database itself;
|
||||||
|
// a fresh database is given the consolidated schema, schema.sql, at once, with every migration recorded as applied.
|
||||||
|
// Nothing else changes the schema: the keeper, the watch and the tools refuse a schema that is behind, and say so.
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"crypto/sha256"
|
||||||
|
"embed"
|
||||||
|
"encoding/hex"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"io/fs"
|
||||||
|
"os"
|
||||||
|
"path"
|
||||||
|
"regexp"
|
||||||
|
"sort"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/jackc/pgx/v5"
|
||||||
|
"github.com/jackc/pgx/v5/pgconn"
|
||||||
|
"github.com/jackc/pgx/v5/pgxpool"
|
||||||
|
)
|
||||||
|
|
||||||
|
//go:embed migrations/*.sql
|
||||||
|
var migrationFiles embed.FS
|
||||||
|
|
||||||
|
//go:embed schema.sql
|
||||||
|
var consolidated string
|
||||||
|
|
||||||
|
// Migration is one numbered change of the schema.
|
||||||
|
type Migration struct {
|
||||||
|
Number int
|
||||||
|
Name string
|
||||||
|
SQL string
|
||||||
|
Checksum string
|
||||||
|
}
|
||||||
|
|
||||||
|
var migrationFile = regexp.MustCompile(`^(\d{4})-([a-z0-9-]+)\.sql$`)
|
||||||
|
|
||||||
|
// Migrations are the migrations this binary carries, in order, numbered from 0001 without a gap.
|
||||||
|
func Migrations() ([]Migration, error) {
|
||||||
|
entries, err := fs.ReadDir(migrationFiles, "migrations")
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
var out []Migration
|
||||||
|
for _, e := range entries {
|
||||||
|
m := migrationFile.FindStringSubmatch(e.Name())
|
||||||
|
if m == nil {
|
||||||
|
return nil, fmt.Errorf("%s is not a migration's name (NNNN-what-it-does.sql)", e.Name())
|
||||||
|
}
|
||||||
|
n, _ := strconv.Atoi(m[1])
|
||||||
|
body, err := fs.ReadFile(migrationFiles, path.Join("migrations", e.Name()))
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(string(body)) == "" {
|
||||||
|
return nil, fmt.Errorf("%s is empty", e.Name())
|
||||||
|
}
|
||||||
|
sum := sha256.Sum256(body)
|
||||||
|
out = append(out, Migration{Number: n, Name: m[2], SQL: string(body), Checksum: hex.EncodeToString(sum[:])})
|
||||||
|
}
|
||||||
|
sort.Slice(out, func(i, j int) bool { return out[i].Number < out[j].Number })
|
||||||
|
for i, m := range out {
|
||||||
|
if m.Number != i+1 {
|
||||||
|
return nil, fmt.Errorf("the migrations are not numbered 0001 on without a gap: %04d is the %d. one", m.Number, i+1)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
const ledger = `create table if not exists migration (
|
||||||
|
number integer primary key,
|
||||||
|
name text not null,
|
||||||
|
checksum text not null,
|
||||||
|
applied timestamptz not null default now())`
|
||||||
|
|
||||||
|
// lockKey is the advisory lock every migration run takes, so two preparations never interleave.
|
||||||
|
const lockKey int64 = 4_250_359_001
|
||||||
|
|
||||||
|
// PgStore is the Store on postgres.
|
||||||
|
type PgStore struct{ pool *pgxpool.Pool }
|
||||||
|
|
||||||
|
// PgStoreFromEnv opens the index the mesh provisioned.
|
||||||
|
func PgStoreFromEnv(ctx context.Context) (*PgStore, error) {
|
||||||
|
file := os.Getenv("DATABASE_URL_FILE")
|
||||||
|
if file == "" {
|
||||||
|
return nil, errors.New("DATABASE_URL_FILE is not set: the index's database is a requirement the mesh resolves")
|
||||||
|
}
|
||||||
|
raw, err := os.ReadFile(file)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("the database's address cannot be read: %w", err)
|
||||||
|
}
|
||||||
|
return OpenPgStore(ctx, strings.TrimSpace(string(raw)))
|
||||||
|
}
|
||||||
|
|
||||||
|
// OpenPgStore opens the index at an address.
|
||||||
|
func OpenPgStore(ctx context.Context, url string) (*PgStore, error) {
|
||||||
|
pool, err := pgxpool.New(ctx, url)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return &PgStore{pool: pool}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Close closes the store.
|
||||||
|
func (s *PgStore) Close() { s.pool.Close() }
|
||||||
|
|
||||||
|
// Prepare brings the schema to what this version needs, and says what it did.
|
||||||
|
func (s *PgStore) Prepare(ctx context.Context) ([]string, error) {
|
||||||
|
migrations, err := Migrations()
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
conn, err := s.pool.Acquire(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
defer conn.Release()
|
||||||
|
if _, err := conn.Exec(ctx, `select pg_advisory_lock($1)`, lockKey); err != nil {
|
||||||
|
return nil, fmt.Errorf("the migration lock cannot be taken: %w", err)
|
||||||
|
}
|
||||||
|
defer func() { _, _ = conn.Exec(context.WithoutCancel(ctx), `select pg_advisory_unlock($1)`, lockKey) }()
|
||||||
|
if _, err := conn.Exec(ctx, ledger); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
applied := map[int]string{}
|
||||||
|
rows, err := conn.Query(ctx, `select number, checksum from migration`)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
highest := 0
|
||||||
|
for rows.Next() {
|
||||||
|
var n int
|
||||||
|
var sum string
|
||||||
|
if err := rows.Scan(&n, &sum); err != nil {
|
||||||
|
rows.Close()
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
applied[n] = sum
|
||||||
|
highest = max(highest, n)
|
||||||
|
}
|
||||||
|
rows.Close()
|
||||||
|
if err := rows.Err(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
record := func(tx pgx.Tx, m Migration) error {
|
||||||
|
_, err := tx.Exec(ctx, `insert into migration (number, name, checksum) values ($1, $2, $3)`, m.Number, m.Name, m.Checksum)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if len(applied) == 0 {
|
||||||
|
var exists bool
|
||||||
|
if err := conn.QueryRow(ctx, `select to_regclass('message') is not null`).Scan(&exists); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if exists {
|
||||||
|
return nil, errors.New("the database holds the index's tables and no record of a migration: its contents are not known, so nothing is changed")
|
||||||
|
}
|
||||||
|
tx, err := conn.Begin(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
defer func() { _ = tx.Rollback(context.WithoutCancel(ctx)) }()
|
||||||
|
if _, err := tx.Exec(ctx, consolidated); err != nil {
|
||||||
|
return nil, fmt.Errorf("the consolidated schema failed, and nothing of it was kept: %w", err)
|
||||||
|
}
|
||||||
|
for _, m := range migrations {
|
||||||
|
if err := record(tx, m); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err := tx.Commit(ctx); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return []string{fmt.Sprintf("a fresh database: the consolidated schema, through migration %04d", len(migrations))}, nil
|
||||||
|
}
|
||||||
|
var problems []string
|
||||||
|
var pending []Migration
|
||||||
|
for _, m := range migrations {
|
||||||
|
sum, ran := applied[m.Number]
|
||||||
|
switch {
|
||||||
|
case ran && sum != m.Checksum:
|
||||||
|
problems = append(problems, fmt.Sprintf("migration %04d-%s ran, and its file has changed since: write a new one", m.Number, m.Name))
|
||||||
|
case !ran && m.Number < highest:
|
||||||
|
problems = append(problems, fmt.Sprintf("migration %04d-%s never ran, and %04d did: renumber it above %04d", m.Number, m.Name, highest, highest))
|
||||||
|
case !ran:
|
||||||
|
pending = append(pending, m)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(problems) > 0 {
|
||||||
|
return nil, errors.New("this database and these migrations disagree: " + strings.Join(problems, "; "))
|
||||||
|
}
|
||||||
|
var said []string
|
||||||
|
for _, m := range pending {
|
||||||
|
err := func() error {
|
||||||
|
tx, err := conn.Begin(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer func() { _ = tx.Rollback(context.WithoutCancel(ctx)) }()
|
||||||
|
if _, err := tx.Exec(ctx, m.SQL); err != nil {
|
||||||
|
return fmt.Errorf("migration %04d-%s failed, and nothing it did was kept: %w", m.Number, m.Name, err)
|
||||||
|
}
|
||||||
|
if err := record(tx, m); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return tx.Commit(ctx)
|
||||||
|
}()
|
||||||
|
if err != nil {
|
||||||
|
return said, err
|
||||||
|
}
|
||||||
|
said = append(said, fmt.Sprintf("migration %04d-%s applied", m.Number, m.Name))
|
||||||
|
}
|
||||||
|
if len(said) == 0 {
|
||||||
|
said = []string{fmt.Sprintf("the schema is at migration %04d, as this version needs", len(migrations))}
|
||||||
|
}
|
||||||
|
return said, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Ready says why the index cannot serve this version, or nil.
|
||||||
|
func (s *PgStore) Ready(ctx context.Context) error {
|
||||||
|
migrations, err := Migrations()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
var at int
|
||||||
|
err = s.pool.QueryRow(ctx, `select coalesce(max(number), 0) from migration`).Scan(&at)
|
||||||
|
var pgErr *pgconn.PgError
|
||||||
|
if errors.As(err, &pgErr) && pgErr.Code == "42P01" {
|
||||||
|
at = 0
|
||||||
|
} else if err != nil {
|
||||||
|
return fmt.Errorf("the database cannot be reached: %w", err)
|
||||||
|
}
|
||||||
|
if at != len(migrations) {
|
||||||
|
return fmt.Errorf("the database's schema is at migration %04d and this version needs %04d: the preparation step "+
|
||||||
|
"(mesh-mail prepare) has not run", at, len(migrations))
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *PgStore) Folder(ctx context.Context, account, folder string) (Folder, bool, error) {
|
||||||
|
f := Folder{Account: account, Name: folder}
|
||||||
|
var validity int64
|
||||||
|
err := s.pool.QueryRow(ctx, `select uidvalidity, messages, synced from folder where account = $1 and name = $2`, account, folder).
|
||||||
|
Scan(&validity, &f.Messages, &f.Synced)
|
||||||
|
if errors.Is(err, pgx.ErrNoRows) {
|
||||||
|
return f, false, nil
|
||||||
|
}
|
||||||
|
f.UIDValidity = uint32(validity)
|
||||||
|
return f, err == nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *PgStore) SetFolder(ctx context.Context, f Folder) error {
|
||||||
|
_, err := s.pool.Exec(ctx, `insert into folder (account, name, uidvalidity, messages, synced) values ($1, $2, $3, $4, $5)
|
||||||
|
on conflict (account, name) do update set uidvalidity = excluded.uidvalidity, messages = excluded.messages, synced = excluded.synced`,
|
||||||
|
f.Account, f.Name, int64(f.UIDValidity), f.Messages, f.Synced)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *PgStore) DropFolder(ctx context.Context, account, folder string) error {
|
||||||
|
return pgx.BeginFunc(ctx, s.pool, func(tx pgx.Tx) error {
|
||||||
|
if _, err := tx.Exec(ctx, `delete from message where account = $1 and folder = $2`, account, folder); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
_, err := tx.Exec(ctx, `delete from folder where account = $1 and name = $2`, account, folder)
|
||||||
|
return err
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *PgStore) Flags(ctx context.Context, account, folder string) (map[uint32][]string, error) {
|
||||||
|
rows, err := s.pool.Query(ctx, `select uid, flags from message where account = $1 and folder = $2`, account, folder)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
defer rows.Close()
|
||||||
|
out := map[uint32][]string{}
|
||||||
|
for rows.Next() {
|
||||||
|
var uid int64
|
||||||
|
var flags []string
|
||||||
|
if err := rows.Scan(&uid, &flags); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
out[uint32(uid)] = flags
|
||||||
|
}
|
||||||
|
return out, rows.Err()
|
||||||
|
}
|
||||||
|
|
||||||
|
func keepEvent(ctx context.Context, tx pgx.Tx, e *Event) error {
|
||||||
|
if e == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
body, err := json.Marshal(e.Body)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
_, err = tx.Exec(ctx, `insert into outbox (key, body) values ($1, $2)`, e.Key, body)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *PgStore) Add(ctx context.Context, m Message, e *Event) error {
|
||||||
|
from, _ := json.Marshal(m.From)
|
||||||
|
to, _ := json.Marshal(m.To)
|
||||||
|
cc, _ := json.Marshal(orNone(m.Cc))
|
||||||
|
atts, _ := json.Marshal(m.Attachments)
|
||||||
|
flags := m.Flags
|
||||||
|
if flags == nil {
|
||||||
|
flags = []string{}
|
||||||
|
}
|
||||||
|
people := strings.Join([]string{peopleText(m.From), peopleText(m.To), peopleText(m.Cc)}, " ")
|
||||||
|
// An address is one word to the search, so its parts are kept beside it: "bob" finds bob@example.org.
|
||||||
|
people += " " + strings.NewReplacer("@", " ", ".", " ", ",", " ").Replace(people)
|
||||||
|
return pgx.BeginFunc(ctx, s.pool, func(tx pgx.Tx) error {
|
||||||
|
if _, err := tx.Exec(ctx, `insert into message (account, folder, uid, uidvalidity, message_id, in_reply_to, date, sender,
|
||||||
|
recipients, copied, people, subject, size, flags, attachments, body)
|
||||||
|
values ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16)
|
||||||
|
on conflict (account, folder, uid) do update set uidvalidity = excluded.uidvalidity, message_id = excluded.message_id,
|
||||||
|
in_reply_to = excluded.in_reply_to, date = excluded.date, sender = excluded.sender, recipients = excluded.recipients,
|
||||||
|
copied = excluded.copied, people = excluded.people, subject = excluded.subject, size = excluded.size,
|
||||||
|
flags = excluded.flags, attachments = excluded.attachments, body = excluded.body`,
|
||||||
|
m.Account, m.Folder, int64(m.UID), int64(m.UIDValidity), m.MessageID, m.InReplyTo, m.Date, from, to, cc, people,
|
||||||
|
strings.ToValidUTF8(strings.ReplaceAll(m.Subject, "\x00", ""), ""), m.Size, flags, atts,
|
||||||
|
strings.ReplaceAll(m.Body, "\x00", "")); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return keepEvent(ctx, tx, e)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func orNone(ps []Person) []Person {
|
||||||
|
if ps == nil {
|
||||||
|
return []Person{}
|
||||||
|
}
|
||||||
|
return ps
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *PgStore) SetFlags(ctx context.Context, account, folder string, uid uint32, flags []string, e *Event) error {
|
||||||
|
return pgx.BeginFunc(ctx, s.pool, func(tx pgx.Tx) error {
|
||||||
|
if _, err := tx.Exec(ctx, `update message set flags = $4 where account = $1 and folder = $2 and uid = $3`,
|
||||||
|
account, folder, int64(uid), flags); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return keepEvent(ctx, tx, e)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *PgStore) Remove(ctx context.Context, account, folder string, uid uint32, e *Event) error {
|
||||||
|
return pgx.BeginFunc(ctx, s.pool, func(tx pgx.Tx) error {
|
||||||
|
if _, err := tx.Exec(ctx, `delete from message where account = $1 and folder = $2 and uid = $3`, account, folder, int64(uid)); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return keepEvent(ctx, tx, e)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
const messageColumns = `account, folder, uid, uidvalidity, message_id, in_reply_to, date, sender, recipients, copied, subject,
|
||||||
|
size, flags, attachments`
|
||||||
|
|
||||||
|
func scanMessage(row pgx.Row, withBody bool) (Message, error) {
|
||||||
|
var m Message
|
||||||
|
var uid, validity int64
|
||||||
|
var from, to, cc, atts []byte
|
||||||
|
dest := []any{&m.Account, &m.Folder, &uid, &validity, &m.MessageID, &m.InReplyTo, &m.Date, &from, &to, &cc, &m.Subject,
|
||||||
|
&m.Size, &m.Flags, &atts}
|
||||||
|
if withBody {
|
||||||
|
dest = append(dest, &m.Body)
|
||||||
|
}
|
||||||
|
if err := row.Scan(dest...); err != nil {
|
||||||
|
return m, err
|
||||||
|
}
|
||||||
|
m.UID, m.UIDValidity, m.Date = uint32(uid), uint32(validity), m.Date.UTC()
|
||||||
|
for _, f := range []struct {
|
||||||
|
raw []byte
|
||||||
|
into any
|
||||||
|
}{{from, &m.From}, {to, &m.To}, {cc, &m.Cc}, {atts, &m.Attachments}} {
|
||||||
|
if err := json.Unmarshal(f.raw, f.into); err != nil {
|
||||||
|
return m, err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return m, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *PgStore) Get(ctx context.Context, account, folder string, uid uint32) (Message, bool, error) {
|
||||||
|
m, err := scanMessage(s.pool.QueryRow(ctx, `select `+messageColumns+`, body from message where account = $1 and folder = $2 and uid = $3`,
|
||||||
|
account, folder, int64(uid)), true)
|
||||||
|
if errors.Is(err, pgx.ErrNoRows) {
|
||||||
|
return Message{}, false, nil
|
||||||
|
}
|
||||||
|
return m, err == nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *PgStore) Search(ctx context.Context, q Query) ([]Message, error) {
|
||||||
|
limit := q.Limit
|
||||||
|
if limit <= 0 {
|
||||||
|
limit = 20
|
||||||
|
}
|
||||||
|
rows, err := s.pool.Query(ctx, `select `+messageColumns+` from message
|
||||||
|
where account = any($1) and ($2 = '' or folder = $2) and search @@ plainto_tsquery('simple'::regconfig, $3)
|
||||||
|
order by date desc limit $4`, q.Accounts, q.Folder, q.Text, limit)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
defer rows.Close()
|
||||||
|
var out []Message
|
||||||
|
for rows.Next() {
|
||||||
|
m, err := scanMessage(rows, false)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
out = append(out, m)
|
||||||
|
}
|
||||||
|
return out, rows.Err()
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *PgStore) Folders(ctx context.Context, account string) ([]Folder, error) {
|
||||||
|
rows, err := s.pool.Query(ctx, `select name, uidvalidity, messages, synced from folder where account = $1 order by name`, account)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
defer rows.Close()
|
||||||
|
var out []Folder
|
||||||
|
for rows.Next() {
|
||||||
|
f := Folder{Account: account}
|
||||||
|
var validity int64
|
||||||
|
if err := rows.Scan(&f.Name, &validity, &f.Messages, &f.Synced); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
f.UIDValidity, f.Synced = uint32(validity), f.Synced.UTC()
|
||||||
|
out = append(out, f)
|
||||||
|
}
|
||||||
|
return out, rows.Err()
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *PgStore) Pending(ctx context.Context, limit int) ([]Event, error) {
|
||||||
|
rows, err := s.pool.Query(ctx, `select id, key, body from outbox order by id limit $1`, limit)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
defer rows.Close()
|
||||||
|
var out []Event
|
||||||
|
for rows.Next() {
|
||||||
|
var e Event
|
||||||
|
var body []byte
|
||||||
|
if err := rows.Scan(&e.ID, &e.Key, &body); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal(body, &e.Body); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
out = append(out, e)
|
||||||
|
}
|
||||||
|
return out, rows.Err()
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *PgStore) Said(ctx context.Context, ids []int64) error {
|
||||||
|
_, err := s.pool.Exec(ctx, `delete from outbox where id = any($1)`, ids)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *PgStore) SetState(ctx context.Context, st AccountState) error {
|
||||||
|
_, err := s.pool.Exec(ctx, `insert into account_state (account, state, why, heard, synced, said) values ($1, $2, $3, $4, $5, $6)
|
||||||
|
on conflict (account) do update set state = excluded.state, why = excluded.why, heard = excluded.heard,
|
||||||
|
synced = excluded.synced, said = excluded.said`,
|
||||||
|
st.Account, st.State, st.Why, nullTime(st.Heard), nullTime(st.Synced), st.Said)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
func nullTime(t time.Time) *time.Time {
|
||||||
|
if t.IsZero() {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return &t
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *PgStore) States(ctx context.Context) (map[string]AccountState, error) {
|
||||||
|
rows, err := s.pool.Query(ctx, `select account, state, why, heard, synced, said from account_state`)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
defer rows.Close()
|
||||||
|
out := map[string]AccountState{}
|
||||||
|
for rows.Next() {
|
||||||
|
var st AccountState
|
||||||
|
var heard, synced *time.Time
|
||||||
|
if err := rows.Scan(&st.Account, &st.State, &st.Why, &heard, &synced, &st.Said); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if heard != nil {
|
||||||
|
st.Heard = heard.UTC()
|
||||||
|
}
|
||||||
|
if synced != nil {
|
||||||
|
st.Synced = synced.UTC()
|
||||||
|
}
|
||||||
|
st.Said = st.Said.UTC()
|
||||||
|
out[st.Account] = st
|
||||||
|
}
|
||||||
|
return out, rows.Err()
|
||||||
|
}
|
||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user